`, // `

`, ``, ``, ``, ``… These are NOT custom // components: remark-mdx parses every JSX tag, lowercase or not, into the same // mdxJsxFlowElement/mdxJsxTextElement node types, so without this table they // would fall through to the generic "unknown → unwrap" rule and silently lose // their formatting (bold/italic/inline-code/heading level). // --------------------------------------------------------------------------- const HTML_HEADING_DEPTH = { h1: 1, h2: 2, h3: 3, h4: 4, h5: 5, h6: 6 }; function degradeHtmlTag(node) { const name = node.name.toLowerCase(); const children = node.children || []; if (HTML_HEADING_DEPTH[name]) { return [ { type: "heading", depth: HTML_HEADING_DEPTH[name], children: inlineOnly(children), }, ]; } if (name === "p") return [paragraph(children)]; if (name === "blockquote") return [{ type: "blockquote", children }]; if (name === "strong" || name === "b") return [{ type: "strong", children }]; if (name === "em" || name === "i") return [{ type: "emphasis", children }]; if (name === "del" || name === "s") return [{ type: "delete", children }]; if (name === "code" || name === "kbd") return [{ type: "inlineCode", value: mdastToText(children) }]; if (name === "a") { const attrs = getAttrs(node); return [{ type: "link", url: attrs.href || "", children }]; } if (name === "img") { const attrs = getAttrs(node); const src = resolveDisplayableImageSrc(attrs.src); if (src) return [{ type: "image", url: src, alt: attrs.alt || "" }]; // Relative/require()'d src can't be resolved to a real URL (see // resolveDisplayableImageSrc) — keep the alt text rather than vanish // entirely, since a bare has no children to fall back to. return attrs.alt ? [textNode(attrs.alt)] : []; } if (name === "br") return [{ type: "break" }]; if (name === "hr") return [{ type: "thematicBreak" }]; // div, span, section, article, and any other purely structural tag: unwrap. return children; } // --------------------------------------------------------------------------- // Tree walk: bottom-up so a nested custom component is already resolved // before its parent inspects `node.children`. // --------------------------------------------------------------------------- function transformChildren(children, ctx) { const out = []; for (const child of children || []) { out.push(...transformNode(child, ctx)); } return out; } function transformNode(node, ctx) { if (node.children) { node.children = transformChildren(node.children, ctx); } if (node.type === "mdxJsxFlowElement" || node.type === "mdxJsxTextElement") { return degradeComponent(node, ctx); } if (node.type === "mdxFlowExpression" || node.type === "mdxTextExpression") { // A bare {expression} in prose, outside any attribute. None exist in this // corpus outside of code fences (verified) — best-effort fallback rather // than a crash if one ever appears. return typeof node.value === "string" && node.value.trim() ? [textNode(node.value)] : []; } if (node.type === "mdxjsEsm") { return []; // import/export statements — not content. } if (node.type === "yaml") { return []; // frontmatter — the caller writes its own header instead. } if (node.type === "comment") { return []; // `` and any other raw HTML comment — not content. } if (node.type === "containerDirective" || node.type === "leafDirective") { return degradeDirective(node); } if (node.type === "textDirective") { // remark-directive's inline `:name` syntax has no word-boundary check, so // ordinary prose like "user_id:group_id", "github.com:company/x.git" or // "nginx:latest" tokenizes as a textDirective too — and every occurrence // ever found in this corpus has been exactly that, never a real directive // (this blog's own `:::tip`-style directives go through containerDirective/ // leafDirective above, not here). Not worth a per-build warning: reconstruct // the exact original characters from the source, silently. Unlike // containerDirective, a directive with no `[label]` has *no children at // all* to fall back to, so keeping `node.children` here would silently // delete the word after the colon instead. if (node.position && typeof ctx.rawSource === "string") { return [ textNode( ctx.rawSource.slice(node.position.start.offset, node.position.end.offset), ), ]; } return node.children || []; } return [node]; } function degradeDirective(node) { // Docusaurus's admonitions plugin (when available — see the try/catch at the // top of this file) annotates the directive with the resolved type/title // instead of changing its node type, so `node.type` is still "containerDirective" // here even for a fully recognised `:::tip[…]`. const type = node.data?.hProperties?.type || node.name; const title = node.data?.hProperties?.title; const label = title || ALERT_LABELS[type] || (type ? type[0].toUpperCase() + type.slice(1) : "Note"); return [ { type: "blockquote", children: [paragraph([strongText(`${label}:`)]), ...(node.children || [])], }, ]; } function degradeComponent(node, ctx) { const name = node.name; if (!name) { // React fragment <>… return node.children || []; } const isHtmlTag = name[0] === name[0].toLowerCase(); if (isHtmlTag) { return degradeHtmlTag(node); } const handler = COMPONENT_RULES[name]; if (handler) { return handler(node, ctx) || []; } // The one rule that can never be skipped: keep the content, drop the // wrapper, and tell the caller so it can warn about the gap. ctx.unknownComponents.add(name); return node.children || []; } // --------------------------------------------------------------------------- // Public entry point // --------------------------------------------------------------------------- // Parse-only processor, always format:"mdx" regardless of the file's own .md/.mdx // extension — see the createProcessor import comment above for why. Never call // `.run()`/`.compile()`/`.process()` on this one: createProcessor bakes in the // full remark→rehype→recma→JS-estree compile chain, and `.parse()` is the only // method that stops cleanly at the mdast stage this file actually wants. function buildParser() { return createProcessor({ format: "mdx" }) .use(remarkFrontmatter) .use(remarkGfm) .use(remarkComment) .use(remarkDirective); } // Separate, bare processor: `.run()`-only, used solely to apply Docusaurus's // own admonitions AST transform (`:::tip[…]` → data.hProperties) to the tree // `buildParser` produced. Kept off the parser above because a bare unified() // has no recma/JS-compile stage to accidentally trigger. const directiveTransformer = admonitionsPlugin ? unified().use(admonitionsPlugin, true) : null; const stringifier = unified().use(remarkStringify).use(remarkGfm); /** * @param {string} rawSource - the full .md/.mdx file content, frontmatter included. * @param {object} opts * @param {string} opts.currentFileDir - absolute directory of the article (for * resolving relative `source="./…"` attributes). * @param {string} [opts.projectRoot] - defaults to process.cwd(). * @returns {Promise<{ markdown: string, unknownComponents: string[] }>} */ async function mdxToMarkdown(rawSource, { currentFileDir, projectRoot = process.cwd() }) { let preprocessed = escapeMarkdownHeadingIds(rawSource); if (admonitionsPlugin) { preprocessed = admonitionTitleToDirectiveLabel(preprocessed, ADMONITION_KEYWORDS); } const tree = buildParser().parse(preprocessed); if (directiveTransformer) { await directiveTransformer.run(tree); } const ctx = { currentFileDir, projectRoot, rawSource: preprocessed, unknownComponents: new Set(), }; tree.children = transformChildren(tree.children, ctx); const markdown = stringifier.stringify(tree); return { markdown, unknownComponents: [...ctx.unknownComponents].sort() }; } module.exports = { mdxToMarkdown }; ```` ### The orchestrator The plugin runs in `postBuild`, degrades each live article, writes the mirrors, then builds the two indexes on top of whatever succeeded: **plugins/markdown-export-plugin/index.cjs** ```cjs /** * @fileoverview Docusaurus Plugin: markdown-export-plugin * * Purpose: * Writes a plain-Markdown mirror of every published blog post next to its HTML * page (`/blog/` → `/blog/.md`), plus a site-wide `/llms.txt` index * and a per-series `/llms/.txt` bundle — so a reader (human or an * LLM) can fetch the article's full content without React, JSX, or the 107 * `defaultOpen={false}` accordions this corpus collapses in the HTML. * * The actual MDX → Markdown degradation lives in ./degrade.cjs; this file only * orchestrates: find the posts that really shipped, degrade each one, write * the mirrors, then build the two kinds of index on top of what didn't fail. * * Usage: * - Add to docusaurus.config.js `plugins` array: `"./plugins/markdown-export-plugin/index.cjs"` * - Runs only in `postBuild` (i.e. only during `yarn build`, never `yarn start`), * same as every other postBuild plugin in this repo. */ const fs = require("fs"); const path = require("path"); const frontMatter = require("front-matter"); const { mdxToMarkdown } = require("./degrade.cjs"); const BLOG_DIR = "blog"; /** Recursively collects every article file under a directory (mirrors the same * pattern used by scripts/internal-link-opportunities.mjs and blog-feed-plugin). */ function findPosts(dir) { const found = []; for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { const target = path.join(dir, entry.name); if (entry.isDirectory()) { found.push(...findPosts(target)); } else if (entry.name === "index.md" || entry.name === "index.mdx") { found.push(target); } } return found; } // Mirrors src/components/Blog/utils/slug.ts's createSlug() exactly. Duplicated // rather than imported: that file is ESM written for Webpack/browser bundling // (`export function`), which a plain Node `require()` here cannot load without // a bundler. Keep both in sync if the slugify algorithm ever changes. function createSlug(text) { return text .replace(/([a-z0-9])([A-Z])/g, "$1-$2") .toLowerCase() .normalize("NFD") .replace(/[\u0300-\u036f]/g, "") // strips diacritics .replace(/[^a-z0-9\s-]/g, "") .replace(/\s+/g, "-") .replace(/-+/g, "-") .replace(/^-+|-+$/g, ""); } // Mirrors src/components/Blog/utils/posts.ts's permalink derivation exactly — // same "why duplicated" rationale as createSlug above. function permalinkFor(attributes, dir) { if (attributes.slug) { return attributes.slug.startsWith("/") ? attributes.slug : `/blog/${attributes.slug.replace(/^\//, "")}`; } return `/blog/${dir}/`; } /** * @param {string} siteDir * @param {string} [translationDir] when building a non-default locale, the matching * `i18n//docusaurus-plugin-content-blog` tree. Any article that has a file there is * read FROM there, so the mirror and the llms.txt index carry the translated title, * description and body. Without it, `/fr/blog/.md` served the English Markdown under a * French URL and `/fr/llms.txt` advertised English titles — the content equivalent of the * duplicate-content problem plugins/i18n-seo-guard solves for HTML. See TODO 0119. */ function loadPosts(siteDir, translationDir = null) { const blogDir = path.join(siteDir, BLOG_DIR); const posts = []; for (const file of findPosts(blogDir)) { let sourceFile = file; if (translationDir) { const relative = path.relative(blogDir, file); const translated = path.join(translationDir, relative); if (fs.existsSync(translated)) sourceFile = translated; } const raw = fs.readFileSync(sourceFile, "utf-8"); const { attributes } = frontMatter(raw); // A Markdown file under blog/ without a title isn't an article (a // fragment, a note) — same rule posts.js applies. if (!attributes.title) continue; const dir = path.relative(blogDir, path.dirname(file)).split(path.sep).join("/"); posts.push({ file, raw, // Deliberately the ENGLISH article's folder, even when `raw` came from the translation: // `files/` and `images/` are co-located with the source, never duplicated under i18n/. currentFileDir: path.dirname(file), permalink: permalinkFor(attributes, dir), title: attributes.title, description: attributes.description || "", date: attributes.date, mainTag: attributes.mainTag || null, series: attributes.series || null, draft: attributes.draft || false, unlisted: attributes.unlisted || false, }); } return posts; } /** * Absolute URL for a site-relative path, honouring the locale's baseUrl. * * `siteConfig.url` is the bare origin and `siteConfig.baseUrl` carries the locale segment * (`/` for the default locale, `/fr/` for French). Concatenating `url + permalink` skips the * locale entirely, so the French llms.txt advertised English URLs for translated articles. * See TODO 0119. */ function absoluteUrl(siteConfig, sitePath) { const base = siteConfig.baseUrl.endsWith("/") ? siteConfig.baseUrl : `${siteConfig.baseUrl}/`; return `${siteConfig.url}${base}${String(sitePath).replace(/^\//, "")}`; } // YAML frontmatter dates (`date: 2024-02-23`) are auto-typed as JS Date // objects by the YAML parser front-matter uses — string-concatenating one // directly calls its verbose default toString(). Normalize to plain // YYYY-MM-DD instead. function formatDate(date) { if (!date) return "—"; return date instanceof Date ? date.toISOString().slice(0, 10) : String(date); } // The comment carries the same info a human reader would otherwise lose once // this file is copied or pasted elsewhere: where the rendered original lives // (images, interactive components, the lot). Kept as an HTML comment — invisible // in Markdown, so a reader pasting the file into a renderer isn't shown a stray // metadata block, while it still reads as plain text in a raw view or in // whatever an LLM is handed. // // Deliberately no build timestamp. One wall-clock value rewrote all 247 mirrors // plus the 25 llms bundles on every single build (5.2 MB), which the deploy then // had to re-upload even when no article had changed. The prose below already // says the file is generated at build time, and `published` carries the date a // reader actually needs. function buildMetadataComment(post, siteConfig) { const url = absoluteUrl(siteConfig, post.permalink); return [ "", "", ].join("\n"); } function buildHeader(post, siteConfig) { const lines = [buildMetadataComment(post, siteConfig), `# ${post.title}`, ""]; if (post.description) { lines.push(`> ${post.description}`, ""); } lines.push("---", ""); return lines.join("\n"); } function writeLlmsTxt(outDir, siteConfig, posts, seriesFiles) { const byTag = new Map(); for (const post of posts) { const tag = post.mainTag || "uncategorized"; if (!byTag.has(tag)) byTag.set(tag, []); byTag.get(tag).push(post); } const lines = [`# ${siteConfig.title}`, ""]; if (siteConfig.tagline) lines.push(`> ${siteConfig.tagline}`, ""); lines.push( `${posts.length} articles. Each has a plain-Markdown mirror at its own permalink ` + `plus \`.md\` (e.g. \`${absoluteUrl(siteConfig, "/blog/.md")}\`).`, "", ); // This is the only place the per-series bundles are ever linked from — without // it they exist on disk but are unreachable from anything an LLM would fetch. if (seriesFiles.length > 0) { lines.push( "## Series (full-text bundles)", "", "Each link is every article in that series, concatenated in reading order — " + "one request instead of many.", "", ); for (const s of seriesFiles) { lines.push( `- [${s.name}](${absoluteUrl(siteConfig, `/llms/${s.slug}.txt`)}) — ${s.count} article(s)`, ); } lines.push(""); } for (const tag of [...byTag.keys()].sort()) { lines.push(`## ${tag}`, ""); const sorted = [...byTag.get(tag)].sort((a, b) => a.title.localeCompare(b.title)); for (const post of sorted) { const desc = post.description ? ` — ${post.description}` : ""; lines.push(`- [${post.title}](${absoluteUrl(siteConfig, post.permalink)})${desc}`); } lines.push(""); } fs.writeFileSync(path.join(outDir, "llms.txt"), lines.join("\n"), "utf-8"); } // Returns the series that were actually written, sorted by name — this list is // what lets writeLlmsTxt() link to them; without it they'd be orphaned on disk. function writeSeriesFull(outDir, posts) { const bySeries = new Map(); for (const post of posts) { if (!post.series) continue; if (!bySeries.has(post.series)) bySeries.set(post.series, []); bySeries.get(post.series).push(post); } if (bySeries.size === 0) return []; const seriesDir = path.join(outDir, "llms"); fs.mkdirSync(seriesDir, { recursive: true }); const written = []; for (const [seriesName, seriesPosts] of bySeries) { const sorted = [...seriesPosts].sort((a, b) => new Date(a.date) - new Date(b.date)); const chunks = [ `# ${seriesName}`, "", `A series of ${sorted.length} article(s), in reading order.`, "", ]; for (const post of sorted) { chunks.push(post.markdown, "", "---", ""); } const slug = createSlug(seriesName); fs.writeFileSync(path.join(seriesDir, `${slug}.txt`), chunks.join("\n"), "utf-8"); written.push({ name: seriesName, slug, count: sorted.length }); } return written.sort((a, b) => a.name.localeCompare(b.name)); } module.exports = function markdownExportPlugin() { return { name: "markdown-export-plugin", async postBuild({ outDir, siteDir, siteConfig, routesPaths, i18n }) { const currentLocale = i18n?.currentLocale; const defaultLocale = i18n?.defaultLocale ?? siteConfig.i18n?.defaultLocale; const isTranslatedLocale = Boolean( currentLocale && currentLocale !== defaultLocale, ); const posts = loadPosts( siteDir, isTranslatedLocale ? path.join(siteDir, "i18n", currentLocale, "docusaurus-plugin-content-blog") : null, ); // In a non-default locale, mirror and index ONLY the articles really translated into it. // `loadPosts()` reads `blog/`, i.e. the English corpus, and Docusaurus's i18n fallback // gives every one of them a live `/fr/` route — so without this, `/fr/llms.txt` would // advertise 257 English articles as the French corpus, and `/fr/blog/.md` would // mirror English prose under a French URL. See TODO 0119. let localeFiltered = posts; if (isTranslatedLocale) { const { collectTranslations, } = require("../translations-manifest-plugin/index.cjs"); const translated = new Set(collectTranslations(siteDir)[currentLocale] ?? []); // These post objects carry a `permalink`, not a `slug` — assuming the latter silently // filtered everything out (`translated.has(undefined)` is always false). localeFiltered = posts.filter((post) => { const match = String(post.permalink ?? "").match(/blog\/([^/]+)\/?$/); return match ? translated.has(match[1]) : false; }); console.log( `[markdown-export] locale ${currentLocale}: ${localeFiltered.length} translated article(s).`, ); } // Only mirror posts that actually got a live route — this is what makes // "draft" (excluded in prod by Docusaurus's own blog plugin) and // ".unpublished/" (outside blog/, never globbed here anyway) fall out // for free, without re-deriving Docusaurus's own draft rules. // `routesPaths` carries the locale's baseUrl (`/fr/blog//`) while `permalinkFor()` // builds a bare `/blog/` — comparing them directly matched nothing under a // non-default locale, so every mirror silently disappeared while the build stayed green. // This is the mirror image of the bug in blog-feed-plugin's getArticleHtml(), where the // prefix was present twice instead of missing once. See TODO 0119. const baseUrl = siteConfig.baseUrl.endsWith("/") ? siteConfig.baseUrl : `${siteConfig.baseUrl}/`; const withBaseUrl = (permalink) => `${baseUrl}${permalink.replace(/^\//, "")}`.replace(/\/$/, ""); const liveRoutes = new Set(routesPaths.map((route) => route.replace(/\/$/, ""))); const live = localeFiltered.filter((post) => liveRoutes.has(withBaseUrl(post.permalink)), ); console.log( `[markdown-export] Mirroring ${live.length}/${posts.length} blog posts to Markdown…`, ); const unknownAll = new Set(); const indexable = []; let failed = 0; for (const post of live) { let markdown; try { const result = await mdxToMarkdown(post.raw, { currentFileDir: post.currentFileDir, projectRoot: siteDir, }); markdown = buildHeader(post, siteConfig) + result.markdown; for (const name of result.unknownComponents) unknownAll.add(name); } catch (err) { // One broken article must never take the whole `yarn build` down — // the "one rule" in degrade.cjs already prevents almost every crash; // this is the last-resort net for whatever it doesn't catch. failed += 1; console.error( `[markdown-export] Failed to degrade ${post.file}: ${err.message}`, ); continue; } const mirrorPath = path.join(outDir, `${post.permalink.replace(/\/$/, "")}.md`); fs.mkdirSync(path.dirname(mirrorPath), { recursive: true }); fs.writeFileSync(mirrorPath, markdown, "utf-8"); if (!post.draft && !post.unlisted) { indexable.push({ ...post, markdown }); } } const seriesFiles = writeSeriesFull(outDir, indexable); writeLlmsTxt(outDir, siteConfig, indexable, seriesFiles); if (unknownAll.size > 0) { console.warn( `[markdown-export] ${unknownAll.size} component(s) had no explicit degrade rule and fell back ` + `to the generic "keep children, drop wrapper" behavior — add a rule in ` + `plugins/markdown-export-plugin/degrade.cjs's COMPONENT_RULES if this is a real gap:\n` + [...unknownAll] .sort() .map((name) => ` - ${name}`) .join("\n"), ); } console.log( `[markdown-export] Done — ${indexable.length} article(s) in /llms.txt, ` + `${live.length - failed}/${live.length} mirrored, ${failed} failed.`, ); }, }; }; ``` Register it like any other plugin: ```javascript title="docusaurus.config.js" plugins: [ "./plugins/markdown-export-plugin/index.cjs", // ... ], ``` > **Self-hosted? Watch the MIME type:** > > `/blog/my-slug` (like `/blog/docusaurus-llms-txt` for this article) is a *directory* containing `index.html`, so its sibling `/blog/my-slug.md` creates no route conflict — that one is free. But Apache has no built-in MIME mapping for `.md`, so without `AddType text/markdown .md` in `.htaccess`, the browser offers a download instead of displaying the file. ## The One Rule That Makes This Safe **An unknown component never fails the export. Its wrapper is dropped and its children take its place.** No exceptions, no errors, ever. That single rule is what allows a 900-line degradation table to sit in a build pipeline without becoming a liability. Add a component in six months, forget to write a rule, and the worst case is that its wrapper vanishes while its content survives. But a silent fallback is a slow leak, so the plugin records every component it did not recognize and warns once per build. **That warning, not the table, is what keeps coverage at 100% over time** — the table is frozen the day you write it; the warning notices the day you break it. ## Making It Discoverable Generating a file is not publishing it: without a link pointing at it, nothing — crawler or human — ever finds `/llms.txt` or a per-series bundle on its own. Four hooks close that gap, each targeting a different consumer: **1. `llms.txt` links its own bundles.** The index opens with a "Series (full-text bundles)" section. It is the only place they are ever referenced. **2. A site-wide ``,** in `headTags`, so it is server-rendered on every page — exactly how RSS has always been discovered: ```javascript title="docusaurus.config.js" { tagName: "link", attributes: { rel: "alternate", type: "text/markdown", href: "https://www.avonture.be/llms.txt", title: "llms.txt — full site index in Markdown, for LLMs and readers", }, }, ``` **3. A per-article equivalent,** pointing at that article's own mirror, injected by a tiny component wired into the blog post page: **src/components/MarkdownAlternate/index.tsx** ```typescript // Points crawlers/tools that scan at this exact // article's plain-Markdown mirror (written by plugins/markdown-export-plugin, // see src/components/CopyAsMarkdown for the human-facing equivalent). Same // discovery pattern as RSS's , // scoped to a single page instead of the whole site — the site-wide // counterpart (pointing at /llms.txt) lives in docusaurus.config.js's // headTags, since that one applies to every page, not just blog posts. import type { JSX } from "react"; import Head from "@docusaurus/Head"; import { useBlogPost } from "@docusaurus/plugin-content-blog/client"; import { translate } from "@docusaurus/Translate"; export default function MarkdownAlternate(): JSX.Element { const { metadata } = useBlogPost(); return ( ); } ``` **4. A breadcrumb in `robots.txt`.** There is no standard directive for this — it is a comment, read by anything already parsing that file and looking for one. It cost two lines. > **One honest limitation:** > > The human-facing "View this series as plain Markdown" link on `/series/` is invisible to a crawler that does not run JavaScript: that page is a client-side React Router route, so its link never appears in server-rendered HTML. The two `` tags above do not have that problem — which is exactly why the discovery hooks that matter live in `headTags` and in a component, not in a page body. And then there is the part no code can do for you: **submitting to the directories**. The correct ones are the ones the official spec page links to. I got `llmstxt.site` and `directory.llmstxt.cloud` registered; `llmstxthub.com` had a broken submission form the day I tried. Worth ten minutes. ## Conclusion What I like about the result has less to do with AI adoption than I expected going in: * **Anyone can `curl` an article and get it whole.** Not just a bot — a script, a colleague, me on a slow connection — gets the full text with no HTML, no JavaScript, no navigation to strip out by hand first. * **The blog is bot-friendly regardless of whether `llms.txt` itself ever catches on.** A plain-Markdown version of every article now exists as a side effect, independent of that convention's fate. * **A Markdown file drops straight into a text editor or an LLM's context window.** No more select-all-and-delete-the-navbar before an article is actually usable somewhere else. * **`/llms.txt` doubles as an inventory of the blog** — every article, one line each, grouped by topic. A table of contents I did not have before, generated for free. * **The per-series bundles turn 23 requests into one.** Handy for an assistant pulling context, just as handy for a reader who wants an entire series in a single file to read or archive. * **None of it needs to be remembered.** It is a build step, not a habit — which is exactly why an unproven convention was worth one hour to try in the first place. If the 107 collapsed [``](/blog/docusaurus-snippets) accordions that started this whole thing are new to you, that component is worth its own read.