docs: add plugin support to generate llms.txt from documentation (#1738)

Author: brandonroberts

apps/docs-app/docs/contributors.mdx

@@ -8,6 +8,8 @@ import styles from './contributors.module.css';
 
 # Contributors
 
+AnalogJS is maintained by a team of contributors and community.
+
 ## Analog core team
 
 ### Brandon Roberts

apps/docs-app/docs/experimental/sfc/index.md

@@ -4,6 +4,8 @@ sidebar_position: 1
 
 # Analog SFCs
 
+Analog SFCs are a new file format for Single File Components (SFCs) that aims to simplify the authoring experience and provide Angular-compatible components and directives.
+
 > **Note:**
 >
 > This file format and API is experimental, is a community-driven initiative, and is not an officially proposed change to Angular. Use it at your own risk.

apps/docs-app/docs/getting-started.md

@@ -7,6 +7,8 @@ import TabItem from '@theme/TabItem';
 
 # Getting Started
 
+Creating an Analog project can be done with minimal steps.
+
 ## System Requirements
 
 Analog requires the following Node and Angular versions:

apps/docs-app/docs/integrations/nx/index.md

@@ -7,12 +7,12 @@ import TabItem from '@theme/TabItem';
 
 # Nx
 
+Analog provides integration with Nx monorepos and workspaces through a workspace preset and an application generator. An Analog application can be created as a standalone project or added to an existing Nx workspace.
+
 ## Overview
 
 [Nx](https://nx.dev) is a smart, fast, extensible build system with first class monorepo support and powerful integrations.
 
-Analog provides integration with Nx monorepos and workspaces through a workspace preset and an application generator. An Analog application can be created as a standalone project or added to an existing Nx workspace.
-
 ## Creating a Standalone Nx project
 
 To scaffold a standalone Nx project, use the `create-nx-workspace` command with the `@analogjs/platform` preset.

apps/docs-app/docusaurus.config.js

@@ -1,5 +1,7 @@
 // @ts-check
 
+import fs from 'node:fs';
+import path from 'node:path';
 import { themes } from 'prism-react-renderer';
 themes.nightOwl['plain'].backgroundColor = '#0a1429';
 
@@ -8,6 +10,8 @@ const projectName = 'analog';
 const title = 'Analog';
 const url = 'https://analogjs.org';
 
+const DOCUSAURUS_BASE_URL = process.env.DOCUSAURUS_BASE_URL ?? '/docs';
+
 /** @type {import('@docusaurus/types').Config} */
 const config = {
   baseUrl: '/',
@@ -50,7 +54,115 @@ const config = {
   onBrokenLinks: 'throw',
   onBrokenMarkdownLinks: 'throw',
   organizationName,
-  plugins: [],
+  plugins: [
+    // Adapted from https://github.com/prisma/docs/blob/22208d52e4168028dbbe8b020b10682e6b526e50/docusaurus.config.ts
+    async function pluginLlmsTxt(context) {
+      return {
+        name: 'llms-txt-plugin',
+        loadContent: async () => {
+          const { siteDir } = context;
+          const contentDir = path.join(siteDir, 'docs');
+          const allMdx = [];
+
+          // recursive function to get all mdx files
+          const getMdFiles = async (dir) => {
+            const entries = await fs.promises.readdir(dir, {
+              withFileTypes: true,
+            });
+
+            for (const entry of entries) {
+              const fullPath = path.join(dir, entry.name);
+              if (entry.isDirectory()) {
+                await getMdFiles(fullPath);
+              } else if (entry.name.endsWith('.md')) {
+                const content = await fs.promises.readFile(fullPath, 'utf8');
+
+                // extract title from frontmatter if it exists
+                const titleMatch = content.match(/^#\s(.*?)$/m);
+
+                const title = titleMatch ? titleMatch[1] : '';
+
+                // Get the relative path for URL construction
+                const relativePath = path.relative(contentDir, fullPath);
+
+                // Convert file path to URL path by:
+                // 1. Removing numeric prefixes (like 100-, 01-, etc.)
+                // 2. Removing the .md extension
+                let urlPath = relativePath
+                  .replace(/^\d+-/, '')
+                  .replace(/\/\d+-/g, '/')
+                  .replace(/index\.md$/, '')
+                  .replace(/\.md$/, '');
+
+                // Construct the full URL
+                const fullUrl = `https://analogjs.org/docs/${urlPath}`;
+
+                // strip frontmatter
+                const contentWithoutFrontmatter = content.replace(
+                  /^---\n[\s\S]*?\n---\n/,
+                  '',
+                );
+
+                // combine title and content with URL
+                const contentWithTitle = title
+                  ? `# ${title}\n\nURL: ${fullUrl}\n${contentWithoutFrontmatter}`
+                  : contentWithoutFrontmatter;
+
+                allMdx.push(contentWithTitle);
+              }
+            }
+          };
+
+          await getMdFiles(contentDir);
+          return { allMdx };
+        },
+        postBuild: async ({ content, routes, outDir }) => {
+          const { allMdx } = content;
+
+          // Write concatenated MDX content
+          const concatenatedPath = path.join(outDir, 'llms-full.txt');
+          await fs.promises.writeFile(
+            concatenatedPath,
+            allMdx.join('\n---\n\n'),
+          );
+
+          // we need to dig down several layers:
+          // find PluginRouteConfig marked by plugin.name === "docusaurus-plugin-content-docs"
+          const docsPluginRouteConfig = routes.filter(
+            (route) => route.plugin.name === 'docusaurus-plugin-content-docs',
+          )[0];
+
+          // docsPluginRouteConfig has a routes property has a record with the path "/" that contains all docs routes.
+          const allDocsRouteConfig = docsPluginRouteConfig.routes?.filter(
+            (route) => route.path === DOCUSAURUS_BASE_URL,
+          )[0];
+
+          // A little type checking first
+          if (!allDocsRouteConfig?.props?.version) {
+            return;
+          }
+
+          // this route config has a `props` property that contains the current documentation.
+          const currentVersionDocsRoutes =
+            allDocsRouteConfig.props.version.docs;
+
+          // for every single docs route we now parse a path (which is the key) and a title
+          const docsRecords = Object.entries(currentVersionDocsRoutes)
+            .filter(([path, rec]) => !!rec.title && !!path)
+            .map(([path, record]) => {
+              return `- [${record.title}](${url}${DOCUSAURUS_BASE_URL}/${path.replace('/index', '')}): ${record.description || record.title}`;
+            });
+
+          // Build up llms.txt file
+          const llmsTxt = `# ${context.siteConfig.title}\n\n## Docs\n\n${docsRecords.join('\n')}\n`;
+
+          // Write llms.txt file
+          const llmsTxtPath = path.join(outDir, 'llms.txt');
+          await fs.promises.writeFile(llmsTxtPath, llmsTxt);
+        },
+      };
+    },
+  ],
   presets: [
     [
       'classic',