From 17be9d1ccc27d68eb0a363bdf448183f04ce9cd9 Mon Sep 17 00:00:00 2001 From: luguellis Date: Tue, 7 Jan 2025 11:10:53 -0300 Subject: [PATCH 1/6] feat: script for generating llm files --- .gitignore | 2 + package.json | 5 +- scripts/generate_llms.res | 114 ++++++++++++++++++++++++++++++++++++++ src/bindings/Node.res | 5 +- 4 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 scripts/generate_llms.res diff --git a/.gitignore b/.gitignore index 4b6ddf80c..fc59e854d 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ lib/ src/**/*.mjs scripts/**/*.mjs +public/llms/manual/**/*.txt +public/llms/react/**/*.txt diff --git a/package.json b/package.json index 2d4984b74..2427c247f 100644 --- a/package.json +++ b/package.json @@ -58,10 +58,11 @@ }, "scripts": { "dev": "next", - "build": "rescript && npm run update-index && next build", + "build": "rescript && npm run generate-llms && npm run update-index && next build", "test": "node scripts/test-examples.mjs && node scripts/test-hrefs.mjs", "reanalyze": "reanalyze -all-cmt .", - "update-index": "node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml" + "update-index": "node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml", + "generate-llms": "node scripts/generate_llms.mjs" }, "devDependencies": { "@mdx-js/react": "^2.3.0", diff --git a/scripts/generate_llms.res b/scripts/generate_llms.res new file mode 100644 index 000000000..c7d658a0d --- /dev/null +++ b/scripts/generate_llms.res @@ -0,0 +1,114 @@ +let readMarkdownFile = (filePath: string): string => { + let fileContent = Node.Fs.readFileSync2(filePath, "utf8") + fileContent +} + +let rec collectFiles = (dirPath: string): array => { + let entries = Node.Fs.readdirSync(dirPath) + entries->Array.reduce([], (acc, entry) => { + let fullPath = Node.Path.join([dirPath, entry]) + let stats = Node.Fs.statSync(fullPath) + switch stats["isDirectory"]() { + | true => acc->Array.concat(collectFiles(fullPath)) + | false => { + acc->Array.push(fullPath) + acc + } + } + }) +} + +let clearFile = (filePath: string): unit => { + Node.Fs.writeFileSync(filePath, "") +} + +let createDirectoryIfNotExists = (dirPath: string): unit => { + if !Node.Fs.existsSync(dirPath) { + Node.Fs.mkdirSync(dirPath) + } +} + +let removeCodeTabTags = (content: string): string => { + let regex = Js.Re.fromStringWithFlags("[\\s\\S]*?", ~flags="g") + Js.String.replaceByRe(regex, "", content) +} + +let removeCodeBlocks = (content: string): string => { + let regex = Js.Re.fromStringWithFlags("```[a-zA-Z]+\\s*[\\s\\S]*?```", ~flags="g") + Js.String.replaceByRe(regex, "", content) +} + +let removeFileTitle = (content: string): string => { + let regex = Js.Re.fromStringWithFlags("---\ntitle[\\s\\S]*?---", ~flags="g") + Js.String.replaceByRe(regex, "", content) +} + +let removeUnnecessaryBreaks = (content: string): string => { + let regex = Js.Re.fromStringWithFlags("^\n{2,}", ~flags="g") + Js.String.replaceByRe(regex, "", content) +} + +let removeToDos = (content: string): string => { + let regex = Js.Re.fromStringWithFlags("", ~flags="g") + Js.String.replaceByRe(regex, "", content) +} + +let createFullFile = (content: string, filePath: string): unit => { + Node.Fs.appendFileSync(filePath, content ++ "\n", "utf8") +} + +let createSmallFile = (content: string, filePath: string): unit => { + let smallContent = + content + ->removeCodeTabTags + ->removeFileTitle + ->removeToDos + ->removeCodeBlocks + ->removeUnnecessaryBreaks + Node.Fs.appendFileSync(filePath, smallContent, "utf8") +} + +let processVersions = ( + versions: array, + docsDirectory: string, + llmsDirectory: string, +): unit => { + let fullFileName = "llm-full.txt" + let smallFileName = "llm-small.txt" + let llmsFileName = "llms.txt" + + versions->Array.forEach(version => { + let versionDir = docsDirectory->Node.Path.join2(version) + let llmsDir = llmsDirectory->Node.Path.join2(version) + let fullFilePath = llmsDir->Node.Path.join2(fullFileName) + let smallFilePath = llmsDir->Node.Path.join2(smallFileName) + + createDirectoryIfNotExists(llmsDir) + clearFile(fullFilePath) + clearFile(smallFilePath) + + versionDir + ->collectFiles + ->Array.forEach(filePath => { + if Js.String.endsWith(".mdx", filePath) { + let content = readMarkdownFile(filePath) + + content->createFullFile(fullFilePath) + + content->createSmallFile(smallFilePath) + } + }) + }) +} + +let manualVersions = ["v12.0.0", "v11.0.0"] +let reactManualVersions = ["latest", "v0.10.0", "v0.11.0"] + +let manualDocsDirectory = "pages/docs/manual" +let reactDocsDirectory = "pages/docs/react" + +let manualLlmsDirectory = "public/llms/manual" +let reactLlmsDirectory = "public/llms/react" + +processVersions(manualVersions, manualDocsDirectory, manualLlmsDirectory) +processVersions(reactManualVersions, reactDocsDirectory, reactLlmsDirectory) \ No newline at end of file diff --git a/src/bindings/Node.res b/src/bindings/Node.res index d688f7baa..df3198ee1 100644 --- a/src/bindings/Node.res +++ b/src/bindings/Node.res @@ -20,10 +20,13 @@ module Process = { module Fs = { @module("fs") external readFileSync: string => string = "readFileSync" + @module("fs") external readFileSync2: (string, string) => string = "readFileSync" @module("fs") external readdirSync: string => array = "readdirSync" @module("fs") external writeFileSync: (string, string) => unit = "writeFileSync" + @module("fs") external appendFileSync: (string, string, string) => unit = "appendFileSync" @module("fs") external existsSync: string => bool = "existsSync" @module("fs") external mkdirSync: string => unit = "mkdirSync" + @module("fs") external statSync: string => {.."isDirectory": unit => bool} = "statSync" } module Buffer = { @@ -34,4 +37,4 @@ module Buffer = { module ChildProcess = { @module("child_process") external execSync: string => Buffer.t = "execSync" -} +} \ No newline at end of file From 16a3b2dd7939703952beeed31a6622727f0193ea Mon Sep 17 00:00:00 2001 From: luguellis Date: Mon, 20 Jan 2025 09:32:25 -0300 Subject: [PATCH 2/6] feat: adding mdx files on generating llm --- .gitignore | 7 +++++-- package.json | 4 ++-- public/llms/manual/template.mdx | 23 +++++++++++++++++++++++ public/llms/manual/template.txt | 19 +++++++++++++++++++ public/llms/react/template.mdx | 23 +++++++++++++++++++++++ public/llms/react/template.txt | 19 +++++++++++++++++++ scripts/generate_llms.res | 25 ++++++++++++++++++++++++- 7 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 public/llms/manual/template.mdx create mode 100644 public/llms/manual/template.txt create mode 100644 public/llms/react/template.mdx create mode 100644 public/llms/react/template.txt diff --git a/.gitignore b/.gitignore index fc59e854d..d364e12a7 100644 --- a/.gitignore +++ b/.gitignore @@ -27,5 +27,8 @@ lib/ src/**/*.mjs scripts/**/*.mjs -public/llms/manual/**/*.txt -public/llms/react/**/*.txt + +# Generated via generate-llms script +public/llms/manual/**/llm*.txt +public/llms/react/**/llm*.txt +pages/docs/**/**/llms.mdx diff --git a/package.json b/package.json index 2427c247f..0befba73d 100644 --- a/package.json +++ b/package.json @@ -58,10 +58,10 @@ }, "scripts": { "dev": "next", - "build": "rescript && npm run generate-llms && npm run update-index && next build", + "build": "rescript && npm run update-index && next build", "test": "node scripts/test-examples.mjs && node scripts/test-hrefs.mjs", "reanalyze": "reanalyze -all-cmt .", - "update-index": "node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml", + "update-index": "npm run generate-llms && node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml", "generate-llms": "node scripts/generate_llms.mjs" }, "devDependencies": { diff --git a/public/llms/manual/template.mdx b/public/llms/manual/template.mdx new file mode 100644 index 000000000..975c1a8c3 --- /dev/null +++ b/public/llms/manual/template.mdx @@ -0,0 +1,23 @@ +--- +title: "LLMs" +description: "Documentation for LLMs" +canonical: "/docs/manual//llms" +--- + +# Documentation for LLMs + +We adhere to the [llms.txt convention](https://llmstxt.org/) to make documentation accessible to large language models and their applications. + +Currently, we have the following files... + +- [/docs/manual/llms.txt](/llms/manual//llms.txt) — a list of the available files for ReScript language. +- [/docs/manual/llm-full.txt](/llms/manual//llm-full.txt) — complete documentation for ReScript language. +- [/docs/manual/llm-small.txt](/llms/manual//llm-small.txt) — compressed version of the former, without examples. + +...and package-level documentation: + +- [/docs/react/llms](/docs/react/latest/llms) — the LLms documentation for ReScript React. + +## Notes + +- The content is automatically generated from the same source as the official documentation for the specific version diff --git a/public/llms/manual/template.txt b/public/llms/manual/template.txt new file mode 100644 index 000000000..d4474f163 --- /dev/null +++ b/public/llms/manual/template.txt @@ -0,0 +1,19 @@ +# ReScript Documentation for LLMs + +> ReScript is a robustly typed language that compiles to efficient and human-readable JavaScript. It comes with a lightning fast compiler toolchain that scales to any codebase size. + +## Documentation Sets + +- [Complete documentation](https://rescript-lang.org/llms/manual//llm-full.txt): The complete ReScript documentation including all examples and additional content +- [Abridged documentation](https://rescript-lang.org/llms/manual//llm-small.txt): A minimal version of the ReScript documentation, with the essential content for quick reference + +## Individual Package Documentation + +- [ReScript React documentation](https://rescript-lang.org/llms/react/latest/llms.txt): This is the developer documentation for ReScript React. + +## Notes + +- The abridged documentation excludes the detailed examples, and supplementary information +- The complete documentation includes all content from the official documentation +- Package-specific documentation files contain only the content relevant to that package +- The content is automatically generated from the same source as the official documentation for the specific version \ No newline at end of file diff --git a/public/llms/react/template.mdx b/public/llms/react/template.mdx new file mode 100644 index 000000000..580a23302 --- /dev/null +++ b/public/llms/react/template.mdx @@ -0,0 +1,23 @@ +--- +title: "LLMs" +description: "Documentation for LLMs" +canonical: "/docs/react//llms" +--- + +# Documentation for LLMs + +We adhere to the [llms.txt convention](https://llmstxt.org/) to make documentation accessible to large language models and their applications. + +Currently, we have the following files... + +- [/docs/react/llms.txt](/llms/react//llms.txt) — a list of the available files for ReScript React. +- [/docs/react/llms-full.txt](/llms/react//llm-full.txt) — complete documentation for ReScript React. +- [/docs/react/llms-small.txt](/llms/react//llm-small.txt) — compressed version of the former, without examples for ReScript React. + +...and the language documentation: + +- [/docs/manual/llms](/docs/manual/latest/llms) — the LLms documentation for ReScript. + +## Notes + +- The content is automatically generated from the same source as the official documentation for the specific version diff --git a/public/llms/react/template.txt b/public/llms/react/template.txt new file mode 100644 index 000000000..0023e410a --- /dev/null +++ b/public/llms/react/template.txt @@ -0,0 +1,19 @@ +# ReScript React Documentation for LLMs + +> ReScript React is a strongly typed interface for React, designed to integrate seamlessly with modern React (>= v18.0) while compiling to idiomatic JavaScript, enabling robust and scalable React applications. + +## ReScript React Documentation + +- [Complete documentation](https://rescript-lang.org/llms/react//llm-full.txt): The complete ReScript React documentation including all examples and additional content +- [Abridged documentation](https://rescript-lang.org/llms/react//llm-small.txt): A minimal version of the ReScript React documentation, with the essential content for quick reference + +## Language Documentation + +- [ReScript documentation](https://rescript-lang.org/llms/manual/latest/llms.txt): This is the developer documentation for ReScript. + +## Notes + +- The abridged documentation excludes the detailed examples, and supplementary information +- The complete documentation includes all content from the official documentation +- Package-specific documentation files contain only the content relevant to that package +- The content is automatically generated from the same source as the official documentation for the specific version \ No newline at end of file diff --git a/scripts/generate_llms.res b/scripts/generate_llms.res index c7d658a0d..9077b8172 100644 --- a/scripts/generate_llms.res +++ b/scripts/generate_llms.res @@ -53,6 +53,11 @@ let removeToDos = (content: string): string => { Js.String.replaceByRe(regex, "", content) } +let fillContentWithVersion = (content: string, version: string): string => { + let regex = Js.Re.fromStringWithFlags("", ~flags="g") + Js.String.replaceByRe(regex, version, content) +} + let createFullFile = (content: string, filePath: string): unit => { Node.Fs.appendFileSync(filePath, content ++ "\n", "utf8") } @@ -68,6 +73,23 @@ let createSmallFile = (content: string, filePath: string): unit => { Node.Fs.appendFileSync(filePath, smallContent, "utf8") } +let createLlmsFiles = (version: string, docsDirectory: string, llmsDirectory: string): unit => { + let mdxFileTemplatePath = llmsDirectory->Node.Path.join2("template.mdx") + let mdxFilePath = docsDirectory->Node.Path.join2(version)->Node.Path.join2("llms.mdx") + let txtFileTemplatePath = llmsDirectory->Node.Path.join2("template.txt") + let txtFilePath = llmsDirectory->Node.Path.join2(version)->Node.Path.join2("llms.txt") + + Node.Fs.writeFileSync( + mdxFilePath, + readMarkdownFile(mdxFileTemplatePath)->fillContentWithVersion(version), + ) + + Node.Fs.writeFileSync( + txtFilePath, + readMarkdownFile(txtFileTemplatePath)->fillContentWithVersion(version), + ) +} + let processVersions = ( versions: array, docsDirectory: string, @@ -75,7 +97,6 @@ let processVersions = ( ): unit => { let fullFileName = "llm-full.txt" let smallFileName = "llm-small.txt" - let llmsFileName = "llms.txt" versions->Array.forEach(version => { let versionDir = docsDirectory->Node.Path.join2(version) @@ -87,6 +108,8 @@ let processVersions = ( clearFile(fullFilePath) clearFile(smallFilePath) + createLlmsFiles(version, docsDirectory, llmsDirectory) + versionDir ->collectFiles ->Array.forEach(filePath => { From ba95da96505e0df6a2580fd8866a633de6c3e037 Mon Sep 17 00:00:00 2001 From: luguellis Date: Mon, 20 Jan 2025 09:33:34 -0300 Subject: [PATCH 3/6] feat: adding llms to sidebar + redirects to latest --- data/sidebar_manual_v1100.json | 3 ++- data/sidebar_manual_v1200.json | 3 ++- data/sidebar_react_latest.json | 3 +++ data/sidebar_react_v0100.json | 3 +++ data/sidebar_react_v0110.json | 3 +++ next.config.mjs | 5 +++++ public/_redirects | 2 ++ 7 files changed, 20 insertions(+), 2 deletions(-) diff --git a/data/sidebar_manual_v1100.json b/data/sidebar_manual_v1100.json index 6e6c86255..a7672bb98 100644 --- a/data/sidebar_manual_v1100.json +++ b/data/sidebar_manual_v1100.json @@ -73,6 +73,7 @@ "Extra": [ "newcomer-examples", "project-structure", - "faq" + "faq", + "llms" ] } \ No newline at end of file diff --git a/data/sidebar_manual_v1200.json b/data/sidebar_manual_v1200.json index d783a4eb3..a68a5336c 100644 --- a/data/sidebar_manual_v1200.json +++ b/data/sidebar_manual_v1200.json @@ -72,6 +72,7 @@ "Extra": [ "newcomer-examples", "project-structure", - "faq" + "faq", + "llms" ] } \ No newline at end of file diff --git a/data/sidebar_react_latest.json b/data/sidebar_react_latest.json index 758517284..a2a47d615 100644 --- a/data/sidebar_react_latest.json +++ b/data/sidebar_react_latest.json @@ -31,5 +31,8 @@ "beyond-jsx", "forwarding-refs", "extensions-of-props" + ], + "Extra": [ + "llms" ] } \ No newline at end of file diff --git a/data/sidebar_react_v0100.json b/data/sidebar_react_v0100.json index b6bd3b260..43b20d93e 100644 --- a/data/sidebar_react_v0100.json +++ b/data/sidebar_react_v0100.json @@ -26,5 +26,8 @@ "Guides": [ "beyond-jsx", "forwarding-refs" + ], + "Extra": [ + "llms" ] } \ No newline at end of file diff --git a/data/sidebar_react_v0110.json b/data/sidebar_react_v0110.json index f7e96a22c..0876b0888 100644 --- a/data/sidebar_react_v0110.json +++ b/data/sidebar_react_v0110.json @@ -27,5 +27,8 @@ "beyond-jsx", "forwarding-refs", "extensions-of-props" + ], + "Extra": [ + "llms" ] } \ No newline at end of file diff --git a/next.config.mjs b/next.config.mjs index d8b12ed46..cdc90c275 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -136,6 +136,11 @@ const config = { destination: `/docs/manual/${process.env.VERSION_NEXT}/:slug*`, permanent: false, }, + { + source: "/llms/manual/latest/:file*", + destination: `/llms/manual/${process.env.VERSION_LATEST}/:file*`, + permanent: false, + }, ]; }, }; diff --git a/public/_redirects b/public/_redirects index e0e869325..dae68ed50 100644 --- a/public/_redirects +++ b/public/_redirects @@ -11,3 +11,5 @@ /docs/manual/latest/:slug* /docs/manual/v11.0.0/:slug* 307 /docs/manual/next/:slug* /docs/manual/v12.0.0/:slug* 307 + +/llms/manual/latest/:file* /llms/manual/v11.0.0/:file* 307 From 1e5c2554eccc966ef4916e58681f48e36274377c Mon Sep 17 00:00:00 2001 From: luguellis Date: Mon, 20 Jan 2025 11:05:42 -0300 Subject: [PATCH 4/6] fix: format --- scripts/generate_llms.res | 2 +- src/bindings/Node.res | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate_llms.res b/scripts/generate_llms.res index 9077b8172..fb55bd28e 100644 --- a/scripts/generate_llms.res +++ b/scripts/generate_llms.res @@ -134,4 +134,4 @@ let manualLlmsDirectory = "public/llms/manual" let reactLlmsDirectory = "public/llms/react" processVersions(manualVersions, manualDocsDirectory, manualLlmsDirectory) -processVersions(reactManualVersions, reactDocsDirectory, reactLlmsDirectory) \ No newline at end of file +processVersions(reactManualVersions, reactDocsDirectory, reactLlmsDirectory) diff --git a/src/bindings/Node.res b/src/bindings/Node.res index df3198ee1..44bcd522b 100644 --- a/src/bindings/Node.res +++ b/src/bindings/Node.res @@ -37,4 +37,4 @@ module Buffer = { module ChildProcess = { @module("child_process") external execSync: string => Buffer.t = "execSync" -} \ No newline at end of file +} From a072fcbfdd3a6bf5b979987925713431311bf600 Mon Sep 17 00:00:00 2001 From: luguellis Date: Mon, 20 Jan 2025 11:41:30 -0300 Subject: [PATCH 5/6] fix: changing to RegExp and String Rescript Core types --- scripts/generate_llms.res | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/generate_llms.res b/scripts/generate_llms.res index fb55bd28e..d86c5d9c0 100644 --- a/scripts/generate_llms.res +++ b/scripts/generate_llms.res @@ -29,33 +29,33 @@ let createDirectoryIfNotExists = (dirPath: string): unit => { } let removeCodeTabTags = (content: string): string => { - let regex = Js.Re.fromStringWithFlags("[\\s\\S]*?", ~flags="g") - Js.String.replaceByRe(regex, "", content) + let regex = RegExp.fromStringWithFlags("[\\s\\S]*?", ~flags="g") + String.replaceRegExp(content, regex, "") } let removeCodeBlocks = (content: string): string => { - let regex = Js.Re.fromStringWithFlags("```[a-zA-Z]+\\s*[\\s\\S]*?```", ~flags="g") - Js.String.replaceByRe(regex, "", content) + let regex = RegExp.fromStringWithFlags("```[a-zA-Z]+\\s*[\\s\\S]*?```", ~flags="g") + String.replaceRegExp(content, regex, "") } let removeFileTitle = (content: string): string => { - let regex = Js.Re.fromStringWithFlags("---\ntitle[\\s\\S]*?---", ~flags="g") - Js.String.replaceByRe(regex, "", content) + let regex = RegExp.fromStringWithFlags("---\ntitle[\\s\\S]*?---", ~flags="g") + String.replaceRegExp(content, regex, "") } let removeUnnecessaryBreaks = (content: string): string => { - let regex = Js.Re.fromStringWithFlags("^\n{2,}", ~flags="g") - Js.String.replaceByRe(regex, "", content) + let regex = RegExp.fromStringWithFlags("^\n{2,}", ~flags="g") + String.replaceRegExp(content, regex, "") } let removeToDos = (content: string): string => { - let regex = Js.Re.fromStringWithFlags("", ~flags="g") - Js.String.replaceByRe(regex, "", content) + let regex = RegExp.fromStringWithFlags("", ~flags="g") + String.replaceRegExp(content, regex, "") } let fillContentWithVersion = (content: string, version: string): string => { - let regex = Js.Re.fromStringWithFlags("", ~flags="g") - Js.String.replaceByRe(regex, version, content) + let regex = RegExp.fromStringWithFlags("", ~flags="g") + String.replaceRegExp(content, regex, version) } let createFullFile = (content: string, filePath: string): unit => { @@ -113,7 +113,7 @@ let processVersions = ( versionDir ->collectFiles ->Array.forEach(filePath => { - if Js.String.endsWith(".mdx", filePath) { + if String.endsWith(filePath, ".mdx") { let content = readMarkdownFile(filePath) content->createFullFile(fullFilePath) From faa974c95d928341af563a257e43ebc09228e4cd Mon Sep 17 00:00:00 2001 From: luguellis Date: Mon, 20 Jan 2025 12:38:34 -0300 Subject: [PATCH 6/6] feat: adding redirect for `next` version --- next.config.mjs | 5 +++++ public/_redirects | 1 + 2 files changed, 6 insertions(+) diff --git a/next.config.mjs b/next.config.mjs index cdc90c275..eb31e01f3 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -141,6 +141,11 @@ const config = { destination: `/llms/manual/${process.env.VERSION_LATEST}/:file*`, permanent: false, }, + { + source: "/llms/manual/next/:file*", + destination: `/llms/manual/${process.env.VERSION_NEXT}/:file*`, + permanent: false, + }, ]; }, }; diff --git a/public/_redirects b/public/_redirects index dae68ed50..9c18f7af1 100644 --- a/public/_redirects +++ b/public/_redirects @@ -13,3 +13,4 @@ /docs/manual/next/:slug* /docs/manual/v12.0.0/:slug* 307 /llms/manual/latest/:file* /llms/manual/v11.0.0/:file* 307 +/llms/manual/next/:file* /llms/manual/v12.0.0/:file* 307