Math and graph integration into Docs (haXathon)

.gitignore

@@ -76,3 +76,7 @@ db.sqlite3
 .vscode/
 *.iml
 .devcontainer
+
+**/package-lock.json
+src/frontend/package-lock.json
+src/frontend/yarn.lock

src/frontend/apps/impress/package.json

@@ -22,17 +22,21 @@
     "@blocknote/react": "0.31.1",
     "@blocknote/xl-docx-exporter": "0.31.1",
     "@blocknote/xl-pdf-exporter": "0.31.1",
+    "@cortex-js/compute-engine": "^0.29.1",
     "@emoji-mart/data": "1.2.1",
     "@emoji-mart/react": "1.1.1",
     "@fontsource/material-icons": "5.2.5",
     "@gouvfr-lasuite/integration": "1.0.3",
     "@gouvfr-lasuite/ui-kit": "0.7.0",
     "@hocuspocus/provider": "2.15.2",
+    "@monaco-editor/react": "^4.7.0",
     "@openfun/cunningham-react": "3.1.0",
     "@react-pdf/renderer": "4.3.0",
     "@sentry/nextjs": "9.22.0",
     "@tanstack/react-query": "5.77.1",
-    "canvg": "4.0.3",
+    "@types/katex": "^0.16.7",
+    "canvg": "^4.0.3",
+    "chart.js": "^4.4.9",
     "clsx": "2.1.1",
     "cmdk": "1.1.1",
     "crisp-sdk-web": "1.0.25",
@@ -41,17 +45,22 @@
     "i18next": "25.2.1",
     "i18next-browser-languagedetector": "8.1.0",
     "idb": "8.0.3",
+    "katex": "^0.16.22",
     "lodash": "4.17.21",
     "luxon": "3.6.1",
+    "mermaid": "^11.6.0",
     "next": "15.3.2",
     "posthog-js": "1.246.0",
     "react": "*",
     "react-aria-components": "1.9.0",
+    "react-chartjs-2": "^5.3.0",
     "react-dom": "*",
     "react-i18next": "15.5.2",
     "react-intersection-observer": "9.16.0",
     "react-select": "5.10.1",
+    "shiki": "^3.4.2",
     "styled-components": "6.1.18",
+    "tex-to-svg": "^0.2.0",
     "use-debounce": "10.0.4",
     "y-protocols": "1.0.6",
     "yjs": "*",

src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx

@@ -2,13 +2,14 @@ import { codeBlock } from '@blocknote/code-block';
 import {
   BlockNoteSchema,
   defaultBlockSpecs,
+  defaultInlineContentSpecs,
   withPageBreak,
 } from '@blocknote/core';
 import '@blocknote/core/fonts/inter.css';
 import * as locales from '@blocknote/core/locales';
 import { BlockNoteView } from '@blocknote/mantine';
 import '@blocknote/mantine/style.css';
-import { useCreateBlockNote } from '@blocknote/react';
+import { SuggestionMenuController, useCreateBlockNote } from '@blocknote/react';
 import { HocuspocusProvider } from '@hocuspocus/provider';
 import { useEffect } from 'react';
 import { useTranslation } from 'react-i18next';
@@ -27,14 +28,35 @@ import { randomColor } from '../utils';
 
 import { BlockNoteSuggestionMenu } from './BlockNoteSuggestionMenu';
 import { BlockNoteToolbar } from './BlockNoteToolBar/BlockNoteToolbar';
-import { CalloutBlock, DividerBlock } from './custom-blocks';
+import parseMarkdownWithLatex from './BlockNoteToolBar/utils';
+import { InlineLatex } from './InlineLatex/';
+import getInlineLatexMenuItems from './InlineLatex/components/InlineLatexContentSpec';
+import { useLatexDetection } from './InlineLatex/hooks/useLatexDetection';
+import {
+  CalloutBlock,
+  DividerBlock,
+  LatexBlock,
+  MermaidBlock,
+  ChartBlock,
+  LatexAIBlock,
+  MermaidAIBlock
+} from './custom-blocks';
 
 export const blockNoteSchema = withPageBreak(
   BlockNoteSchema.create({
     blockSpecs: {
       ...defaultBlockSpecs,
       callout: CalloutBlock,
       divider: DividerBlock,
+      latex: LatexBlock,
+      mermaid: MermaidBlock,
+      chart: ChartBlock,
+      latexai: LatexAIBlock,
+      mermaidai: MermaidAIBlock
+    },
+    inlineContentSpecs: {
+      ...defaultInlineContentSpecs,
+      inlineLatex: InlineLatex,
     },
   }),
 );
@@ -63,6 +85,8 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => {
     : user?.full_name || user?.email || t('Anonymous');
   const showCursorLabels: 'always' | 'activity' | (string & {}) = 'activity';
 
+  let editorWillPaste = false;
+
   const editor: DocsBlockNoteEditor = useCreateBlockNote(
     {
       codeBlock,
@@ -124,12 +148,35 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => {
       },
       uploadFile,
       schema: blockNoteSchema,
+      pasteHandler: ({ event, editor }) => {
+        const text = event.clipboardData?.getData('text/plain');
+        if (text) {
+          editorWillPaste = true;
+          void editor.pasteMarkdown(text);
+        }
+        return true;
+      },
     },
     [collabName, lang, provider, uploadFile],
   );
 
+  editor.onChange((editor, { getChanges }) => {
+    if (editorWillPaste) {
+      editorWillPaste = false;
+
+      const changes = getChanges();
+      changes.forEach((change) => {
+        if (change.type === 'update' || change.type === 'insert') {
+          const [newBlock] = parseMarkdownWithLatex([change.block]);
+          editor.updateBlock(change.block.id, newBlock);
+        }
+      });
+    }
+  });
+
   useHeadings(editor);
   useUploadStatus(editor);
+  useLatexDetection(editor);
 
   useEffect(() => {
     setEditor(editor);
@@ -164,7 +211,13 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => {
         theme="light"
       >
         <BlockNoteSuggestionMenu />
+
         <BlockNoteToolbar />
+
+        <SuggestionMenuController
+          triggerCharacter="$"
+          getItems={async (query) => getInlineLatexMenuItems(editor, query)}
+        />
       </BlockNoteView>
     </Box>
   );
@@ -206,6 +259,10 @@ export const BlockNoteEditorVersion = ({
   return (
     <Box $css={cssEditor(readOnly)} className="--docs--editor-container">
       <BlockNoteView editor={editor} editable={!readOnly} theme="light" />
+      <SuggestionMenuController
+        triggerCharacter="$"
+        getItems={async (query) => getInlineLatexMenuItems(editor)}
+      />
     </Box>
   );
 };

src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteSuggestionMenu.tsx

@@ -14,6 +14,11 @@ import { DocsBlockSchema } from '../types';
 import {
   getCalloutReactSlashMenuItems,
   getDividerReactSlashMenuItems,
+  getLatexReactSlashMenuItems,
+  getMermaidReactSlashMenuItems,
+  getChartReactSlashMenuItems,
+  getLatexAIReactSlashMenuItems,
+  getMermaidAIReactSlashMenuItems
 } from './custom-blocks';
 
 export const BlockNoteSuggestionMenu = () => {
@@ -30,6 +35,11 @@ export const BlockNoteSuggestionMenu = () => {
             getPageBreakReactSlashMenuItems(editor),
             getCalloutReactSlashMenuItems(editor, t, basicBlocksName),
             getDividerReactSlashMenuItems(editor, t, basicBlocksName),
+            getLatexReactSlashMenuItems(editor, t, basicBlocksName),
+            getMermaidReactSlashMenuItems(editor, t, basicBlocksName),
+            getChartReactSlashMenuItems(editor, t, basicBlocksName),
+            getLatexAIReactSlashMenuItems(editor, t, basicBlocksName),
+            getMermaidAIReactSlashMenuItems(editor, t, basicBlocksName),
           ),
           query,
         ),

src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteToolBar/BlockNoteToolbar.tsx

@@ -10,6 +10,7 @@ import { useTranslation } from 'react-i18next';
 
 import { useConfig } from '@/core/config/api';
 
+import { InlineLatexButton } from '../InlineLatex/components/InlineLatexButton';
 import { getCalloutFormattingToolbarItems } from '../custom-blocks';
 
 import { AIGroupButton } from './AIButton';
@@ -58,6 +59,9 @@ export const BlockNoteToolbar = () => {
       <FormattingToolbar>
         {toolbarItems}
 
+        {/* Extra button to insert latex */}
+        <InlineLatexButton key="InlineLatexButton" />
+
         {/* Extra button to do some AI powered actions */}
         {conf?.AI_FEATURE_ENABLED && <AIGroupButton key="AIButton" />}
 

src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteToolBar/MarkdownButton.tsx

@@ -7,6 +7,8 @@ import { forEach, isArray } from 'lodash';
 import React, { useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 
+import parseMarkdownWithLatex from './utils';
+
 type Block = {
   type: string;
   text: string;
@@ -63,7 +65,9 @@ export function MarkdownButton() {
 
         const blockMarkdown =
           await editor.tryParseMarkdownToBlocks(fullContent);
-        editor.replaceBlocks([block.id], blockMarkdown);
+        const blockMarkdownWithLatex = parseMarkdownWithLatex(blockMarkdown);
+
+        editor.replaceBlocks([block.id], blockMarkdownWithLatex);
       } catch (error) {
         console.error('Error parsing Markdown:', error);
       }

src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteToolBar/utils.ts

@@ -0,0 +1,105 @@
+/* eslint-disable prettier/prettier */
+/* eslint-disable @typescript-eslint/restrict-plus-operands */
+import { Block, StyleSchema, StyledText } from '@blocknote/core';
+
+export default function parseMarkdownWithLatex(blocks: Block[]): Block[] {
+  return blocks.map((block) => {
+    if (
+      (block.type === 'paragraph' ||
+        block.type == 'bulletListItem' ||
+        block.type == 'numberedListItem' ||
+        block.type == 'heading') &&
+      block.content
+    ) {
+      const newContent: StyledText<StyleSchema>[] = [];
+      let currentText = '';
+      let isInLatex = false;
+      let latexContent = '';
+
+      // Process each content element
+      for (let i = 0; i < block.content.length; i++) {
+        const content = block.content[i];
+        if (content.type === 'text') {
+          // Process each character in the text content
+          for (let j = 0; j < content.text.length; j++) {
+            const char = content.text[j];
+            if (char === '$') {
+              if (isInLatex) {
+                // End of LaTeX block
+                if (currentText) {
+                  newContent.push({
+                    type: 'text',
+                    text: currentText,
+                    styles: content.styles,
+                  });
+                  currentText = '';
+                }
+                newContent.push({
+                  type: 'inlineLatex',
+                  props: {
+                    formula: latexContent,
+                  },
+                });
+                latexContent = '';
+                isInLatex = false;
+              } else {
+                // Start of LaTeX block
+                if (currentText) {
+                  newContent.push({
+                    type: 'text',
+                    text: currentText,
+                    styles: content.styles,
+                  });
+                  currentText = '';
+                }
+                isInLatex = true;
+              }
+            } else {
+              if (isInLatex) {
+                latexContent += char;
+              } else {
+                currentText += char;
+              }
+            }
+          }
+        } else {
+          // Handle non-text content
+          if (currentText) {
+            newContent.push({
+              type: 'text',
+              text: currentText,
+              styles: {},
+            });
+            currentText = '';
+          }
+          // Skip non-text content for now
+          continue;
+        }
+      }
+
+      // Add any remaining text
+      if (currentText) {
+        newContent.push({
+          type: 'text',
+          text: currentText,
+          styles: {},
+        });
+      }
+
+      // If we're still in a LaTeX block at the end, treat it as regular text
+      if (isInLatex) {
+        newContent.push({
+          type: 'text',
+          text: '$' + latexContent,
+          styles: {},
+        });
+      }
+
+      return {
+        ...block,
+        content: newContent,
+      };
+    }
+    return block;
+  });
+}