Use specific languages for shiki (#1938)

<!-- CURSOR_SUMMARY -->
> [!NOTE]
> Refactors code highlighting to use react-shiki/core with a singleton
highlighter and preloaded langs/themes, adds a fallback renderer, bumps
react-shiki, and adds tests for message summarization.
> 
> - **Frontend**
> - **CodeHighlight** (`src/components/chat/CodeHighlight.tsx`): Replace
`useShikiHighlighter` with `ShikiHighlighter` from `react-shiki/core`
using a singleton `createHighlighterCore` and JS regex engine; preload
common languages and GitHub light/dark themes; add `<pre><code>`
fallback while loading.
> - **Tests**
> - Add/expand Vitest suite for `formatMessagesForSummary`
(`src/__tests__/formatMessagesForSummary.test.ts`): covers truncation,
ordering, special chars, undefined content, and edge cases.
> - **Dependencies**
>   - Upgrade `react-shiki` to `^0.9.0` in `package.json`.
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
b32d224cd21d3c76e77799f2995905e523406bf9. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->





<!-- This is an auto-generated description by cubic. -->
---
## Summary by cubic
Preloaded a specific set of Shiki languages and themes using react-shiki
core, and updated CodeHighlight to use a singleton highlighter. This
reduces bundle size and stabilizes code rendering with a simple fallback
while loading.

- **Refactors**
- Switched to react-shiki/core with ShikiHighlighter and a singleton
highlighter.
- Preloaded common languages
(js/ts/jsx/tsx/html/css/json/markdown/python/etc.) and GitHub light/dark
themes.
- Used the JavaScript regex engine and added a plain <pre><code>
fallback until the highlighter is ready.

- **Dependencies**
  - Upgraded react-shiki to ^0.9.0.

<sup>Written for commit b32d224cd21d3c76e77799f2995905e523406bf9.
Summary will update automatically on new commits.</sup>

<!-- End of auto-generated description by cubic. -->

src/components/chat/CodeHighlight.tsx

@@ -1,16 +1,78 @@
-import React, {
-  useState,
-  useEffect,
-  useRef,
-  memo,
-  type ReactNode,
-} from "react";
-import { isInlineCode, useShikiHighlighter } from "react-shiki";
-import github from "@shikijs/themes/github-light-default";
-import githubDark from "@shikijs/themes/github-dark-default";
+import React, { useState, useEffect, memo, type ReactNode } from "react";
+import ShikiHighlighter, {
+  isInlineCode,
+  createHighlighterCore,
+  createJavaScriptRegexEngine,
+} from "react-shiki/core";
 import type { Element as HastElement } from "hast";
 import { useTheme } from "../../contexts/ThemeContext";
 import { Copy, Check } from "lucide-react";
+import github from "@shikijs/themes/github-light-default";
+import githubDark from "@shikijs/themes/github-dark-default";
+// common languages
+import astro from "@shikijs/langs/astro";
+import css from "@shikijs/langs/css";
+import graphql from "@shikijs/langs/graphql";
+import html from "@shikijs/langs/html";
+import java from "@shikijs/langs/java";
+import javascript from "@shikijs/langs/javascript";
+import json from "@shikijs/langs/json";
+import jsx from "@shikijs/langs/jsx";
+import less from "@shikijs/langs/less";
+import markdown from "@shikijs/langs/markdown";
+import python from "@shikijs/langs/python";
+import sass from "@shikijs/langs/sass";
+import scss from "@shikijs/langs/scss";
+import shell from "@shikijs/langs/shell";
+import sql from "@shikijs/langs/sql";
+import tsx from "@shikijs/langs/tsx";
+import typescript from "@shikijs/langs/typescript";
+import vue from "@shikijs/langs/vue";
+
+type HighlighterCore = Awaited<ReturnType<typeof createHighlighterCore>>;
+
+// Create a singleton highlighter instance
+let highlighterPromise: Promise<HighlighterCore> | null = null;
+
+function getHighlighter(): Promise<HighlighterCore> {
+  if (!highlighterPromise) {
+    highlighterPromise = createHighlighterCore({
+      themes: [github, githubDark],
+      langs: [
+        astro,
+        css,
+        graphql,
+        html,
+        java,
+        javascript,
+        json,
+        jsx,
+        less,
+        markdown,
+        python,
+        sass,
+        scss,
+        shell,
+        sql,
+        tsx,
+        typescript,
+        vue,
+      ],
+      engine: createJavaScriptRegexEngine(),
+    });
+  }
+  return highlighterPromise as Promise<HighlighterCore>;
+}
+
+function useHighlighter() {
+  const [highlighter, setHighlighter] = useState<HighlighterCore>();
+
+  useEffect(() => {
+    getHighlighter().then(setHighlighter);
+  }, []);
+
+  return highlighter;
+}
 
 interface CodeHighlightProps {
   className?: string | undefined;
@@ -32,29 +94,8 @@ export const CodeHighlight = memo(
     };
 
     const { isDarkMode } = useTheme();
+    const highlighter = useHighlighter();
 
-    // Cache for the highlighted code
-    const highlightedCodeCache = useRef<ReactNode | null>(null);
-
-    // Only update the highlighted code if the inputs change
-    const highlightedCode = useShikiHighlighter(
-      code,
-      language,
-      isDarkMode ? githubDark : github,
-      {
-        delay: 150,
-      },
-    );
-
-    // Update the cache whenever we get a new highlighted code
-    useEffect(() => {
-      if (highlightedCode) {
-        highlightedCodeCache.current = highlightedCode;
-      }
-    }, [highlightedCode]);
-
-    // Use the cached version during transitions to prevent flickering
-    const displayedCode = highlightedCode || highlightedCodeCache.current;
     return !isInline ? (
       <div
         className="shiki not-prose relative [&_pre]:overflow-auto 
@@ -77,7 +118,20 @@ export const CodeHighlight = memo(
             )}
           </div>
         ) : null}
-        {displayedCode}
+        {highlighter ? (
+          <ShikiHighlighter
+            highlighter={highlighter}
+            language={language}
+            theme={isDarkMode ? "github-dark-default" : "github-light-default"}
+            delay={150}
+          >
+            {code}
+          </ShikiHighlighter>
+        ) : (
+          <pre>
+            <code>{code}</code>
+          </pre>
+        )}
       </div>
     ) : (
       <code className={className} {...props}>