Smart Context: deep (#1527)

<!-- CURSOR_SUMMARY -->
> [!NOTE]
> Introduce a new "deep" Smart Context mode that supplies versioned
files (by commit) to the engine, adds code search rendering, stores
source commit hashes, improves search-replace recovery, and updates
UI/tests.
> 
> - **Smart Context (deep)**:
> - Replace `conservative` with `deep`; limit context to ~200 turns;
send `sourceCommitHash` per message.
> - Build and pass `versioned_files` (hash-id map + per-message file
refs) and `app_id` to engine.
> - **DB**:
>   - Add `messages.source_commit_hash` (+ migration/snapshot).
> - **Engine/Processing**:
> - Retry Turbo Edits v2: first re-read then fallback to `dyad-write` if
search-replace fails.
> - Include provider options and versioned files in requests; add
`getCurrentCommitHash`/`getFileAtCommit`.
> - **UI**:
>   - Pro mode selector: new `deep` option; tooltips polish.
> - Add `DyadCodeSearch` and `DyadCodeSearchResult` components; parser
supports new tags.
> - **Tests/E2E**:
> - New `smart_context_deep` e2e; update snapshots to include `app_id`
and deep mode; adjust Playwright timeout.
>   - Unit tests for versioned codebase context.
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
e3d3bffabb2bc6caf52103461f9d6f2d5ad39df8. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->

src/ipc/utils/git_utils.ts

@@ -6,7 +6,8 @@ import pathModule from "node:path";
 import { exec } from "node:child_process";
 import { promisify } from "node:util";
 import { readSettings } from "../../main/settings";
-
+import log from "electron-log";
+const logger = log.scope("git_utils");
 const execAsync = promisify(exec);
 
 async function verboseExecAsync(
@@ -26,6 +27,18 @@ async function verboseExecAsync(
   }
 }
 
+export async function getCurrentCommitHash({
+  path,
+}: {
+  path: string;
+}): Promise<string> {
+  return await git.resolveRef({
+    fs,
+    dir: path,
+    ref: "HEAD",
+  });
+}
+
 export async function gitCommit({
   path,
   message,
@@ -166,3 +179,45 @@ export async function gitAddAll({ path }: { path: string }): Promise<void> {
     return git.add({ fs, dir: path, filepath: "." });
   }
 }
+
+export async function getFileAtCommit({
+  path,
+  filePath,
+  commitHash,
+}: {
+  path: string;
+  filePath: string;
+  commitHash: string;
+}): Promise<string | null> {
+  const settings = readSettings();
+  if (settings.enableNativeGit) {
+    try {
+      const { stdout } = await execAsync(
+        `git -C "${path}" show "${commitHash}:${filePath}"`,
+      );
+      return stdout;
+    } catch (error: any) {
+      logger.error(
+        `Error getting file at commit ${commitHash}: ${error.message}`,
+      );
+      // File doesn't exist at this commit
+      return null;
+    }
+  } else {
+    try {
+      const { blob } = await git.readBlob({
+        fs,
+        dir: path,
+        oid: commitHash,
+        filepath: filePath,
+      });
+      return Buffer.from(blob).toString("utf-8");
+    } catch (error: any) {
+      logger.error(
+        `Error getting file at commit ${commitHash}: ${error.message}`,
+      );
+      // File doesn't exist at this commit
+      return null;
+    }
+  }
+}

src/ipc/utils/llm_engine_provider.ts

@@ -42,7 +42,7 @@ or to provide a custom fetch implementation for e.g. testing.
     enableLazyEdits?: boolean;
     enableSmartFilesContext?: boolean;
     enableWebSearch?: boolean;
-    smartContextMode?: "balanced" | "conservative";
+    smartContextMode?: "balanced" | "conservative" | "deep";
   };
   settings: UserSettings;
 }
@@ -125,6 +125,10 @@ export function createDyadEngine(
               options.settings,
             ),
           };
+          const dyadVersionedFiles = parsedBody.dyadVersionedFiles;
+          if ("dyadVersionedFiles" in parsedBody) {
+            delete parsedBody.dyadVersionedFiles;
+          }
           const dyadFiles = parsedBody.dyadFiles;
           if ("dyadFiles" in parsedBody) {
             delete parsedBody.dyadFiles;
@@ -133,6 +137,10 @@ export function createDyadEngine(
           if ("dyadRequestId" in parsedBody) {
             delete parsedBody.dyadRequestId;
           }
+          const dyadAppId = parsedBody.dyadAppId;
+          if ("dyadAppId" in parsedBody) {
+            delete parsedBody.dyadAppId;
+          }
           const dyadDisableFiles = parsedBody.dyadDisableFiles;
           if ("dyadDisableFiles" in parsedBody) {
             delete parsedBody.dyadDisableFiles;
@@ -151,14 +159,16 @@ export function createDyadEngine(
           }
 
           // Add files to the request if they exist
-          if (dyadFiles?.length && !dyadDisableFiles) {
+          if (!dyadDisableFiles) {
             parsedBody.dyad_options = {
               files: dyadFiles,
+              versioned_files: dyadVersionedFiles,
               enable_lazy_edits: options.dyadOptions.enableLazyEdits,
               enable_smart_files_context:
                 options.dyadOptions.enableSmartFilesContext,
               smart_context_mode: options.dyadOptions.smartContextMode,
               enable_web_search: options.dyadOptions.enableWebSearch,
+              app_id: dyadAppId,
             };
             if (dyadMentionedApps?.length) {
               parsedBody.dyad_options.mentioned_apps = dyadMentionedApps;

src/ipc/utils/versioned_codebase_context.ts

@@ -0,0 +1,219 @@
+import { CodebaseFile, CodebaseFileReference } from "@/utils/codebase";
+import { ModelMessage } from "@ai-sdk/provider-utils";
+import crypto from "node:crypto";
+import log from "electron-log";
+import { getFileAtCommit } from "./git_utils";
+import { normalizePath } from "../../../shared/normalizePath";
+
+const logger = log.scope("versioned_codebase_context");
+
+export interface VersionedFiles {
+  fileIdToContent: Record<string, string>;
+  fileReferences: CodebaseFileReference[];
+  messageIndexToFilePathToFileId: Record<number, Record<string, string>>;
+}
+
+interface DyadEngineProviderOptions {
+  sourceCommitHash: string;
+}
+
+/**
+ * Parse file paths from assistant message content.
+ * Extracts files from <dyad-read> and <dyad-code-search-result> tags.
+ */
+export function parseFilesFromMessage(content: string): string[] {
+  const filePaths: string[] = [];
+  const seenPaths = new Set<string>();
+
+  // Create an array of matches with their positions to maintain order
+  interface TagMatch {
+    index: number;
+    filePaths: string[];
+  }
+  const matches: TagMatch[] = [];
+
+  // Parse <dyad-read path="$filePath"></dyad-read>
+  const dyadReadRegex = /<dyad-read\s+path="([^"]+)"\s*><\/dyad-read>/gs;
+  let match: RegExpExecArray | null;
+  while ((match = dyadReadRegex.exec(content)) !== null) {
+    const filePath = normalizePath(match[1].trim());
+    if (filePath) {
+      matches.push({
+        index: match.index,
+        filePaths: [filePath],
+      });
+    }
+  }
+
+  // Parse <dyad-code-search-result>...</dyad-code-search-result>
+  const codeSearchRegex =
+    /<dyad-code-search-result>(.*?)<\/dyad-code-search-result>/gs;
+  while ((match = codeSearchRegex.exec(content)) !== null) {
+    const innerContent = match[1];
+    const paths: string[] = [];
+    // Split by newlines and extract each file path
+    const lines = innerContent.split("\n");
+    for (const line of lines) {
+      const trimmedLine = line.trim();
+      if (
+        trimmedLine &&
+        !trimmedLine.startsWith("<") &&
+        !trimmedLine.startsWith(">")
+      ) {
+        paths.push(normalizePath(trimmedLine));
+      }
+    }
+    if (paths.length > 0) {
+      matches.push({
+        index: match.index,
+        filePaths: paths,
+      });
+    }
+  }
+
+  // Sort matches by their position in the original content
+  matches.sort((a, b) => a.index - b.index);
+
+  // Add file paths in order, deduplicating as we go
+  for (const match of matches) {
+    for (const path of match.filePaths) {
+      if (!seenPaths.has(path)) {
+        seenPaths.add(path);
+        filePaths.push(path);
+      }
+    }
+  }
+
+  return filePaths;
+}
+
+export async function processChatMessagesWithVersionedFiles({
+  files,
+  chatMessages,
+  appPath,
+}: {
+  files: CodebaseFile[];
+  chatMessages: ModelMessage[];
+  appPath: string;
+}): Promise<VersionedFiles> {
+  const fileIdToContent: Record<string, string> = {};
+  const fileReferences: CodebaseFileReference[] = [];
+  const messageIndexToFilePathToFileId: Record<
+    number,
+    Record<string, string>
+  > = {};
+  for (const file of files) {
+    // Generate SHA-256 hash of content as fileId
+    const fileId = crypto
+      .createHash("sha256")
+      .update(file.content)
+      .digest("hex");
+
+    fileIdToContent[fileId] = file.content;
+    const { content: _content, ...restOfFile } = file;
+
+    fileReferences.push({
+      ...restOfFile,
+      fileId,
+    });
+  }
+
+  for (
+    let messageIndex = 0;
+    messageIndex < chatMessages.length;
+    messageIndex++
+  ) {
+    const message = chatMessages[messageIndex];
+
+    // Only process assistant messages
+    if (message.role !== "assistant") {
+      continue;
+    }
+
+    // Extract sourceCommitHash from providerOptions
+    const engineOptions = message.providerOptions?.[
+      "dyad-engine"
+    ] as unknown as DyadEngineProviderOptions;
+    const sourceCommitHash = engineOptions?.sourceCommitHash;
+
+    // Skip messages without sourceCommitHash
+    if (!sourceCommitHash) {
+      continue;
+    }
+
+    // Get message content as text
+    const content = message.content;
+    let textContent: string;
+
+    if (typeof content !== "string") {
+      // Handle array of parts (text, images, etc.)
+      textContent = content
+        .filter((part) => part.type === "text")
+        .map((part) => part.text)
+        .join("\n");
+
+      if (!textContent) {
+        continue;
+      }
+    } else {
+      // Message content is already a string
+      textContent = content;
+    }
+
+    // Parse file paths from message content
+    const filePaths = parseFilesFromMessage(textContent);
+    const filePathsToFileIds: Record<string, string> = {};
+    messageIndexToFilePathToFileId[messageIndex] = filePathsToFileIds;
+
+    // Parallelize file content fetching
+    const fileContentPromises = filePaths.map((filePath) =>
+      getFileAtCommit({
+        path: appPath,
+        filePath,
+        commitHash: sourceCommitHash,
+      }).then(
+        (content) => ({ filePath, content, status: "fulfilled" as const }),
+        (error) => ({ filePath, error, status: "rejected" as const }),
+      ),
+    );
+
+    const results = await Promise.all(fileContentPromises);
+
+    for (const result of results) {
+      if (result.status === "rejected") {
+        logger.error(
+          `Error reading file ${result.filePath} at commit ${sourceCommitHash}:`,
+          result.error,
+        );
+        continue;
+      }
+
+      const { filePath, content: fileContent } = result;
+
+      if (fileContent === null) {
+        logger.warn(
+          `File ${filePath} not found at commit ${sourceCommitHash} for message ${messageIndex}`,
+        );
+        continue;
+      }
+
+      // Generate SHA-256 hash of content as fileId
+      const fileId = crypto
+        .createHash("sha256")
+        .update(fileContent)
+        .digest("hex");
+
+      // Store in fileIdToContent
+      fileIdToContent[fileId] = fileContent;
+
+      // Add to this message's file IDs
+      filePathsToFileIds[filePath] = fileId;
+    }
+  }
+
+  return {
+    fileIdToContent,
+    fileReferences,
+    messageIndexToFilePathToFileId,
+  };
+}