E2E test for fixing errors (#351)

e2e-tests/fix_error.spec.ts

@@ -0,0 +1,21 @@
+import { testSkipIfWindows } from "./helpers/test_helper";
+
+testSkipIfWindows("fix error with AI", async ({ po }) => {
+  await po.setUp({ autoApprove: true });
+  await po.sendPrompt("tc=create-error");
+
+  await po.snapshotPreviewErrorBanner();
+
+  await po.page.getByText("Error Line 6 error", { exact: true }).click();
+  await po.snapshotPreviewErrorBanner();
+
+  await po.clickFixErrorWithAI();
+  await po.waitForChatCompletion();
+  await po.snapshotMessages();
+
+  // TODO: this is an actual bug where the error banner should not
+  // be shown, however there's some kind of race condition and
+  // we don't reliably detect when the HMR update has completed.
+  // await po.locatePreviewErrorBanner().waitFor({ state: "hidden" });
+  await po.snapshotPreview();
+});

e2e-tests/fixtures/create-error.md

@@ -0,0 +1,25 @@
+I will intentionally add an error
+
+<dyad-write path="src/pages/Index.tsx" description="intentionally add an error">
+// Update this page (the content is just a fallback if you fail to update the page)
+
+import { MadeWithDyad } from "@/components/made-with-dyad";
+
+const Index = () => {
+throw new Error("Line 6 error");
+return (
+
+<div className="min-h-screen flex items-center justify-center bg-gray-100">
+<div className="text-center">
+<h1 className="text-4xl font-bold mb-4">Welcome to Your Blank App</h1>
+<p className="text-xl text-gray-600">
+Start building your amazing project here!
+</p>
+</div>
+<MadeWithDyad />
+</div>
+);
+};
+
+export default Index;
+</dyad-write>

e2e-tests/helpers/test_helper.ts

@@ -10,10 +10,10 @@ import { generateAppFilesSnapshotData } from "./generateAppFilesSnapshotData";
 const showDebugLogs = process.env.DEBUG_LOGS === "true";
 
 export const Timeout = {
-  // Why make this a constant? In some platforms, perhaps locally,
+  // Things generally take longer on CI, so we make them longer.
-  // we may want to shorten this.
+  EXTRA_LONG: process.env.CI ? 120_000 : 60_000,
-  LONG: os.platform() === "win32" ? 60_000 : 30_000,
+  LONG: process.env.CI ? 60_000 : 30_000,
-  MEDIUM: os.platform() === "win32" ? 30_000 : 15_000,
+  MEDIUM: process.env.CI ? 30_000 : 15_000,
 };
 
 export class PageObject {
@@ -165,6 +165,20 @@ export class PageObject {
     return this.page.getByTestId("preview-iframe-element");
   }
 
+  async clickFixErrorWithAI() {
+    await this.page.getByRole("button", { name: "Fix error with AI" }).click();
+  }
+
+  async snapshotPreviewErrorBanner() {
+    await expect(this.locatePreviewErrorBanner()).toMatchAriaSnapshot({
+      timeout: Timeout.LONG,
+    });
+  }
+
+  locatePreviewErrorBanner() {
+    return this.page.getByTestId("preview-error-banner");
+  }
+
   async snapshotPreview({ name }: { name?: string } = {}) {
     const iframe = this.getPreviewIframeElement();
     await expect(iframe.contentFrame().locator("body")).toMatchAriaSnapshot({

e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-1.aria.yml

@@ -0,0 +1,8 @@
+- button:
+  - img
+- img
+- text: Error Line 6 error
+- img
+- text: "Tip: Check if restarting the app fixes the error."
+- button "Fix error with AI":
+  - img

e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-2.aria.yml

@@ -0,0 +1,8 @@
+- button:
+  - img
+- img
+- text: "/Error Line 6 error Stack trace: Index \\(http:\\/\\/localhost:\\d+\\/src\\/pages\\/Index\\.tsx:6:6\\)/"
+- img
+- text: "Tip: Check if restarting the app fixes the error."
+- button "Fix error with AI":
+  - img

e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-3.aria.yml

@@ -0,0 +1,20 @@
+- paragraph: tc=create-error
+- paragraph: I will intentionally add an error
+- img
+- text: Index.tsx
+- img
+- text: "src/pages/Index.tsx Summary: intentionally add an error"
+- img
+- text: Approved
+- paragraph: "/Fix error: Error Line 6 error Stack trace: Index \\(http:\\/\\/localhost:\\d+\\/src\\/pages\\/Index\\.tsx:6:6\\)/"
+- code: Fixing the error...
+- img
+- text: Index.tsx
+- img
+- text: src/pages/Index.tsx
+- img
+- text: Approved
+- button "Undo":
+  - img
+- button "Retry":
+  - img

e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-4.aria.yml

@@ -0,0 +1,6 @@
+- region "Notifications (F8)":
+  - list
+- region "Notifications alt+T"
+- heading "No more errors!" [level=1]
+- link "Made with Dyad":
+  - /url: https://www.dyad.sh/

playwright.config.ts

@@ -5,7 +5,7 @@ const config: PlaywrightTestConfig = {
   workers: 1,
   retries: process.env.CI ? 1 : 0,
   // maxFailures: 1,
-  timeout: process.env.CI ? 120_000 : 30_000,
+  timeout: process.env.CI ? 180_000 : 30_000,
   // Use a custom snapshot path template because Playwright's default
   // is platform-specific which isn't necessary for Dyad e2e tests
   // which should be platform agnostic (we don't do screenshots; only textual diffs).

src/components/preview_panel/PreviewIframe.tsx

@@ -51,7 +51,10 @@ const ErrorBanner = ({ error, onDismiss, onAIFix }: ErrorBannerProps) => {
   };
 
   return (
-    <div className="absolute top-2 left-2 right-2 z-10 bg-red-50 dark:bg-red-950 border border-red-200 dark:border-red-800 rounded-md shadow-sm p-2">
+    <div
+      className="absolute top-2 left-2 right-2 z-10 bg-red-50 dark:bg-red-950 border border-red-200 dark:border-red-800 rounded-md shadow-sm p-2"
+      data-testid="preview-error-banner"
+    >
       {/* Close button in top left */}
       <button
         onClick={onDismiss}

testing/fake-llm-server/chatCompletionHandler.ts

@@ -24,6 +24,35 @@ export const createChatCompletionHandler =
     }
 
     let messageContent = CANNED_MESSAGE;
+
+    if (
+      lastMessage &&
+      typeof lastMessage.content === "string" &&
+      lastMessage.content.startsWith("Fix error: Error Line 6 error")
+    ) {
+      messageContent = `
+      Fixing the error...
+      <dyad-write path="src/pages/Index.tsx">
+      
+
+import { MadeWithDyad } from "@/components/made-with-dyad";
+
+const Index = () => {
+  return (
+    <div className="min-h-screen flex items-center justify-center bg-gray-100">
+      <div className="text-center">
+        <h1 className="text-4xl font-bold mb-4">No more errors!</h1>
+      </div>
+      <MadeWithDyad />
+    </div>
+  );
+};
+
+export default Index;
+
+      </dyad-write>
+      `;
+    }
     console.error("LASTMESSAGE", lastMessage);
     // Check if the last message is "[dump]" to write messages to file and return path
     if (