From 4907fcafd8204fd6d199eed3e114abea0d5bd4bc Mon Sep 17 00:00:00 2001 From: Will Chen Date: Thu, 5 Jun 2025 23:29:52 -0700 Subject: [PATCH] E2E test for fixing errors (#351) --- e2e-tests/fix_error.spec.ts | 21 ++++++++++++++ e2e-tests/fixtures/create-error.md | 25 ++++++++++++++++ e2e-tests/helpers/test_helper.ts | 22 +++++++++++--- ...error.spec.ts_fix-error-with-AI-1.aria.yml | 8 +++++ ...error.spec.ts_fix-error-with-AI-2.aria.yml | 8 +++++ ...error.spec.ts_fix-error-with-AI-3.aria.yml | 20 +++++++++++++ ...error.spec.ts_fix-error-with-AI-4.aria.yml | 6 ++++ playwright.config.ts | 2 +- .../preview_panel/PreviewIframe.tsx | 5 +++- .../fake-llm-server/chatCompletionHandler.ts | 29 +++++++++++++++++++ 10 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 e2e-tests/fix_error.spec.ts create mode 100644 e2e-tests/fixtures/create-error.md create mode 100644 e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-1.aria.yml create mode 100644 e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-2.aria.yml create mode 100644 e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-3.aria.yml create mode 100644 e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-4.aria.yml diff --git a/e2e-tests/fix_error.spec.ts b/e2e-tests/fix_error.spec.ts new file mode 100644 index 0000000..08f7202 --- /dev/null +++ b/e2e-tests/fix_error.spec.ts @@ -0,0 +1,21 @@ +import { testSkipIfWindows } from "./helpers/test_helper"; + +testSkipIfWindows("fix error with AI", async ({ po }) => { + await po.setUp({ autoApprove: true }); + await po.sendPrompt("tc=create-error"); + + await po.snapshotPreviewErrorBanner(); + + await po.page.getByText("Error Line 6 error", { exact: true }).click(); + await po.snapshotPreviewErrorBanner(); + + await po.clickFixErrorWithAI(); + await po.waitForChatCompletion(); + await po.snapshotMessages(); + + // TODO: this is an actual bug where the error banner should not + // be shown, however there's some kind of race condition and + // we don't reliably detect when the HMR update has completed. + // await po.locatePreviewErrorBanner().waitFor({ state: "hidden" }); + await po.snapshotPreview(); +}); diff --git a/e2e-tests/fixtures/create-error.md b/e2e-tests/fixtures/create-error.md new file mode 100644 index 0000000..d5ed315 --- /dev/null +++ b/e2e-tests/fixtures/create-error.md @@ -0,0 +1,25 @@ +I will intentionally add an error + + +// Update this page (the content is just a fallback if you fail to update the page) + +import { MadeWithDyad } from "@/components/made-with-dyad"; + +const Index = () => { +throw new Error("Line 6 error"); +return ( + +
+
+

Welcome to Your Blank App

+

+Start building your amazing project here! +

+
+ +
+); +}; + +export default Index; + diff --git a/e2e-tests/helpers/test_helper.ts b/e2e-tests/helpers/test_helper.ts index 8d4e5c6..540fa6e 100644 --- a/e2e-tests/helpers/test_helper.ts +++ b/e2e-tests/helpers/test_helper.ts @@ -10,10 +10,10 @@ import { generateAppFilesSnapshotData } from "./generateAppFilesSnapshotData"; const showDebugLogs = process.env.DEBUG_LOGS === "true"; export const Timeout = { - // Why make this a constant? In some platforms, perhaps locally, - // we may want to shorten this. - LONG: os.platform() === "win32" ? 60_000 : 30_000, - MEDIUM: os.platform() === "win32" ? 30_000 : 15_000, + // Things generally take longer on CI, so we make them longer. + EXTRA_LONG: process.env.CI ? 120_000 : 60_000, + LONG: process.env.CI ? 60_000 : 30_000, + MEDIUM: process.env.CI ? 30_000 : 15_000, }; export class PageObject { @@ -165,6 +165,20 @@ export class PageObject { return this.page.getByTestId("preview-iframe-element"); } + async clickFixErrorWithAI() { + await this.page.getByRole("button", { name: "Fix error with AI" }).click(); + } + + async snapshotPreviewErrorBanner() { + await expect(this.locatePreviewErrorBanner()).toMatchAriaSnapshot({ + timeout: Timeout.LONG, + }); + } + + locatePreviewErrorBanner() { + return this.page.getByTestId("preview-error-banner"); + } + async snapshotPreview({ name }: { name?: string } = {}) { const iframe = this.getPreviewIframeElement(); await expect(iframe.contentFrame().locator("body")).toMatchAriaSnapshot({ diff --git a/e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-1.aria.yml b/e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-1.aria.yml new file mode 100644 index 0000000..f598756 --- /dev/null +++ b/e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-1.aria.yml @@ -0,0 +1,8 @@ +- button: + - img +- img +- text: Error Line 6 error +- img +- text: "Tip: Check if restarting the app fixes the error." +- button "Fix error with AI": + - img \ No newline at end of file diff --git a/e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-2.aria.yml b/e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-2.aria.yml new file mode 100644 index 0000000..5c6cf9e --- /dev/null +++ b/e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-2.aria.yml @@ -0,0 +1,8 @@ +- button: + - img +- img +- text: "/Error Line 6 error Stack trace: Index \\(http:\\/\\/localhost:\\d+\\/src\\/pages\\/Index\\.tsx:6:6\\)/" +- img +- text: "Tip: Check if restarting the app fixes the error." +- button "Fix error with AI": + - img \ No newline at end of file diff --git a/e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-3.aria.yml b/e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-3.aria.yml new file mode 100644 index 0000000..baa9924 --- /dev/null +++ b/e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-3.aria.yml @@ -0,0 +1,20 @@ +- paragraph: tc=create-error +- paragraph: I will intentionally add an error +- img +- text: Index.tsx +- img +- text: "src/pages/Index.tsx Summary: intentionally add an error" +- img +- text: Approved +- paragraph: "/Fix error: Error Line 6 error Stack trace: Index \\(http:\\/\\/localhost:\\d+\\/src\\/pages\\/Index\\.tsx:6:6\\)/" +- code: Fixing the error... +- img +- text: Index.tsx +- img +- text: src/pages/Index.tsx +- img +- text: Approved +- button "Undo": + - img +- button "Retry": + - img \ No newline at end of file diff --git a/e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-4.aria.yml b/e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-4.aria.yml new file mode 100644 index 0000000..89fc71a --- /dev/null +++ b/e2e-tests/snapshots/fix_error.spec.ts_fix-error-with-AI-4.aria.yml @@ -0,0 +1,6 @@ +- region "Notifications (F8)": + - list +- region "Notifications alt+T" +- heading "No more errors!" [level=1] +- link "Made with Dyad": + - /url: https://www.dyad.sh/ \ No newline at end of file diff --git a/playwright.config.ts b/playwright.config.ts index d1e4883..70c1248 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -5,7 +5,7 @@ const config: PlaywrightTestConfig = { workers: 1, retries: process.env.CI ? 1 : 0, // maxFailures: 1, - timeout: process.env.CI ? 120_000 : 30_000, + timeout: process.env.CI ? 180_000 : 30_000, // Use a custom snapshot path template because Playwright's default // is platform-specific which isn't necessary for Dyad e2e tests // which should be platform agnostic (we don't do screenshots; only textual diffs). diff --git a/src/components/preview_panel/PreviewIframe.tsx b/src/components/preview_panel/PreviewIframe.tsx index ac08472..2953028 100644 --- a/src/components/preview_panel/PreviewIframe.tsx +++ b/src/components/preview_panel/PreviewIframe.tsx @@ -51,7 +51,10 @@ const ErrorBanner = ({ error, onDismiss, onAIFix }: ErrorBannerProps) => { }; return ( -
+
{/* Close button in top left */}