Create tests: dumps message, "retry" (#281)

e2e-tests/1.spec.ts

@@ -0,0 +1,9 @@
+import { expect } from "@playwright/test";
+import { test } from "./helpers/test_helper";
+
+test("renders the first page", async ({ electronApp }) => {
+  const page = await electronApp.firstWindow();
+  await page.waitForSelector("h1");
+  const text = await page.$eval("h1", (el) => el.textContent);
+  expect(text).toBe("Build your dream app");
+});

e2e-tests/dump_messages.spec.ts

@@ -0,0 +1,8 @@
+import { test } from "./helpers/test_helper";
+
+// This is useful to make sure the messages are being sent correctly.
+test("dump messages", async ({ po }) => {
+  await po.setUp();
+  await po.sendPrompt("[dump]");
+  await po.snapshotServerDump();
+});

e2e-tests/helpers/test_helper.ts

@@ -1,6 +1,7 @@
 import { test as base, Page, expect } from "@playwright/test";
 import { findLatestBuild, parseElectronApp } from "electron-playwright-helpers";
 import { ElectronApplication, _electron as electron } from "playwright";
+import fs from "fs";
 
 const showDebugLogs = process.env.DEBUG_LOGS === "true";
 
@@ -15,14 +16,44 @@ class PageObject {
     await this.selectTestModel();
   }
 
-  async dumpMessages() {
+  async snapshotMessages() {
     await expect(this.page.getByTestId("messages-list")).toMatchAriaSnapshot();
   }
 
+  async snapshotServerDump() {
+    // Get the text content of the messages list
+    const messagesListText = await this.page
+      .getByTestId("messages-list")
+      .textContent();
+
+    // Find the dump path using regex
+    const dumpPathMatch = messagesListText?.match(
+      /\[\[dyad-dump-path=([^\]]+)\]\]/,
+    );
+
+    if (!dumpPathMatch) {
+      throw new Error("No dump path found in messages list");
+    }
+
+    const dumpFilePath = dumpPathMatch[1];
+
+    // Read the JSON file
+    const dumpContent = fs.readFileSync(dumpFilePath, "utf-8");
+
+    // Perform snapshot comparison
+    expect(prettifyDump(dumpContent)).toMatchSnapshot("server-dump.txt");
+  }
+
   async waitForChatCompletion() {
-    await expect(
+    await expect(this.getRetryButton()).toBeVisible();
-      this.page.getByRole("button", { name: "Retry" }),
+  }
-    ).toBeVisible();
+
+  async clickRetry() {
+    await this.getRetryButton().click();
+  }
+
+  private getRetryButton() {
+    return this.page.getByRole("button", { name: "Retry" });
   }
 
   async sendPrompt(prompt: string) {
@@ -79,6 +110,56 @@ class PageObject {
   async goToAppsTab() {
     await this.page.getByRole("link", { name: "Apps" }).click();
   }
+
+  ////////////////////////////////
+  // Toast assertions
+  ////////////////////////////////
+
+  async expectNoToast() {
+    await expect(this.page.locator("[data-sonner-toast]")).toHaveCount(0);
+  }
+
+  async waitForToast(
+    type?: "success" | "error" | "warning" | "info",
+    timeout = 5000,
+  ) {
+    const selector = type
+      ? `[data-sonner-toast][data-type="${type}"]`
+      : "[data-sonner-toast]";
+
+    await this.page.waitForSelector(selector, { timeout });
+  }
+
+  async waitForToastWithText(text: string, timeout = 5000) {
+    await this.page.waitForSelector(`[data-sonner-toast]:has-text("${text}")`, {
+      timeout,
+    });
+  }
+
+  async assertToastVisible(type?: "success" | "error" | "warning" | "info") {
+    const selector = type
+      ? `[data-sonner-toast][data-type="${type}"]`
+      : "[data-sonner-toast]";
+
+    await expect(this.page.locator(selector)).toBeVisible();
+  }
+
+  async assertToastWithText(text: string) {
+    await expect(
+      this.page.locator(`[data-sonner-toast]:has-text("${text}")`),
+    ).toBeVisible();
+  }
+
+  async dismissAllToasts() {
+    // Click all close buttons if they exist
+    const closeButtons = this.page.locator(
+      "[data-sonner-toast] button[data-close-button]",
+    );
+    const count = await closeButtons.count();
+    for (let i = 0; i < count; i++) {
+      await closeButtons.nth(i).click();
+    }
+  }
 }
 
 // From https://github.com/microsoft/playwright/issues/8208#issuecomment-1435475930
@@ -114,52 +195,68 @@ export const test = base.extend<{
     },
     { auto: true },
   ],
-  electronApp: async ({}, use) => {
+  electronApp: [
-    // find the latest build in the out directory
+    async ({}, use) => {
-    const latestBuild = findLatestBuild();
+      // find the latest build in the out directory
-    // parse the directory and find paths and other info
+      const latestBuild = findLatestBuild();
-    const appInfo = parseElectronApp(latestBuild);
+      // parse the directory and find paths and other info
-    process.env.E2E_TEST_BUILD = "true";
+      const appInfo = parseElectronApp(latestBuild);
-    // This is just a hack to avoid the AI setup screen.
+      process.env.E2E_TEST_BUILD = "true";
-    process.env.OPENAI_API_KEY = "sk-test";
+      // This is just a hack to avoid the AI setup screen.
-    const electronApp = await electron.launch({
+      process.env.OPENAI_API_KEY = "sk-test";
-      args: [
+      const electronApp = await electron.launch({
-        appInfo.main,
+        args: [
-        "--enable-logging",
+          appInfo.main,
-        `--user-data-dir=/tmp/dyad-e2e-tests-${Date.now()}`,
+          "--enable-logging",
-      ],
+          `--user-data-dir=/tmp/dyad-e2e-tests-${Date.now()}`,
-      executablePath: appInfo.executable,
+        ],
-    });
+        executablePath: appInfo.executable,
-
-    console.log("electronApp launched!");
-    if (showDebugLogs) {
-      // Listen to main process output immediately
-      electronApp.process().stdout?.on("data", (data) => {
-        console.log(`MAIN_PROCESS_STDOUT: ${data.toString()}`);
       });
-      electronApp.process().stderr?.on("data", (data) => {
-        console.error(`MAIN_PROCESS_STDERR: ${data.toString()}`);
-      });
-    }
-    electronApp.on("close", () => {
-      console.log(`Electron app closed listener:`);
-    });
 
-    electronApp.on("window", async (page) => {
+      console.log("electronApp launched!");
-      const filename = page.url()?.split("/").pop();
+      if (showDebugLogs) {
-      console.log(`Window opened: ${filename}`);
+        // Listen to main process output immediately
-
+        electronApp.process().stdout?.on("data", (data) => {
-      // capture errors
+          console.log(`MAIN_PROCESS_STDOUT: ${data.toString()}`);
-      page.on("pageerror", (error) => {
+        });
-        console.error(error);
+        electronApp.process().stderr?.on("data", (data) => {
+          console.error(`MAIN_PROCESS_STDERR: ${data.toString()}`);
+        });
+      }
+      electronApp.on("close", () => {
+        console.log(`Electron app closed listener:`);
       });
-      // capture console messages
-      page.on("console", (msg) => {
-        console.log(msg.text());
-      });
-    });
 
-    await use(electronApp);
+      electronApp.on("window", async (page) => {
-    await electronApp.close();
+        const filename = page.url()?.split("/").pop();
-  },
+        console.log(`Window opened: ${filename}`);
+
+        // capture errors
+        page.on("pageerror", (error) => {
+          console.error(error);
+        });
+        // capture console messages
+        page.on("console", (msg) => {
+          console.log(msg.text());
+        });
+      });
+
+      await use(electronApp);
+      await electronApp.close();
+    },
+    { auto: true },
+  ],
 });
+
+function prettifyDump(dumpContent: string) {
+  const parsedDump = JSON.parse(dumpContent) as Array<{
+    role: string;
+    content: string;
+  }>;
+
+  return parsedDump
+    .map((message) => {
+      return `===\nrole: ${message.role}\nmessage: ${message.content}`;
+    })
+    .join("\n\n");
+}

e2e-tests/main.spec.ts

@@ -1,21 +1,13 @@
-import { expect } from "@playwright/test";
 import { test } from "./helpers/test_helper";
 
-test("renders the first page", async ({ electronApp }) => {
-  const page = await electronApp.firstWindow();
-  await page.waitForSelector("h1");
-  const text = await page.$eval("h1", (el) => el.textContent);
-  expect(text).toBe("Build your dream app");
-});
-
 test("simple message to custom test model", async ({ po }) => {
   await po.setUp();
   await po.sendPrompt("hi");
-  await po.dumpMessages();
+  await po.snapshotMessages();
 });
 
 test("basic message to custom test model", async ({ po }) => {
   await po.setUp();
   await po.sendPrompt("tc=basic");
-  await po.dumpMessages();
+  await po.snapshotMessages();
 });

e2e-tests/retry.spec.ts

@@ -0,0 +1,13 @@
+import { test } from "./helpers/test_helper";
+
+test("retry - should work", async ({ po }) => {
+  await po.setUp();
+  await po.sendPrompt("[increment]");
+  await po.snapshotMessages();
+
+  await po.dismissAllToasts();
+  await po.clickRetry();
+  await po.expectNoToast();
+  // The counter should be incremented in the snapshotted messages.
+  await po.snapshotMessages();
+});

e2e-tests/snapshots/retry.spec.ts_retry---should-work-1.aria.yml

@@ -0,0 +1,4 @@
+- paragraph: "[increment]"
+- paragraph: counter=1
+- button "Retry":
+  - img

e2e-tests/snapshots/retry.spec.ts_retry---should-work-2.aria.yml

@@ -0,0 +1,4 @@
+- paragraph: "[increment]"
+- paragraph: counter=2
+- button "Retry":
+  - img

playwright.config.ts

@@ -2,8 +2,9 @@ import { PlaywrightTestConfig } from "@playwright/test";
 
 const config: PlaywrightTestConfig = {
   testDir: "./e2e-tests",
+  workers: 1,
   maxFailures: 1,
-  timeout: process.env.CI ? 30_000 : 15_000,
+  timeout: process.env.CI ? 60_000 : 15_000,
   // Use a custom snapshot path template because Playwright's default
   // is platform-specific which isn't necessary for Dyad e2e tests
   // which should be platform agnostic (we don't do screenshots; only textual diffs).

testing/fake-llm-server/index.ts

@@ -56,6 +56,8 @@ app.get("/health", (req, res) => {
   res.send("OK");
 });
 
+let globalCounter = 0;
+
 // Handle POST requests to /v1/chat/completions
 app.post("/v1/chat/completions", (req, res) => {
   const { stream = false, messages = [] } = req.body;
@@ -74,8 +76,40 @@ app.post("/v1/chat/completions", (req, res) => {
     });
   }
 
-  // Check if the last message starts with "tc=" to load test case file
   let messageContent = CANNED_MESSAGE;
+
+  // Check if the last message is "[dump]" to write messages to file and return path
+  if (lastMessage && lastMessage.content === "[dump]") {
+    const timestamp = Date.now();
+    const generatedDir = path.join(__dirname, "generated");
+
+    // Create generated directory if it doesn't exist
+    if (!fs.existsSync(generatedDir)) {
+      fs.mkdirSync(generatedDir, { recursive: true });
+    }
+
+    const dumpFilePath = path.join(generatedDir, `${timestamp}.json`);
+
+    try {
+      fs.writeFileSync(
+        dumpFilePath,
+        JSON.stringify(messages, null, 2),
+        "utf-8",
+      );
+      console.log(`* Dumped messages to: ${dumpFilePath}`);
+      messageContent = `[[dyad-dump-path=${dumpFilePath}]]`;
+    } catch (error) {
+      console.error(`* Error writing dump file: ${error}`);
+      messageContent = `Error: Could not write dump file: ${error}`;
+    }
+  }
+
+  if (lastMessage && lastMessage.content === "[increment]") {
+    globalCounter++;
+    messageContent = `counter=${globalCounter}`;
+  }
+
+  // Check if the last message starts with "tc=" to load test case file
   if (
     lastMessage &&
     lastMessage.content &&