Create tests: dumps message, "retry" (#281)

e2e-tests/1.spec.ts

@@ -0,0 +1,9 @@
+import { expect } from "@playwright/test";
+import { test } from "./helpers/test_helper";
+
+test("renders the first page", async ({ electronApp }) => {
+  const page = await electronApp.firstWindow();
+  await page.waitForSelector("h1");
+  const text = await page.$eval("h1", (el) => el.textContent);
+  expect(text).toBe("Build your dream app");
+});

e2e-tests/dump_messages.spec.ts

@@ -0,0 +1,8 @@
+import { test } from "./helpers/test_helper";
+
+// This is useful to make sure the messages are being sent correctly.
+test("dump messages", async ({ po }) => {
+  await po.setUp();
+  await po.sendPrompt("[dump]");
+  await po.snapshotServerDump();
+});

e2e-tests/helpers/test_helper.ts

@@ -1,6 +1,7 @@
 import { test as base, Page, expect } from "@playwright/test";
 import { findLatestBuild, parseElectronApp } from "electron-playwright-helpers";
 import { ElectronApplication, _electron as electron } from "playwright";
+import fs from "fs";
 
 const showDebugLogs = process.env.DEBUG_LOGS === "true";
 
@@ -15,14 +16,44 @@ class PageObject {
     await this.selectTestModel();
   }
 
-  async dumpMessages() {
+  async snapshotMessages() {
     await expect(this.page.getByTestId("messages-list")).toMatchAriaSnapshot();
   }
 
+  async snapshotServerDump() {
+    // Get the text content of the messages list
+    const messagesListText = await this.page
+      .getByTestId("messages-list")
+      .textContent();
+
+    // Find the dump path using regex
+    const dumpPathMatch = messagesListText?.match(
+      /\[\[dyad-dump-path=([^\]]+)\]\]/,
+    );
+
+    if (!dumpPathMatch) {
+      throw new Error("No dump path found in messages list");
+    }
+
+    const dumpFilePath = dumpPathMatch[1];
+
+    // Read the JSON file
+    const dumpContent = fs.readFileSync(dumpFilePath, "utf-8");
+
+    // Perform snapshot comparison
+    expect(prettifyDump(dumpContent)).toMatchSnapshot("server-dump.txt");
+  }
+
   async waitForChatCompletion() {
-    await expect(
-      this.page.getByRole("button", { name: "Retry" }),
-    ).toBeVisible();
+    await expect(this.getRetryButton()).toBeVisible();
+  }
+
+  async clickRetry() {
+    await this.getRetryButton().click();
+  }
+
+  private getRetryButton() {
+    return this.page.getByRole("button", { name: "Retry" });
   }
 
   async sendPrompt(prompt: string) {
@@ -79,6 +110,56 @@ class PageObject {
   async goToAppsTab() {
     await this.page.getByRole("link", { name: "Apps" }).click();
   }
+
+  ////////////////////////////////
+  // Toast assertions
+  ////////////////////////////////
+
+  async expectNoToast() {
+    await expect(this.page.locator("[data-sonner-toast]")).toHaveCount(0);
+  }
+
+  async waitForToast(
+    type?: "success" | "error" | "warning" | "info",
+    timeout = 5000,
+  ) {
+    const selector = type
+      ? `[data-sonner-toast][data-type="${type}"]`
+      : "[data-sonner-toast]";
+
+    await this.page.waitForSelector(selector, { timeout });
+  }
+
+  async waitForToastWithText(text: string, timeout = 5000) {
+    await this.page.waitForSelector(`[data-sonner-toast]:has-text("${text}")`, {
+      timeout,
+    });
+  }
+
+  async assertToastVisible(type?: "success" | "error" | "warning" | "info") {
+    const selector = type
+      ? `[data-sonner-toast][data-type="${type}"]`
+      : "[data-sonner-toast]";
+
+    await expect(this.page.locator(selector)).toBeVisible();
+  }
+
+  async assertToastWithText(text: string) {
+    await expect(
+      this.page.locator(`[data-sonner-toast]:has-text("${text}")`),
+    ).toBeVisible();
+  }
+
+  async dismissAllToasts() {
+    // Click all close buttons if they exist
+    const closeButtons = this.page.locator(
+      "[data-sonner-toast] button[data-close-button]",
+    );
+    const count = await closeButtons.count();
+    for (let i = 0; i < count; i++) {
+      await closeButtons.nth(i).click();
+    }
+  }
 }
 
 // From https://github.com/microsoft/playwright/issues/8208#issuecomment-1435475930
@@ -114,7 +195,8 @@ export const test = base.extend<{
     },
     { auto: true },
   ],
-  electronApp: async ({}, use) => {
+  electronApp: [
+    async ({}, use) => {
       // find the latest build in the out directory
       const latestBuild = findLatestBuild();
       // parse the directory and find paths and other info
@@ -162,4 +244,19 @@ export const test = base.extend<{
       await use(electronApp);
       await electronApp.close();
     },
+    { auto: true },
+  ],
 });
+
+function prettifyDump(dumpContent: string) {
+  const parsedDump = JSON.parse(dumpContent) as Array<{
+    role: string;
+    content: string;
+  }>;
+
+  return parsedDump
+    .map((message) => {
+      return `===\nrole: ${message.role}\nmessage: ${message.content}`;
+    })
+    .join("\n\n");
+}

e2e-tests/main.spec.ts

@@ -1,21 +1,13 @@
-import { expect } from "@playwright/test";
 import { test } from "./helpers/test_helper";
 
-test("renders the first page", async ({ electronApp }) => {
-  const page = await electronApp.firstWindow();
-  await page.waitForSelector("h1");
-  const text = await page.$eval("h1", (el) => el.textContent);
-  expect(text).toBe("Build your dream app");
-});
-
 test("simple message to custom test model", async ({ po }) => {
   await po.setUp();
   await po.sendPrompt("hi");
-  await po.dumpMessages();
+  await po.snapshotMessages();
 });
 
 test("basic message to custom test model", async ({ po }) => {
   await po.setUp();
   await po.sendPrompt("tc=basic");
-  await po.dumpMessages();
+  await po.snapshotMessages();
 });

e2e-tests/retry.spec.ts

@@ -0,0 +1,13 @@
+import { test } from "./helpers/test_helper";
+
+test("retry - should work", async ({ po }) => {
+  await po.setUp();
+  await po.sendPrompt("[increment]");
+  await po.snapshotMessages();
+
+  await po.dismissAllToasts();
+  await po.clickRetry();
+  await po.expectNoToast();
+  // The counter should be incremented in the snapshotted messages.
+  await po.snapshotMessages();
+});

e2e-tests/snapshots/retry.spec.ts_retry---should-work-1.aria.yml

@@ -0,0 +1,4 @@
+- paragraph: "[increment]"
+- paragraph: counter=1
+- button "Retry":
+  - img

e2e-tests/snapshots/retry.spec.ts_retry---should-work-2.aria.yml

@@ -0,0 +1,4 @@
+- paragraph: "[increment]"
+- paragraph: counter=2
+- button "Retry":
+  - img

playwright.config.ts

@@ -2,8 +2,9 @@ import { PlaywrightTestConfig } from "@playwright/test";
 
 const config: PlaywrightTestConfig = {
   testDir: "./e2e-tests",
+  workers: 1,
   maxFailures: 1,
-  timeout: process.env.CI ? 30_000 : 15_000,
+  timeout: process.env.CI ? 60_000 : 15_000,
   // Use a custom snapshot path template because Playwright's default
   // is platform-specific which isn't necessary for Dyad e2e tests
   // which should be platform agnostic (we don't do screenshots; only textual diffs).

testing/fake-llm-server/index.ts

@@ -56,6 +56,8 @@ app.get("/health", (req, res) => {
   res.send("OK");
 });
 
+let globalCounter = 0;
+
 // Handle POST requests to /v1/chat/completions
 app.post("/v1/chat/completions", (req, res) => {
   const { stream = false, messages = [] } = req.body;
@@ -74,8 +76,40 @@ app.post("/v1/chat/completions", (req, res) => {
     });
   }
 
-  // Check if the last message starts with "tc=" to load test case file
   let messageContent = CANNED_MESSAGE;
+
+  // Check if the last message is "[dump]" to write messages to file and return path
+  if (lastMessage && lastMessage.content === "[dump]") {
+    const timestamp = Date.now();
+    const generatedDir = path.join(__dirname, "generated");
+
+    // Create generated directory if it doesn't exist
+    if (!fs.existsSync(generatedDir)) {
+      fs.mkdirSync(generatedDir, { recursive: true });
+    }
+
+    const dumpFilePath = path.join(generatedDir, `${timestamp}.json`);
+
+    try {
+      fs.writeFileSync(
+        dumpFilePath,
+        JSON.stringify(messages, null, 2),
+        "utf-8",
+      );
+      console.log(`* Dumped messages to: ${dumpFilePath}`);
+      messageContent = `[[dyad-dump-path=${dumpFilePath}]]`;
+    } catch (error) {
+      console.error(`* Error writing dump file: ${error}`);
+      messageContent = `Error: Could not write dump file: ${error}`;
+    }
+  }
+
+  if (lastMessage && lastMessage.content === "[increment]") {
+    globalCounter++;
+    messageContent = `counter=${globalCounter}`;
+  }
+
+  // Check if the last message starts with "tc=" to load test case file
   if (
     lastMessage &&
     lastMessage.content &&