make it easy to write multiple e2e tests (#280)

e2e-tests/fixtures/basic.md

@@ -0,0 +1 @@
+This is a simple basic response

e2e-tests/helpers/test_helper.ts

@@ -1,9 +1,86 @@
-import { test as base } from "@playwright/test";
+import { test as base, Page, expect } from "@playwright/test";
 import { findLatestBuild, parseElectronApp } from "electron-playwright-helpers";
 import { ElectronApplication, _electron as electron } from "playwright";
 
 const showDebugLogs = process.env.DEBUG_LOGS === "true";
 
+class PageObject {
+  constructor(private page: Page) {}
+
+  async setUp() {
+    await this.goToSettingsTab();
+    await this.setUpTestProvider();
+    await this.setUpTestModel();
+    await this.goToAppsTab();
+    await this.selectTestModel();
+  }
+
+  async dumpMessages() {
+    await expect(this.page.getByTestId("messages-list")).toMatchAriaSnapshot();
+  }
+
+  async waitForChatCompletion() {
+    await expect(
+      this.page.getByRole("button", { name: "Retry" }),
+    ).toBeVisible();
+  }
+
+  async sendPrompt(prompt: string) {
+    await this.page
+      .getByRole("textbox", { name: "Ask Dyad to build..." })
+      .click();
+    await this.page
+      .getByRole("textbox", { name: "Ask Dyad to build..." })
+      .fill(prompt);
+    await this.page.getByRole("button", { name: "Start new chat" }).click();
+    await this.waitForChatCompletion();
+  }
+
+  async selectTestModel() {
+    await this.page.getByRole("button", { name: "Model: Auto" }).click();
+    await this.page.getByText("test-provider").click();
+    await this.page.getByText("test-model").click();
+  }
+
+  async setUpTestProvider() {
+    await this.page.getByText("Add custom providerConnect to").click();
+    // Fill out provider dialog
+    await this.page
+      .getByRole("textbox", { name: "Provider ID" })
+      .fill("testing");
+    await this.page.getByRole("textbox", { name: "Display Name" }).click();
+    await this.page
+      .getByRole("textbox", { name: "Display Name" })
+      .fill("test-provider");
+    await this.page.getByText("API Base URLThe base URL for").click();
+    await this.page
+      .getByRole("textbox", { name: "API Base URL" })
+      .fill("http://localhost:3500/v1");
+    await this.page.getByRole("button", { name: "Add Provider" }).click();
+  }
+
+  async setUpTestModel() {
+    await this.page
+      .getByRole("heading", { name: "test-provider Needs Setup" })
+      .click();
+    await this.page.getByRole("button", { name: "Add Custom Model" }).click();
+    await this.page
+      .getByRole("textbox", { name: "Model ID*" })
+      .fill("test-model");
+    await this.page.getByRole("textbox", { name: "Model ID*" }).press("Tab");
+    await this.page.getByRole("textbox", { name: "Name*" }).fill("test-model");
+    await this.page.getByRole("button", { name: "Add Model" }).click();
+  }
+
+  async goToSettingsTab() {
+    await this.page.getByRole("link", { name: "Settings" }).click();
+  }
+
+  async goToAppsTab() {
+    await this.page.getByRole("link", { name: "Apps" }).click();
+  }
+}
+
 // From https://github.com/microsoft/playwright/issues/8208#issuecomment-1435475930
 //
 // Note how we mark the fixture as { auto: true }.
@@ -11,7 +88,17 @@ const showDebugLogs = process.env.DEBUG_LOGS === "true";
 export const test = base.extend<{
   attachScreenshotsToReport: void;
   electronApp: ElectronApplication;
+  po: PageObject;
 }>({
+  po: [
+    async ({ electronApp }, use) => {
+      const page = await electronApp.firstWindow();
+
+      const po = new PageObject(page);
+      await use(po);
+    },
+    { auto: true },
+  ],
   attachScreenshotsToReport: [
     async ({ page }, use, testInfo) => {
       await use();

e2e-tests/main.spec.ts

@@ -8,45 +8,14 @@ test("renders the first page", async ({ electronApp }) => {
   expect(text).toBe("Build your dream app");
 });
 
-test("simple message to custom test model", async ({ electronApp }) => {
-  const page = await electronApp.firstWindow();
-  await page.getByRole("link", { name: "Settings" }).click();
-  await page.getByText("Add custom providerConnect to").click();
-
-  // Fill out provider dialog
-  await page.getByRole("textbox", { name: "Provider ID" }).fill("testing");
-  await page.getByRole("textbox", { name: "Display Name" }).click();
-  await page
-    .getByRole("textbox", { name: "Display Name" })
-    .fill("test-provider");
-  await page.getByText("API Base URLThe base URL for").click();
-  await page
-    .getByRole("textbox", { name: "API Base URL" })
-    .fill("http://localhost:3500/v1");
-  await page.getByRole("button", { name: "Add Provider" }).click();
-
-  // Create custom model
-  await page
-    .getByRole("heading", { name: "test-provider Needs Setup" })
-    .click();
-  await page.getByRole("button", { name: "Add Custom Model" }).click();
-  await page.getByRole("textbox", { name: "Model ID*" }).fill("test-model");
-  await page.getByRole("textbox", { name: "Model ID*" }).press("Tab");
-  await page.getByRole("textbox", { name: "Name*" }).fill("test-model");
-  await page.getByRole("button", { name: "Add Model" }).click();
-
-  // Go to apps page and select custom model
-  await page.getByRole("link", { name: "Apps" }).click();
-  await page.getByRole("button", { name: "Model: Auto" }).click();
-  await page.getByText("test-provider").click();
-  await page.getByText("test-model").click();
-
-  // Enter prompt and send
-  await page.getByRole("textbox", { name: "Ask Dyad to build..." }).click();
-  await page.getByRole("textbox", { name: "Ask Dyad to build..." }).fill("hi");
-  await page.getByRole("button", { name: "Start new chat" }).click();
-
-  // Make sure it's done
-  await expect(page.getByRole("button", { name: "Retry" })).not.toBeVisible();
-  await expect(page.getByTestId("messages-list")).toMatchAriaSnapshot();
+test("simple message to custom test model", async ({ po }) => {
+  await po.setUp();
+  await po.sendPrompt("hi");
+  await po.dumpMessages();
+});
+
+test("basic message to custom test model", async ({ po }) => {
+  await po.setUp();
+  await po.sendPrompt("tc=basic");
+  await po.dumpMessages();
 });

e2e-tests/snapshots/main.spec.ts_basic-message-to-custom-test-model-1.aria.yml

@@ -0,0 +1,4 @@
+- paragraph: tc=basic
+- paragraph: This is a simple basic response
+- button "Retry":
+  - img

playwright.config.ts

@@ -3,7 +3,7 @@ import { PlaywrightTestConfig } from "@playwright/test";
 const config: PlaywrightTestConfig = {
   testDir: "./e2e-tests",
   maxFailures: 1,
-  timeout: process.env.CI ? 30_000 : 10_000,
+  timeout: process.env.CI ? 30_000 : 15_000,
   // Use a custom snapshot path template because Playwright's default
   // is platform-specific which isn't necessary for Dyad e2e tests
   // which should be platform agnostic (we don't do screenshots; only textual diffs).

src/app/TitleBar.tsx

@@ -72,6 +72,7 @@ export const TitleBar = () => {
         <div className="pl-20"></div>
         <img src={logo} alt="Dyad Logo" className="w-6 h-6 mr-2" />
         <Button
+          data-testid="title-bar-app-name-button"
           variant="outline"
           size="sm"
           className={`hidden @md:block no-app-region-drag text-sm font-medium ${
@@ -83,6 +84,7 @@ export const TitleBar = () => {
         </Button>
         {isDyadPro && (
           <Button
+            data-testid="title-bar-dyad-pro-button"
             onClick={() => {
               navigate({
                 to: providerSettingsRoute.id,

src/lib/schemas.ts

@@ -122,6 +122,11 @@ export const UserSettingsSchema = z.object({
   enableProSmartFilesContextMode: z.boolean().optional(),
   selectedTemplateId: z.string().optional(),
 
+  ////////////////////////////////
+  // E2E TESTING ONLY.
+  ////////////////////////////////
+  isTestMode: z.boolean().optional(),
+
   ////////////////////////////////
   // DEPRECATED.
   ////////////////////////////////

src/main.ts

@@ -58,6 +58,11 @@ export async function onFirstRunMaybe() {
       hasRunBefore: true,
     });
   }
+  if (process.env.E2E_TEST_BUILD) {
+    writeSettings({
+      isTestMode: true,
+    });
+  }
 }
 
 /**

src/pages/app-details.tsx

@@ -161,7 +161,10 @@ export default function AppDetailsPage() {
   const fullAppPath = appBasePath.replace("$APP_BASE_PATH", selectedApp.path);
 
   return (
-    <div className="relative min-h-screen p-4 w-full">
+    <div
+      className="relative min-h-screen p-4 w-full"
+      data-testid="app-details-page"
+    >
       <Button
         onClick={() => router.history.back()}
         variant="outline"

src/pages/home.tsx

@@ -124,7 +124,9 @@ export default function HomePage() {
         chatId: result.chatId,
         attachments,
       });
-      await new Promise((resolve) => setTimeout(resolve, 2000));
+      await new Promise((resolve) =>
+        setTimeout(resolve, settings?.isTestMode ? 0 : 2000),
+      );
 
       setInputValue("");
       setSelectedAppId(result.app.id);

src/paths/paths.ts

@@ -2,6 +2,9 @@ import path from "node:path";
 import os from "node:os";
 
 export function getDyadAppPath(appPath: string): string {
+  if (process.env.E2E_TEST_BUILD) {
+    return path.join("/tmp", "dyad-apps-test", appPath);
+  }
   return path.join(os.homedir(), "dyad-apps", appPath);
 }
 

testing/fake-llm-server/index.ts

@@ -1,6 +1,8 @@
 import express from "express";
 import { createServer } from "http";
 import cors from "cors";
+import fs from "fs";
+import path from "path";
 
 // Create Express app
 const app = express();
@@ -72,6 +74,38 @@ app.post("/v1/chat/completions", (req, res) => {
     });
   }
 
+  // Check if the last message starts with "tc=" to load test case file
+  let messageContent = CANNED_MESSAGE;
+  if (
+    lastMessage &&
+    lastMessage.content &&
+    lastMessage.content.startsWith("tc=")
+  ) {
+    const testCaseName = lastMessage.content.slice(3); // Remove "tc=" prefix
+    const testFilePath = path.join(
+      __dirname,
+      "..",
+      "..",
+      "..",
+      "e2e-tests",
+      "fixtures",
+      `${testCaseName}.md`,
+    );
+
+    try {
+      if (fs.existsSync(testFilePath)) {
+        messageContent = fs.readFileSync(testFilePath, "utf-8");
+        console.log(`* Loaded test case: ${testCaseName}`);
+      } else {
+        console.log(`* Test case file not found: ${testFilePath}`);
+        messageContent = `Error: Test case file not found: ${testCaseName}.md`;
+      }
+    } catch (error) {
+      console.error(`* Error reading test case file: ${error}`);
+      messageContent = `Error: Could not read test case file: ${testCaseName}.md`;
+    }
+  }
+
   // Non-streaming response
   if (!stream) {
     return res.json({
@@ -84,7 +118,7 @@ app.post("/v1/chat/completions", (req, res) => {
           index: 0,
           message: {
             role: "assistant",
-            content: CANNED_MESSAGE,
+            content: messageContent,
           },
           finish_reason: "stop",
         },
@@ -97,27 +131,30 @@ app.post("/v1/chat/completions", (req, res) => {
   res.setHeader("Cache-Control", "no-cache");
   res.setHeader("Connection", "keep-alive");
 
-  // Split the "hello world" message into characters to simulate streaming
-  const message = CANNED_MESSAGE;
+  // Split the message into characters to simulate streaming
+  const message = messageContent;
   const messageChars = message.split("");
 
   // Stream each character with a delay
   let index = 0;
+  const batchSize = 8;
 
   // Send role first
   res.write(createStreamChunk("", "assistant"));
 
   const interval = setInterval(() => {
     if (index < messageChars.length) {
-      res.write(createStreamChunk(messageChars[index]));
-      index++;
+      // Get the next batch of characters (up to batchSize)
+      const batch = messageChars.slice(index, index + batchSize).join("");
+      res.write(createStreamChunk(batch));
+      index += batchSize;
     } else {
       // Send the final chunk
       res.write(createStreamChunk("", "assistant", true));
       clearInterval(interval);
       res.end();
     }
-  }, 10);
+  }, 1);
 });
 
 // Start the server