Boilerplate free tests (#277)

.github/workflows/ci.yml

@@ -41,6 +41,8 @@ jobs:
         run: npx playwright install chromium --with-deps
       - name: Build
         run: npm run pre:e2e
+      - name: Prep test server
+        run: cd testing/fake-llm-server && npm install && npm run build && cd -
       - name: E2E tests
         # Add debug logging to make it easier to see what's failing
         run: DEBUG=pw:browser npm run e2e

.gitignore

@@ -6,6 +6,9 @@ yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
 
+# dist
+dist/
+
 # playwright
 playwright-report/
 test-results/

.oxlintrc.json

@@ -1,5 +1,6 @@
 {
   "rules": {
-    "eslint/no-unused-vars": "error"
+    "eslint/no-unused-vars": "error",
+    "eslint/no-empty-pattern": "off"
   }
 }

.prettierignore

@@ -4,3 +4,4 @@ coverage
 # generated files
 drizzle/
 **/pnpm-lock.yaml
+**/snapshots/**

e2e-tests/helpers/codegen.js

@@ -0,0 +1,24 @@
+/*
+ * From: https://github.com/microsoft/playwright/issues/5181#issuecomment-2769098576
+ *
+ * Usage:
+ * cd e2e-tests/helpers && node codegen.js
+ */
+
+const { _electron: electron } = require("playwright");
+
+(async () => {
+  const browser = await electron.launch({
+    args: [
+      "../../out/dyad-darwin-arm64/dyad.app/Contents/Resources/app.asar/.vite/build/main.js",
+      "--enable-logging",
+      "--user-data-dir=/tmp/dyad-e2e-tests",
+    ],
+    executablePath: "../../out/dyad-darwin-arm64/dyad.app/Contents/MacOS/dyad",
+  });
+  const context = await browser.context();
+  await context.route("**/*", (route) => route.continue());
+
+  await require("node:timers/promises").setTimeout(3000); // wait for the window to load
+  await browser.windows()[0].pause(); // .pause() opens the Playwright-Inspector for manual recording
+})();

e2e-tests/helpers/test_helper.ts

@@ -0,0 +1,78 @@
+import { test as base } from "@playwright/test";
+import { findLatestBuild, parseElectronApp } from "electron-playwright-helpers";
+import { ElectronApplication, _electron as electron } from "playwright";
+
+const showDebugLogs = process.env.DEBUG_LOGS === "true";
+
+// From https://github.com/microsoft/playwright/issues/8208#issuecomment-1435475930
+//
+// Note how we mark the fixture as { auto: true }.
+// This way it is always instantiated, even if the test does not use it explicitly.
+export const test = base.extend<{
+  attachScreenshotsToReport: void;
+  electronApp: ElectronApplication;
+}>({
+  attachScreenshotsToReport: [
+    async ({ page }, use, testInfo) => {
+      await use();
+
+      // After the test we can check whether the test passed or failed.
+      if (testInfo.status !== testInfo.expectedStatus) {
+        const screenshot = await page.screenshot();
+        await testInfo.attach("screenshot", {
+          body: screenshot,
+          contentType: "image/png",
+        });
+      }
+    },
+    { auto: true },
+  ],
+  electronApp: async ({}, use) => {
+    // find the latest build in the out directory
+    const latestBuild = findLatestBuild();
+    // parse the directory and find paths and other info
+    const appInfo = parseElectronApp(latestBuild);
+    process.env.E2E_TEST_BUILD = "true";
+    // This is just a hack to avoid the AI setup screen.
+    process.env.OPENAI_API_KEY = "sk-test";
+    const electronApp = await electron.launch({
+      args: [
+        appInfo.main,
+        "--enable-logging",
+        `--user-data-dir=/tmp/dyad-e2e-tests-${Date.now()}`,
+      ],
+      executablePath: appInfo.executable,
+    });
+
+    console.log("electronApp launched!");
+    if (showDebugLogs) {
+      // Listen to main process output immediately
+      electronApp.process().stdout?.on("data", (data) => {
+        console.log(`MAIN_PROCESS_STDOUT: ${data.toString()}`);
+      });
+      electronApp.process().stderr?.on("data", (data) => {
+        console.error(`MAIN_PROCESS_STDERR: ${data.toString()}`);
+      });
+    }
+    electronApp.on("close", () => {
+      console.log(`Electron app closed listener:`);
+    });
+
+    electronApp.on("window", async (page) => {
+      const filename = page.url()?.split("/").pop();
+      console.log(`Window opened: ${filename}`);
+
+      // capture errors
+      page.on("pageerror", (error) => {
+        console.error(error);
+      });
+      // capture console messages
+      page.on("console", (msg) => {
+        console.log(msg.text());
+      });
+    });
+
+    await use(electronApp);
+    await electronApp.close();
+  },
+});

e2e-tests/main.spec.ts

@@ -1,91 +1,52 @@
-/**
+import { expect } from "@playwright/test";
- * Example Playwright script for Electron
+import { test } from "./helpers/test_helper";
- * showing/testing various API features
- * in both renderer and main processes
- */
 
-import { expect, test as base } from "@playwright/test";
+test("renders the first page", async ({ electronApp }) => {
-import { findLatestBuild, parseElectronApp } from "electron-playwright-helpers";
+  const page = await electronApp.firstWindow();
-import { ElectronApplication, Page, _electron as electron } from "playwright";
-
-let electronApp: ElectronApplication;
-
-// From https://github.com/microsoft/playwright/issues/8208#issuecomment-1435475930
-//
-// Note how we mark the fixture as { auto: true }.
-// This way it is always instantiated, even if the test does not use it explicitly.
-export const test = base.extend<{ attachScreenshotsToReport: void }>({
-  attachScreenshotsToReport: [
-    async ({}, use, testInfo) => {
-      await use();
-
-      // After the test we can check whether the test passed or failed.
-      if (testInfo.status !== testInfo.expectedStatus) {
-        const screenshot = await page.screenshot();
-        await testInfo.attach("screenshot", {
-          body: screenshot,
-          contentType: "image/png",
-        });
-      }
-    },
-    { auto: true },
-  ],
-});
-
-test.beforeAll(async () => {
-  // find the latest build in the out directory
-  const latestBuild = findLatestBuild();
-  // parse the directory and find paths and other info
-  const appInfo = parseElectronApp(latestBuild);
-  process.env.E2E_TEST_BUILD = "true";
-  // This is just a hack to avoid the AI setup screen.
-  process.env.OPENAI_API_KEY = "sk-test";
-  electronApp = await electron.launch({
-    args: [
-      appInfo.main,
-      "--enable-logging",
-      "--user-data-dir=/tmp/dyad-e2e-tests",
-    ],
-    executablePath: appInfo.executable,
-  });
-
-  console.log("electronApp launched!");
-
-  // Listen to main process output immediately
-  electronApp.process().stdout?.on("data", (data) => {
-    console.log(`MAIN_PROCESS_STDOUT: ${data.toString()}`);
-  });
-  electronApp.process().stderr?.on("data", (data) => {
-    console.error(`MAIN_PROCESS_STDERR: ${data.toString()}`);
-  });
-  electronApp.on("close", () => {
-    console.log(`Electron app closed listener:`);
-  });
-
-  electronApp.on("window", async (page) => {
-    const filename = page.url()?.split("/").pop();
-    console.log(`Window opened: ${filename}`);
-
-    // capture errors
-    page.on("pageerror", (error) => {
-      console.error(error);
-    });
-    // capture console messages
-    page.on("console", (msg) => {
-      console.log(msg.text());
-    });
-  });
-});
-
-test.afterAll(async () => {
-  await electronApp.close();
-});
-
-let page: Page;
-
-test("renders the first page", async () => {
-  page = await electronApp.firstWindow();
   await page.waitForSelector("h1");
   const text = await page.$eval("h1", (el) => el.textContent);
   expect(text).toBe("Build your dream app");
 });
+
+test("simple message to custom test model", async ({ electronApp }) => {
+  const page = await electronApp.firstWindow();
+  await page.getByRole("link", { name: "Settings" }).click();
+  await page.getByText("Add custom providerConnect to").click();
+
+  // Fill out provider dialog
+  await page.getByRole("textbox", { name: "Provider ID" }).fill("testing");
+  await page.getByRole("textbox", { name: "Display Name" }).click();
+  await page
+    .getByRole("textbox", { name: "Display Name" })
+    .fill("test-provider");
+  await page.getByText("API Base URLThe base URL for").click();
+  await page
+    .getByRole("textbox", { name: "API Base URL" })
+    .fill("http://localhost:3500/v1");
+  await page.getByRole("button", { name: "Add Provider" }).click();
+
+  // Create custom model
+  await page
+    .getByRole("heading", { name: "test-provider Needs Setup" })
+    .click();
+  await page.getByRole("button", { name: "Add Custom Model" }).click();
+  await page.getByRole("textbox", { name: "Model ID*" }).fill("test-model");
+  await page.getByRole("textbox", { name: "Model ID*" }).press("Tab");
+  await page.getByRole("textbox", { name: "Name*" }).fill("test-model");
+  await page.getByRole("button", { name: "Add Model" }).click();
+
+  // Go to apps page and select custom model
+  await page.getByRole("link", { name: "Apps" }).click();
+  await page.getByRole("button", { name: "Model: Auto" }).click();
+  await page.getByText("test-provider").click();
+  await page.getByText("test-model").click();
+
+  // Enter prompt and send
+  await page.getByRole("textbox", { name: "Ask Dyad to build..." }).click();
+  await page.getByRole("textbox", { name: "Ask Dyad to build..." }).fill("hi");
+  await page.getByRole("button", { name: "Start new chat" }).click();
+
+  // Make sure it's done
+  await expect(page.getByRole("button", { name: "Retry" })).not.toBeVisible();
+  await expect(page.getByTestId("messages-list")).toMatchAriaSnapshot();
+});

e2e-tests/snapshots/main.spec.ts_simple-message-to-custom-test-model-1.aria.yml

@@ -0,0 +1,16 @@
+- paragraph: hi
+- 'button "Thinking `＜dyad-write>`: I''ll think about the problem and write a bug report. ＜dyad-write> ＜dyad-write path=\"file1.txt\"> Fake dyad write ＜/dyad-write>"':
+  - img
+  - img
+  - paragraph:
+    - code: "`＜dyad-write>`"
+    - text: ": I'll think about the problem and write a bug report."
+  - paragraph: ＜dyad-write>
+  - paragraph: ＜dyad-write path="file1.txt"> Fake dyad write ＜/dyad-write>
+- img
+- text: file1.txt
+- img
+- text: file1.txt
+- paragraph: More EOM
+- button "Retry":
+  - img

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "dyad",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "dyad",
-      "version": "0.6.0",
+      "version": "0.7.0",
       "license": "MIT",
       "dependencies": {
         "@ai-sdk/anthropic": "^1.2.8",

playwright.config.ts

@@ -3,6 +3,12 @@ import { PlaywrightTestConfig } from "@playwright/test";
 const config: PlaywrightTestConfig = {
   testDir: "./e2e-tests",
   maxFailures: 1,
+  timeout: process.env.CI ? 30_000 : 10_000,
+  // Use a custom snapshot path template because Playwright's default
+  // is platform-specific which isn't necessary for Dyad e2e tests
+  // which should be platform agnostic (we don't do screenshots; only textual diffs).
+  snapshotPathTemplate:
+    "{testDir}/{testFileDir}/snapshots/{testFileName}_{arg}{ext}",
 
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: "html",
@@ -18,6 +24,11 @@ const config: PlaywrightTestConfig = {
     // screenshot: "on",
     // video: "retain-on-failure",
   },
+
+  webServer: {
+    command: `cd testing/fake-llm-server && npm start`,
+    url: "http://localhost:3500/health",
+  },
 };
 
 export default config;

src/components/chat/MessagesList.tsx

@@ -34,7 +34,11 @@ export const MessagesList = forwardRef<HTMLDivElement, MessagesListProps>(
     const selectedChatId = useAtomValue(selectedChatIdAtom);
 
     return (
-      <div className="flex-1 overflow-y-auto p-4" ref={ref}>
+      <div
+        className="flex-1 overflow-y-auto p-4"
+        ref={ref}
+        data-testid="messages-list"
+      >
         {messages.length > 0 ? (
           messages.map((message, index) => (
             <ChatMessage

testing/fake-llm-server/dist/index.js

@@ -1,90 +0,0 @@
-"use strict";
-var __importDefault =
-  (this && this.__importDefault) ||
-  function (mod) {
-    return mod && mod.__esModule ? mod : { default: mod };
-  };
-Object.defineProperty(exports, "__esModule", { value: true });
-const express_1 = __importDefault(require("express"));
-const http_1 = require("http");
-const cors_1 = __importDefault(require("cors"));
-// Create Express app
-const app = (0, express_1.default)();
-app.use((0, cors_1.default)());
-app.use(express_1.default.json());
-const PORT = 3500;
-// Helper function to create OpenAI-like streaming response chunks
-function createStreamChunk(content, role = "assistant", isLast = false) {
-  const chunk = {
-    id: `chatcmpl-${Date.now()}`,
-    object: "chat.completion.chunk",
-    created: Math.floor(Date.now() / 1000),
-    model: "fake-model",
-    choices: [
-      {
-        index: 0,
-        delta: isLast ? {} : { content, role },
-        finish_reason: isLast ? "stop" : null,
-      },
-    ],
-  };
-  return `data: ${JSON.stringify(chunk)}\n\n${isLast ? "data: [DONE]\n\n" : ""}`;
-}
-// Handle POST requests to /v1/chat/completions
-app.post("/v1/chat/completions", (req, res) => {
-  const { stream = false } = req.body;
-  // Non-streaming response
-  if (!stream) {
-    return res.json({
-      id: `chatcmpl-${Date.now()}`,
-      object: "chat.completion",
-      created: Math.floor(Date.now() / 1000),
-      model: "fake-model",
-      choices: [
-        {
-          index: 0,
-          message: {
-            role: "assistant",
-            content: "hello world",
-          },
-          finish_reason: "stop",
-        },
-      ],
-    });
-  }
-  // Streaming response
-  res.setHeader("Content-Type", "text/event-stream");
-  res.setHeader("Cache-Control", "no-cache");
-  res.setHeader("Connection", "keep-alive");
-  // Split the "hello world" message into characters to simulate streaming
-  const message = "hello world";
-  const messageChars = message.split("");
-  // Stream each character with a delay
-  let index = 0;
-  // Send role first
-  res.write(createStreamChunk("", "assistant"));
-  const interval = setInterval(() => {
-    if (index < messageChars.length) {
-      res.write(createStreamChunk(messageChars[index]));
-      index++;
-    } else {
-      // Send the final chunk
-      res.write(createStreamChunk("", "assistant", true));
-      clearInterval(interval);
-      res.end();
-    }
-  }, 100);
-});
-// Start the server
-const server = (0, http_1.createServer)(app);
-server.listen(PORT, () => {
-  console.log(`Fake LLM server running on http://localhost:${PORT}`);
-});
-// Handle SIGINT (Ctrl+C)
-process.on("SIGINT", () => {
-  console.log("Shutting down fake LLM server");
-  server.close(() => {
-    console.log("Server closed");
-    process.exit(0);
-  });
-});

testing/fake-llm-server/index.ts

@@ -50,6 +50,10 @@ const CANNED_MESSAGE = `
   More
   EOM`;
 
+app.get("/health", (req, res) => {
+  res.send("OK");
+});
+
 // Handle POST requests to /v1/chat/completions
 app.post("/v1/chat/completions", (req, res) => {
   const { stream = false, messages = [] } = req.body;