Simple playwright e2e test (#234)

.github/workflows/ci.yml

@@ -17,7 +17,11 @@ defaults:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    # Why Mac?
+    # I can't run electron playwright on ubuntu-latest and
+    # Linux support for Dyad is experimental so not as important
+    # as Mac + Windows
+    runs-on: macos-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -33,3 +37,16 @@ jobs:
         run: npm run presubmit
       - name: Type-checking
         run: npm run ts
+      - name: Install Chromium browser for Playwright
+        run: npx playwright install chromium --with-deps
+      - name: Build
+        run: npm run pre:e2e
+      - name: E2E tests
+        # Add debug logging to make it easier to see what's failing
+        run: DEBUG=pw:browser npm run e2e
+      - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        if: failure()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 3

.gitignore

@@ -6,6 +6,10 @@ yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
 
+# playwright
+playwright-report/
+test-results/
+
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 

e2e-tests/main.spec.ts

@@ -0,0 +1,91 @@
+/**
+ * Example Playwright script for Electron
+ * showing/testing various API features
+ * in both renderer and main processes
+ */
+
+import { expect, test as base } from "@playwright/test";
+import { findLatestBuild, parseElectronApp } from "electron-playwright-helpers";
+import { ElectronApplication, Page, _electron as electron } from "playwright";
+
+let electronApp: ElectronApplication;
+
+// From https://github.com/microsoft/playwright/issues/8208#issuecomment-1435475930
+//
+// Note how we mark the fixture as { auto: true }.
+// This way it is always instantiated, even if the test does not use it explicitly.
+export const test = base.extend<{ attachScreenshotsToReport: void }>({
+  attachScreenshotsToReport: [
+    async ({}, use, testInfo) => {
+      await use();
+
+      // After the test we can check whether the test passed or failed.
+      if (testInfo.status !== testInfo.expectedStatus) {
+        const screenshot = await page.screenshot();
+        await testInfo.attach("screenshot", {
+          body: screenshot,
+          contentType: "image/png",
+        });
+      }
+    },
+    { auto: true },
+  ],
+});
+
+test.beforeAll(async () => {
+  // find the latest build in the out directory
+  const latestBuild = findLatestBuild();
+  // parse the directory and find paths and other info
+  const appInfo = parseElectronApp(latestBuild);
+  process.env.E2E_TEST_BUILD = "true";
+  // This is just a hack to avoid the AI setup screen.
+  process.env.OPENAI_API_KEY = "sk-test";
+  electronApp = await electron.launch({
+    args: [
+      appInfo.main,
+      "--enable-logging",
+      "--user-data-dir=/tmp/dyad-e2e-tests",
+    ],
+    executablePath: appInfo.executable,
+  });
+
+  console.log("electronApp launched!");
+
+  // Listen to main process output immediately
+  electronApp.process().stdout?.on("data", (data) => {
+    console.log(`MAIN_PROCESS_STDOUT: ${data.toString()}`);
+  });
+  electronApp.process().stderr?.on("data", (data) => {
+    console.error(`MAIN_PROCESS_STDERR: ${data.toString()}`);
+  });
+  electronApp.on("close", () => {
+    console.log(`Electron app closed listener:`);
+  });
+
+  electronApp.on("window", async (page) => {
+    const filename = page.url()?.split("/").pop();
+    console.log(`Window opened: ${filename}`);
+
+    // capture errors
+    page.on("pageerror", (error) => {
+      console.error(error);
+    });
+    // capture console messages
+    page.on("console", (msg) => {
+      console.log(msg.text());
+    });
+  });
+});
+
+test.afterAll(async () => {
+  await electronApp.close();
+});
+
+let page: Page;
+
+test("renders the first page", async () => {
+  page = await electronApp.firstWindow();
+  await page.waitForSelector("h1");
+  const text = await page.$eval("h1", (el) => el.textContent);
+  expect(text).toBe("Build your dream app");
+});

forge.config.ts

@@ -41,6 +41,8 @@ const ignore = (file: string) => {
   return true;
 };
 
+const isEndToEndTestBuild = process.env.E2E_TEST_BUILD === "true";
+
 const config: ForgeConfig = {
   packagerConfig: {
     protocols: [
@@ -51,14 +53,18 @@ const config: ForgeConfig = {
     ],
     icon: "./assets/icon/logo",
 
-    osxSign: {
+    osxSign: isEndToEndTestBuild
-      identity: process.env.APPLE_TEAM_ID,
+      ? undefined
-    },
+      : {
-    osxNotarize: {
+          identity: process.env.APPLE_TEAM_ID,
-      appleId: process.env.APPLE_ID!,
+        },
-      appleIdPassword: process.env.APPLE_PASSWORD!,
+    osxNotarize: isEndToEndTestBuild
-      teamId: process.env.APPLE_TEAM_ID!,
+      ? undefined
-    },
+      : {
+          appleId: process.env.APPLE_ID!,
+          appleIdPassword: process.env.APPLE_PASSWORD!,
+          teamId: process.env.APPLE_TEAM_ID!,
+        },
     asar: true,
     ignore,
     // ignore: [/node_modules\/(?!(better-sqlite3|bindings|file-uri-to-path)\/)/],
@@ -124,7 +130,7 @@ const config: ForgeConfig = {
       [FuseV1Options.RunAsNode]: false,
       [FuseV1Options.EnableCookieEncryption]: true,
       [FuseV1Options.EnableNodeOptionsEnvironmentVariable]: false,
-      [FuseV1Options.EnableNodeCliInspectArguments]: false,
+      [FuseV1Options.EnableNodeCliInspectArguments]: isEndToEndTestBuild,
       [FuseV1Options.EnableEmbeddedAsarIntegrityValidation]: true,
       [FuseV1Options.OnlyLoadAppFromAsar]: true,
     }),

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "dyad",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "dyad",
-      "version": "0.5.0",
+      "version": "0.6.0",
       "license": "MIT",
       "dependencies": {
         "@ai-sdk/anthropic": "^1.2.8",
@@ -45,6 +45,7 @@
         "dotenv": "^16.4.7",
         "drizzle-orm": "^0.41.0",
         "electron-log": "^5.3.3",
+        "electron-playwright-helpers": "^1.7.1",
         "electron-squirrel-startup": "^1.0.1",
         "esbuild-register": "^3.6.0",
         "fix-path": "^4.0.0",
@@ -85,6 +86,7 @@
         "@electron-forge/plugin-vite": "^7.8.0",
         "@electron-forge/publisher-github": "^7.8.0",
         "@electron/fuses": "^1.8.0",
+        "@playwright/test": "^1.52.0",
         "@testing-library/react": "^16.3.0",
         "@types/better-sqlite3": "^7.6.13",
         "@types/kill-port": "^2.0.3",
@@ -1140,7 +1142,6 @@
       "version": "3.4.1",
       "resolved": "https://registry.npmjs.org/@electron/asar/-/asar-3.4.1.tgz",
       "integrity": "sha512-i4/rNPRS84t0vSRa2HorerGRXWyF4vThfHesw0dmcWHp+cspK743UanA0suA5Q5y8kzY2y6YKrvbIUn69BCAiA==",
-      "dev": true,
       "license": "MIT",
       "dependencies": {
         "commander": "^5.0.0",
@@ -1158,7 +1159,6 @@
       "version": "5.1.0",
       "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz",
       "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==",
-      "dev": true,
       "license": "MIT",
       "engines": {
         "node": ">= 6"
@@ -3726,6 +3726,22 @@
       "devOptional": true,
       "license": "MIT"
     },
+    "node_modules/@playwright/test": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.52.0.tgz",
+      "integrity": "sha512-uh6W7sb55hl7D6vsAeA+V2p5JnlAqzhqFyF0VcJkKZXkgnFcVG9PziERRHQfPLfNGx1C292a4JqbWzhR8L4R1g==",
+      "devOptional": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright": "1.52.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/@polka/url": {
       "version": "1.0.0-next.29",
       "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz",
@@ -6925,7 +6941,6 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
-      "dev": true,
       "license": "MIT"
     },
     "node_modules/base64-js": {
@@ -7055,7 +7070,6 @@
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
       "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
-      "dev": true,
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0",
@@ -7781,7 +7795,6 @@
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
       "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
-      "dev": true,
       "license": "MIT"
     },
     "node_modules/content-disposition": {
@@ -9387,6 +9400,15 @@
         "node": ">= 14"
       }
     },
+    "node_modules/electron-playwright-helpers": {
+      "version": "1.7.1",
+      "resolved": "https://registry.npmjs.org/electron-playwright-helpers/-/electron-playwright-helpers-1.7.1.tgz",
+      "integrity": "sha512-S9mo7LfpERgub2WIuYVPpib4XKFeAqBP+mxYf5Bv7E0B5GUB+LUbSj6Fpu39h18Ar635Nf9nQYTmypjuvaYJng==",
+      "license": "MIT",
+      "dependencies": {
+        "@electron/asar": "^3.2.4"
+      }
+    },
     "node_modules/electron-squirrel-startup": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/electron-squirrel-startup/-/electron-squirrel-startup-1.0.1.tgz",
@@ -10894,7 +10916,6 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
       "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
-      "dev": true,
       "license": "ISC"
     },
     "node_modules/fsevents": {
@@ -11243,7 +11264,6 @@
       "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
       "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
       "deprecated": "Glob versions prior to v9 are no longer supported",
-      "dev": true,
       "license": "ISC",
       "dependencies": {
         "fs.realpath": "^1.0.0",
@@ -11842,7 +11862,6 @@
       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
       "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
       "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.",
-      "dev": true,
       "license": "ISC",
       "dependencies": {
         "once": "^1.3.0",
@@ -14619,7 +14638,6 @@
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
       "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
-      "dev": true,
       "license": "ISC",
       "dependencies": {
         "brace-expansion": "^1.1.7"
@@ -15597,7 +15615,6 @@
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
       "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
-      "dev": true,
       "license": "MIT",
       "engines": {
         "node": ">=0.10.0"
@@ -15715,6 +15732,53 @@
         "node": ">=6"
       }
     },
+    "node_modules/playwright": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.52.0.tgz",
+      "integrity": "sha512-JAwMNMBlxJ2oD1kce4KPtMkDeKGHQstdpFPcPH3maElAXon/QZeTvtsfXmTMRyO9TslfoYOXkSsvao2nE1ilTw==",
+      "devOptional": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.52.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.52.0.tgz",
+      "integrity": "sha512-l2osTgLXSMeuLZOML9qYODUQoPPnUsKsb5/P6LJ2e6uPKXUdPK5WYhN4z03G+YNbWmGDY4YENauNu4ZKczreHg==",
+      "devOptional": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/playwright/node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
     "node_modules/plist": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/plist/-/plist-3.1.0.tgz",

package.json

@@ -33,7 +33,9 @@
     "test:watch": "vitest",
     "test:ui": "vitest --ui",
     "extract-codebase": "ts-node scripts/extract-codebase.ts",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "pre:e2e": "E2E_TEST_BUILD=true npm run package",
+    "e2e": "playwright test"
   },
   "keywords": [],
   "author": {
@@ -52,6 +54,7 @@
     "@electron-forge/plugin-vite": "^7.8.0",
     "@electron-forge/publisher-github": "^7.8.0",
     "@electron/fuses": "^1.8.0",
+    "@playwright/test": "^1.52.0",
     "@testing-library/react": "^16.3.0",
     "@types/better-sqlite3": "^7.6.13",
     "@types/kill-port": "^2.0.3",
@@ -111,6 +114,7 @@
     "dotenv": "^16.4.7",
     "drizzle-orm": "^0.41.0",
     "electron-log": "^5.3.3",
+    "electron-playwright-helpers": "^1.7.1",
     "electron-squirrel-startup": "^1.0.1",
     "esbuild-register": "^3.6.0",
     "fix-path": "^4.0.0",

playwright.config.ts

@@ -0,0 +1,23 @@
+import { PlaywrightTestConfig } from "@playwright/test";
+
+const config: PlaywrightTestConfig = {
+  testDir: "./e2e-tests",
+  maxFailures: 1,
+
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* See https://playwright.dev/docs/trace-viewer */
+    trace: "retain-on-failure",
+
+    // These options do NOT work for electron playwright.
+    // Instead, you need to do a workaround.
+    // See https://github.com/microsoft/playwright/issues/8208
+    //
+    // screenshot: "on",
+    // video: "retain-on-failure",
+  },
+};
+
+export default config;

src/main.ts

@@ -65,6 +65,10 @@ export async function onFirstRunMaybe() {
  * applications folder.
  */
 async function promptMoveToApplicationsFolder(): Promise<void> {
+  // Why not in e2e tests?
+  // There's no way to stub this dialog in time, so we just skip it
+  // in e2e testing mode.
+  if (process.env.E2E_TEST_BUILD) return;
   if (process.platform !== "darwin") return;
   if (app.isInApplicationsFolder()) return;
   logger.log("Prompting user to move to applications folder");