Stabilize E2E test by alphabetically sorting files for context (#321)

.github/workflows/ci.yml

@@ -58,7 +58,7 @@ jobs:
       - name: E2E tests
         # You can add debug logging to make it easier to see what's failing
         # by adding "DEBUG=pw:browser" in front.
-        run: npm run e2e
+        run: DEBUG=pw:browser npm run e2e
       - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         if: failure()
         with:

e2e-tests/dump_messages.spec.ts

@@ -1,10 +1,7 @@
-import { testSkipIfWindows } from "./helpers/test_helper";
+import { test } from "./helpers/test_helper";
 
 // This is useful to make sure the messages are being sent correctly.
-//
-// Why skip on Windows? The file ordering is not stable between runs
-// but unclear why.
-testSkipIfWindows("dump messages", async ({ po }) => {
+test("dump messages", async ({ po }) => {
   await po.setUp();
   await po.sendPrompt("[dump]");
   await po.snapshotServerDump();

e2e-tests/helpers/test_helper.ts

@@ -4,6 +4,7 @@ import { ElectronApplication, _electron as electron } from "playwright";
 import fs from "fs";
 import path from "path";
 import os from "os";
+import { execSync } from "child_process";
 
 const showDebugLogs = process.env.DEBUG_LOGS === "true";
 
@@ -478,7 +479,11 @@ export const test = base.extend<{
       process.env.E2E_TEST_BUILD = "true";
       // This is just a hack to avoid the AI setup screen.
       process.env.OPENAI_API_KEY = "sk-test";
-      const USER_DATA_DIR = `/tmp/dyad-e2e-tests-${Date.now()}`;
+      const baseTmpDir = os.tmpdir();
+      const USER_DATA_DIR = path.join(
+        baseTmpDir,
+        `dyad-e2e-tests-${Date.now()}`,
+      );
       const electronApp = await electron.launch({
         args: [
           appInfo.main,
@@ -486,9 +491,11 @@ export const test = base.extend<{
           `--user-data-dir=${USER_DATA_DIR}`,
         ],
         executablePath: appInfo.executable,
-        recordVideo: {
-          dir: "test-results",
-        },
+        // Strong suspicion this is causing issues on Windows with tests hanging due to error:
+        // ffmpeg failed to write: Error [ERR_STREAM_WRITE_AFTER_END]: write after end
+        // recordVideo: {
+        //   dir: "test-results",
+        // },
       });
       (electronApp as any).$dyadUserDataDir = USER_DATA_DIR;
 
@@ -527,7 +534,14 @@ export const test = base.extend<{
       // because the electron app does NOT ever fully quit due to
       // Windows' strict resource locking (e.g. file locking).
       if (os.platform() === "win32") {
-        electronApp.process().kill();
+        try {
+          execSync("taskkill /f /im dyad.exe");
+        } catch (error) {
+          console.warn(
+            "Failed to kill dyad.exe: (continuing with test cleanup)",
+            error,
+          );
+        }
       } else {
         await electronApp.close();
       }

e2e-tests/snapshots/dump_messages.spec.ts_dump-messages-1.txt

@@ -418,11 +418,6 @@ Available packages and libraries:
 
 </dyad-file>
 
-<dyad-file path="README.md">
-# Welcome to your Dyad app
-
-</dyad-file>
-
 <dyad-file path="eslint.config.js">
 // Contents omitted for brevity
 </dyad-file>
@@ -454,6 +449,11 @@ export default {
 
 </dyad-file>
 
+<dyad-file path="README.md">
+# Welcome to your Dyad app
+
+</dyad-file>
+
 <dyad-file path="src/App.css">
 #root {
   max-width: 1280px;

src/utils/codebase.ts

@@ -370,6 +370,15 @@ async function sortFilesByModificationTime(files: string[]): Promise<string[]> {
     }),
   );
 
+  if (process.env.E2E_TEST_BUILD) {
+    // Why? For some reason, file ordering is not stable on Windows.
+    // This is a workaround to ensure stable ordering, although
+    // ideally we'd like to sort it by modification time which is
+    // important for cache-ability.
+    return fileStats
+      .sort((a, b) => a.file.localeCompare(b.file))
+      .map((item) => item.file);
+  }
   // Sort by modification time (oldest first)
   return fileStats.sort((a, b) => a.mtime - b.mtime).map((item) => item.file);
 }