From b8f7490288109ee6cacd67fb1f1744862c45740d Mon Sep 17 00:00:00 2001
From: Will Chen <willchen90@gmail.com>
Date: Tue, 3 Jun 2025 17:58:59 -0700
Subject: [PATCH] Stabilize E2E test by alphabetically sorting files for
 context (#321)

---
 .github/workflows/ci.yml                      |  2 +-
 e2e-tests/dump_messages.spec.ts               |  7 ++----
 e2e-tests/helpers/test_helper.ts              | 24 +++++++++++++++----
 .../dump_messages.spec.ts_dump-messages-1.txt | 10 ++++----
 src/utils/codebase.ts                         |  9 +++++++
 5 files changed, 36 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 956d7c8..83308bf 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -58,7 +58,7 @@ jobs:
       - name: E2E tests
         # You can add debug logging to make it easier to see what's failing
         # by adding "DEBUG=pw:browser" in front.
-        run: npm run e2e
+        run: DEBUG=pw:browser npm run e2e
       - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         if: failure()
         with:
diff --git a/e2e-tests/dump_messages.spec.ts b/e2e-tests/dump_messages.spec.ts
index c715160..a2dd3ef 100644
--- a/e2e-tests/dump_messages.spec.ts
+++ b/e2e-tests/dump_messages.spec.ts
@@ -1,10 +1,7 @@
-import { testSkipIfWindows } from "./helpers/test_helper";
+import { test } from "./helpers/test_helper";
 
 // This is useful to make sure the messages are being sent correctly.
-//
-// Why skip on Windows? The file ordering is not stable between runs
-// but unclear why.
-testSkipIfWindows("dump messages", async ({ po }) => {
+test("dump messages", async ({ po }) => {
   await po.setUp();
   await po.sendPrompt("[dump]");
   await po.snapshotServerDump();
diff --git a/e2e-tests/helpers/test_helper.ts b/e2e-tests/helpers/test_helper.ts
index 54576d4..ea3d9ce 100644
--- a/e2e-tests/helpers/test_helper.ts
+++ b/e2e-tests/helpers/test_helper.ts
@@ -4,6 +4,7 @@ import { ElectronApplication, _electron as electron } from "playwright";
 import fs from "fs";
 import path from "path";
 import os from "os";
+import { execSync } from "child_process";
 
 const showDebugLogs = process.env.DEBUG_LOGS === "true";
 
@@ -478,7 +479,11 @@ export const test = base.extend<{
       process.env.E2E_TEST_BUILD = "true";
       // This is just a hack to avoid the AI setup screen.
       process.env.OPENAI_API_KEY = "sk-test";
-      const USER_DATA_DIR = `/tmp/dyad-e2e-tests-${Date.now()}`;
+      const baseTmpDir = os.tmpdir();
+      const USER_DATA_DIR = path.join(
+        baseTmpDir,
+        `dyad-e2e-tests-${Date.now()}`,
+      );
       const electronApp = await electron.launch({
         args: [
           appInfo.main,
@@ -486,9 +491,11 @@ export const test = base.extend<{
           `--user-data-dir=${USER_DATA_DIR}`,
         ],
         executablePath: appInfo.executable,
-        recordVideo: {
-          dir: "test-results",
-        },
+        // Strong suspicion this is causing issues on Windows with tests hanging due to error:
+        // ffmpeg failed to write: Error [ERR_STREAM_WRITE_AFTER_END]: write after end
+        // recordVideo: {
+        //   dir: "test-results",
+        // },
       });
       (electronApp as any).$dyadUserDataDir = USER_DATA_DIR;
 
@@ -527,7 +534,14 @@ export const test = base.extend<{
       // because the electron app does NOT ever fully quit due to
       // Windows' strict resource locking (e.g. file locking).
       if (os.platform() === "win32") {
-        electronApp.process().kill();
+        try {
+          execSync("taskkill /f /im dyad.exe");
+        } catch (error) {
+          console.warn(
+            "Failed to kill dyad.exe: (continuing with test cleanup)",
+            error,
+          );
+        }
       } else {
         await electronApp.close();
       }
diff --git a/e2e-tests/snapshots/dump_messages.spec.ts_dump-messages-1.txt b/e2e-tests/snapshots/dump_messages.spec.ts_dump-messages-1.txt
index 84d3f94..f85ca78 100644
--- a/e2e-tests/snapshots/dump_messages.spec.ts_dump-messages-1.txt
+++ b/e2e-tests/snapshots/dump_messages.spec.ts_dump-messages-1.txt
@@ -418,11 +418,6 @@ Available packages and libraries:
 
 </dyad-file>
 
-<dyad-file path="README.md">
-# Welcome to your Dyad app
-
-</dyad-file>
-
 <dyad-file path="eslint.config.js">
 // Contents omitted for brevity
 </dyad-file>
@@ -454,6 +449,11 @@ export default {
 
 </dyad-file>
 
+<dyad-file path="README.md">
+# Welcome to your Dyad app
+
+</dyad-file>
+
 <dyad-file path="src/App.css">
 #root {
   max-width: 1280px;
diff --git a/src/utils/codebase.ts b/src/utils/codebase.ts
index 4d479d0..e5c9dae 100644
--- a/src/utils/codebase.ts
+++ b/src/utils/codebase.ts
@@ -370,6 +370,15 @@ async function sortFilesByModificationTime(files: string[]): Promise<string[]> {
     }),
   );
 
+  if (process.env.E2E_TEST_BUILD) {
+    // Why? For some reason, file ordering is not stable on Windows.
+    // This is a workaround to ensure stable ordering, although
+    // ideally we'd like to sort it by modification time which is
+    // important for cache-ability.
+    return fileStats
+      .sort((a, b) => a.file.localeCompare(b.file))
+      .map((item) => item.file);
+  }
   // Sort by modification time (oldest first)
   return fileStats.sort((a, b) => a.mtime - b.mtime).map((item) => item.file);
 }