diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90dcf5e..a3a9254 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,8 @@ jobs: run: npx playwright install chromium --with-deps - name: Build run: npm run pre:e2e + - name: Prep test server + run: cd testing/fake-llm-server && npm install && npm run build && cd - - name: E2E tests # Add debug logging to make it easier to see what's failing run: DEBUG=pw:browser npm run e2e diff --git a/.gitignore b/.gitignore index 8882f3e..1e653f7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ yarn-debug.log* yarn-error.log* lerna-debug.log* +# dist +dist/ + # playwright playwright-report/ test-results/ diff --git a/.oxlintrc.json b/.oxlintrc.json index 0325337..6f65cf7 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -1,5 +1,6 @@ { "rules": { - "eslint/no-unused-vars": "error" + "eslint/no-unused-vars": "error", + "eslint/no-empty-pattern": "off" } } diff --git a/.prettierignore b/.prettierignore index 785f0fb..0352b32 100644 --- a/.prettierignore +++ b/.prettierignore @@ -3,4 +3,5 @@ build coverage # generated files drizzle/ -**/pnpm-lock.yaml \ No newline at end of file +**/pnpm-lock.yaml +**/snapshots/** \ No newline at end of file diff --git a/e2e-tests/helpers/codegen.js b/e2e-tests/helpers/codegen.js new file mode 100644 index 0000000..18df6f7 --- /dev/null +++ b/e2e-tests/helpers/codegen.js @@ -0,0 +1,24 @@ +/* + * From: https://github.com/microsoft/playwright/issues/5181#issuecomment-2769098576 + * + * Usage: + * cd e2e-tests/helpers && node codegen.js + */ + +const { _electron: electron } = require("playwright"); + +(async () => { + const browser = await electron.launch({ + args: [ + "../../out/dyad-darwin-arm64/dyad.app/Contents/Resources/app.asar/.vite/build/main.js", + "--enable-logging", + "--user-data-dir=/tmp/dyad-e2e-tests", + ], + executablePath: "../../out/dyad-darwin-arm64/dyad.app/Contents/MacOS/dyad", + }); + const context = await browser.context(); + await context.route("**/*", (route) => route.continue()); + + await require("node:timers/promises").setTimeout(3000); // wait for the window to load + await browser.windows()[0].pause(); // .pause() opens the Playwright-Inspector for manual recording +})(); diff --git a/e2e-tests/helpers/test_helper.ts b/e2e-tests/helpers/test_helper.ts new file mode 100644 index 0000000..a3618b9 --- /dev/null +++ b/e2e-tests/helpers/test_helper.ts @@ -0,0 +1,78 @@ +import { test as base } from "@playwright/test"; +import { findLatestBuild, parseElectronApp } from "electron-playwright-helpers"; +import { ElectronApplication, _electron as electron } from "playwright"; + +const showDebugLogs = process.env.DEBUG_LOGS === "true"; + +// From https://github.com/microsoft/playwright/issues/8208#issuecomment-1435475930 +// +// Note how we mark the fixture as { auto: true }. +// This way it is always instantiated, even if the test does not use it explicitly. +export const test = base.extend<{ + attachScreenshotsToReport: void; + electronApp: ElectronApplication; +}>({ + attachScreenshotsToReport: [ + async ({ page }, use, testInfo) => { + await use(); + + // After the test we can check whether the test passed or failed. + if (testInfo.status !== testInfo.expectedStatus) { + const screenshot = await page.screenshot(); + await testInfo.attach("screenshot", { + body: screenshot, + contentType: "image/png", + }); + } + }, + { auto: true }, + ], + electronApp: async ({}, use) => { + // find the latest build in the out directory + const latestBuild = findLatestBuild(); + // parse the directory and find paths and other info + const appInfo = parseElectronApp(latestBuild); + process.env.E2E_TEST_BUILD = "true"; + // This is just a hack to avoid the AI setup screen. + process.env.OPENAI_API_KEY = "sk-test"; + const electronApp = await electron.launch({ + args: [ + appInfo.main, + "--enable-logging", + `--user-data-dir=/tmp/dyad-e2e-tests-${Date.now()}`, + ], + executablePath: appInfo.executable, + }); + + console.log("electronApp launched!"); + if (showDebugLogs) { + // Listen to main process output immediately + electronApp.process().stdout?.on("data", (data) => { + console.log(`MAIN_PROCESS_STDOUT: ${data.toString()}`); + }); + electronApp.process().stderr?.on("data", (data) => { + console.error(`MAIN_PROCESS_STDERR: ${data.toString()}`); + }); + } + electronApp.on("close", () => { + console.log(`Electron app closed listener:`); + }); + + electronApp.on("window", async (page) => { + const filename = page.url()?.split("/").pop(); + console.log(`Window opened: ${filename}`); + + // capture errors + page.on("pageerror", (error) => { + console.error(error); + }); + // capture console messages + page.on("console", (msg) => { + console.log(msg.text()); + }); + }); + + await use(electronApp); + await electronApp.close(); + }, +}); diff --git a/e2e-tests/main.spec.ts b/e2e-tests/main.spec.ts index 4df82eb..224abc2 100644 --- a/e2e-tests/main.spec.ts +++ b/e2e-tests/main.spec.ts @@ -1,91 +1,52 @@ -/** - * Example Playwright script for Electron - * showing/testing various API features - * in both renderer and main processes - */ +import { expect } from "@playwright/test"; +import { test } from "./helpers/test_helper"; -import { expect, test as base } from "@playwright/test"; -import { findLatestBuild, parseElectronApp } from "electron-playwright-helpers"; -import { ElectronApplication, Page, _electron as electron } from "playwright"; - -let electronApp: ElectronApplication; - -// From https://github.com/microsoft/playwright/issues/8208#issuecomment-1435475930 -// -// Note how we mark the fixture as { auto: true }. -// This way it is always instantiated, even if the test does not use it explicitly. -export const test = base.extend<{ attachScreenshotsToReport: void }>({ - attachScreenshotsToReport: [ - async ({}, use, testInfo) => { - await use(); - - // After the test we can check whether the test passed or failed. - if (testInfo.status !== testInfo.expectedStatus) { - const screenshot = await page.screenshot(); - await testInfo.attach("screenshot", { - body: screenshot, - contentType: "image/png", - }); - } - }, - { auto: true }, - ], -}); - -test.beforeAll(async () => { - // find the latest build in the out directory - const latestBuild = findLatestBuild(); - // parse the directory and find paths and other info - const appInfo = parseElectronApp(latestBuild); - process.env.E2E_TEST_BUILD = "true"; - // This is just a hack to avoid the AI setup screen. - process.env.OPENAI_API_KEY = "sk-test"; - electronApp = await electron.launch({ - args: [ - appInfo.main, - "--enable-logging", - "--user-data-dir=/tmp/dyad-e2e-tests", - ], - executablePath: appInfo.executable, - }); - - console.log("electronApp launched!"); - - // Listen to main process output immediately - electronApp.process().stdout?.on("data", (data) => { - console.log(`MAIN_PROCESS_STDOUT: ${data.toString()}`); - }); - electronApp.process().stderr?.on("data", (data) => { - console.error(`MAIN_PROCESS_STDERR: ${data.toString()}`); - }); - electronApp.on("close", () => { - console.log(`Electron app closed listener:`); - }); - - electronApp.on("window", async (page) => { - const filename = page.url()?.split("/").pop(); - console.log(`Window opened: ${filename}`); - - // capture errors - page.on("pageerror", (error) => { - console.error(error); - }); - // capture console messages - page.on("console", (msg) => { - console.log(msg.text()); - }); - }); -}); - -test.afterAll(async () => { - await electronApp.close(); -}); - -let page: Page; - -test("renders the first page", async () => { - page = await electronApp.firstWindow(); +test("renders the first page", async ({ electronApp }) => { + const page = await electronApp.firstWindow(); await page.waitForSelector("h1"); const text = await page.$eval("h1", (el) => el.textContent); expect(text).toBe("Build your dream app"); }); + +test("simple message to custom test model", async ({ electronApp }) => { + const page = await electronApp.firstWindow(); + await page.getByRole("link", { name: "Settings" }).click(); + await page.getByText("Add custom providerConnect to").click(); + + // Fill out provider dialog + await page.getByRole("textbox", { name: "Provider ID" }).fill("testing"); + await page.getByRole("textbox", { name: "Display Name" }).click(); + await page + .getByRole("textbox", { name: "Display Name" }) + .fill("test-provider"); + await page.getByText("API Base URLThe base URL for").click(); + await page + .getByRole("textbox", { name: "API Base URL" }) + .fill("http://localhost:3500/v1"); + await page.getByRole("button", { name: "Add Provider" }).click(); + + // Create custom model + await page + .getByRole("heading", { name: "test-provider Needs Setup" }) + .click(); + await page.getByRole("button", { name: "Add Custom Model" }).click(); + await page.getByRole("textbox", { name: "Model ID*" }).fill("test-model"); + await page.getByRole("textbox", { name: "Model ID*" }).press("Tab"); + await page.getByRole("textbox", { name: "Name*" }).fill("test-model"); + await page.getByRole("button", { name: "Add Model" }).click(); + + // Go to apps page and select custom model + await page.getByRole("link", { name: "Apps" }).click(); + await page.getByRole("button", { name: "Model: Auto" }).click(); + await page.getByText("test-provider").click(); + await page.getByText("test-model").click(); + + // Enter prompt and send + await page.getByRole("textbox", { name: "Ask Dyad to build..." }).click(); + await page.getByRole("textbox", { name: "Ask Dyad to build..." }).fill("hi"); + await page.getByRole("button", { name: "Start new chat" }).click(); + + // Make sure it's done + await expect(page.getByRole("button", { name: "Retry" })).not.toBeVisible(); + await expect(page.getByTestId("messages-list")).toMatchAriaSnapshot(); +}); diff --git a/e2e-tests/snapshots/main.spec.ts_simple-message-to-custom-test-model-1.aria.yml b/e2e-tests/snapshots/main.spec.ts_simple-message-to-custom-test-model-1.aria.yml new file mode 100644 index 0000000..777e2f1 --- /dev/null +++ b/e2e-tests/snapshots/main.spec.ts_simple-message-to-custom-test-model-1.aria.yml @@ -0,0 +1,16 @@ +- paragraph: hi +- 'button "Thinking `＜dyad-write>`: I''ll think about the problem and write a bug report. ＜dyad-write> ＜dyad-write path=\"file1.txt\"> Fake dyad write ＜/dyad-write>"': + - img + - img + - paragraph: + - code: "`＜dyad-write>`" + - text: ": I'll think about the problem and write a bug report." + - paragraph: ＜dyad-write> + - paragraph: ＜dyad-write path="file1.txt"> Fake dyad write ＜/dyad-write> +- img +- text: file1.txt +- img +- text: file1.txt +- paragraph: More EOM +- button "Retry": + - img \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 0455cf9..ecc606f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "dyad", - "version": "0.6.0", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dyad", - "version": "0.6.0", + "version": "0.7.0", "license": "MIT", "dependencies": { "@ai-sdk/anthropic": "^1.2.8", diff --git a/playwright.config.ts b/playwright.config.ts index cb07260..2bbf85f 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -3,6 +3,12 @@ import { PlaywrightTestConfig } from "@playwright/test"; const config: PlaywrightTestConfig = { testDir: "./e2e-tests", maxFailures: 1, + timeout: process.env.CI ? 30_000 : 10_000, + // Use a custom snapshot path template because Playwright's default + // is platform-specific which isn't necessary for Dyad e2e tests + // which should be platform agnostic (we don't do screenshots; only textual diffs). + snapshotPathTemplate: + "{testDir}/{testFileDir}/snapshots/{testFileName}_{arg}{ext}", /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: "html", @@ -18,6 +24,11 @@ const config: PlaywrightTestConfig = { // screenshot: "on", // video: "retain-on-failure", }, + + webServer: { + command: `cd testing/fake-llm-server && npm start`, + url: "http://localhost:3500/health", + }, }; export default config; diff --git a/src/components/chat/MessagesList.tsx b/src/components/chat/MessagesList.tsx index 6f5c84b..994b22f 100644 --- a/src/components/chat/MessagesList.tsx +++ b/src/components/chat/MessagesList.tsx @@ -34,7 +34,11 @@ export const MessagesList = forwardRef( const selectedChatId = useAtomValue(selectedChatIdAtom); return ( -
+
{messages.length > 0 ? ( messages.map((message, index) => ( { - const { stream = false } = req.body; - // Non-streaming response - if (!stream) { - return res.json({ - id: `chatcmpl-${Date.now()}`, - object: "chat.completion", - created: Math.floor(Date.now() / 1000), - model: "fake-model", - choices: [ - { - index: 0, - message: { - role: "assistant", - content: "hello world", - }, - finish_reason: "stop", - }, - ], - }); - } - // Streaming response - res.setHeader("Content-Type", "text/event-stream"); - res.setHeader("Cache-Control", "no-cache"); - res.setHeader("Connection", "keep-alive"); - // Split the "hello world" message into characters to simulate streaming - const message = "hello world"; - const messageChars = message.split(""); - // Stream each character with a delay - let index = 0; - // Send role first - res.write(createStreamChunk("", "assistant")); - const interval = setInterval(() => { - if (index < messageChars.length) { - res.write(createStreamChunk(messageChars[index])); - index++; - } else { - // Send the final chunk - res.write(createStreamChunk("", "assistant", true)); - clearInterval(interval); - res.end(); - } - }, 100); -}); -// Start the server -const server = (0, http_1.createServer)(app); -server.listen(PORT, () => { - console.log(`Fake LLM server running on http://localhost:${PORT}`); -}); -// Handle SIGINT (Ctrl+C) -process.on("SIGINT", () => { - console.log("Shutting down fake LLM server"); - server.close(() => { - console.log("Server closed"); - process.exit(0); - }); -}); diff --git a/testing/fake-llm-server/index.ts b/testing/fake-llm-server/index.ts index efa3b2a..82ccf7b 100644 --- a/testing/fake-llm-server/index.ts +++ b/testing/fake-llm-server/index.ts @@ -50,6 +50,10 @@ const CANNED_MESSAGE = ` More EOM`; +app.get("/health", (req, res) => { + res.send("OK"); +}); + // Handle POST requests to /v1/chat/completions app.post("/v1/chat/completions", (req, res) => { const { stream = false, messages = [] } = req.body;