Files
moreminimore-vibe/e2e-tests/helpers/test_helper.ts
2025-06-04 21:37:05 -07:00

692 lines
21 KiB
TypeScript

import { test as base, Page, expect } from "@playwright/test";
import { findLatestBuild, parseElectronApp } from "electron-playwright-helpers";
import { ElectronApplication, _electron as electron } from "playwright";
import fs from "fs";
import path from "path";
import os from "os";
import { execSync } from "child_process";
const showDebugLogs = process.env.DEBUG_LOGS === "true";
export const Timeout = {
// Why make this a constant? In some platforms, perhaps locally,
// we may want to shorten this.
LONG: os.platform() === "win32" ? 60_000 : 30_000,
MEDIUM: os.platform() === "win32" ? 30_000 : 15_000,
};
export class PageObject {
private userDataDir: string;
constructor(
public electronApp: ElectronApplication,
public page: Page,
{ userDataDir }: { userDataDir: string },
) {
this.userDataDir = userDataDir;
}
async setUp({
autoApprove = false,
nativeGit = false,
}: { autoApprove?: boolean; nativeGit?: boolean } = {}) {
await this.goToSettingsTab();
if (autoApprove) {
await this.toggleAutoApprove();
}
if (nativeGit) {
await this.toggleNativeGit();
}
await this.setUpTestProvider();
await this.setUpTestModel();
await this.goToAppsTab();
await this.selectTestModel();
}
async setUpDyadPro({ autoApprove = false }: { autoApprove?: boolean } = {}) {
await this.goToSettingsTab();
if (autoApprove) {
await this.toggleAutoApprove();
}
await this.setUpDyadProvider();
await this.goToAppsTab();
}
async setUpDyadProvider() {
// await page.getByRole('link', { name: 'Settings' }).click();
await this.page
.locator("div")
.filter({ hasText: /^DyadNeeds Setup$/ })
.nth(1)
.click();
await this.page.getByRole("textbox", { name: "Set Dyad API Key" }).click();
await this.page
.getByRole("textbox", { name: "Set Dyad API Key" })
.fill("testdyadkey");
await this.page.getByRole("button", { name: "Save Key" }).click();
// await page.getByRole('link', { name: 'Apps' }).click();
// await page.getByTestId('home-chat-input-container').getByRole('button', { name: 'Pro' }).click();
// await page.getByRole('switch', { name: 'Turbo Edits' }).click();
// await page.getByRole('switch', { name: 'Turbo Edits' }).click();
// await page.locator('div').filter({ hasText: /^Import App$/ }).click();
// await page.getByRole('button', { name: 'Select Folder' }).press('Escape');
}
async snapshotAppFiles({ name }: { name?: string } = {}) {
const appPath = await this.getCurrentAppPath();
if (!appPath || !fs.existsSync(appPath)) {
throw new Error(`App path does not exist: ${appPath}`);
}
await expect(() => {
const filesData = generateAppFilesSnapshotData(appPath, appPath, [
".git",
"node_modules",
// Avoid snapshotting lock files because they are getting generated
// automatically and cause noise, and not super important anyways.
"package-lock.json",
"pnpm-lock.yaml",
]);
// Sort by relative path to ensure deterministic output
filesData.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
const snapshotContent = filesData
.map((file) => `=== ${file.relativePath} ===\n${file.content}`)
.join("\n\n");
if (name) {
expect(snapshotContent).toMatchSnapshot(name);
} else {
expect(snapshotContent).toMatchSnapshot();
}
}).toPass();
}
async snapshotMessages({
replaceDumpPath = false,
}: { replaceDumpPath?: boolean } = {}) {
if (replaceDumpPath) {
// Update page so that "[[dyad-dump-path=*]]" is replaced with a placeholder path
// which is stable across runs.
await this.page.evaluate(() => {
const messagesList = document.querySelector(
"[data-testid=messages-list]",
);
if (!messagesList) {
throw new Error("Messages list not found");
}
messagesList.innerHTML = messagesList.innerHTML.replace(
/\[\[dyad-dump-path=([^\]]+)\]\]/,
"[[dyad-dump-path=*]]",
);
});
}
await expect(this.page.getByTestId("messages-list")).toMatchAriaSnapshot();
}
async approveProposal() {
await this.page.getByTestId("approve-proposal-button").click();
}
async rejectProposal() {
await this.page.getByTestId("reject-proposal-button").click();
}
async clickRestart() {
await this.page.getByRole("button", { name: "Restart" }).click();
}
async clickRebuild() {
await this.clickPreviewMoreOptions();
await this.page.getByText("Rebuild").click();
}
async clickPreviewMoreOptions() {
await this.page.getByTestId("preview-more-options-button").click();
}
async clickPreviewRefresh() {
await this.page.getByTestId("preview-refresh-button").click();
}
async clickPreviewNavigateBack() {
await this.page.getByTestId("preview-navigate-back-button").click();
}
async clickPreviewNavigateForward() {
await this.page.getByTestId("preview-navigate-forward-button").click();
}
async clickPreviewOpenBrowser() {
await this.page.getByTestId("preview-open-browser-button").click();
}
locateLoadingAppPreview() {
return this.page.getByText("Loading app preview...");
}
getPreviewIframeElement() {
return this.page.getByTestId("preview-iframe-element");
}
async snapshotPreview({ name }: { name?: string } = {}) {
const iframe = this.getPreviewIframeElement();
await expect(iframe.contentFrame().locator("body")).toMatchAriaSnapshot({
name,
timeout: Timeout.LONG,
});
}
async snapshotServerDump(
type: "all-messages" | "last-message" | "request" = "all-messages",
{ name = "" }: { name?: string } = {},
) {
// Get the text content of the messages list
const messagesListText = await this.page
.getByTestId("messages-list")
.textContent();
// Find the dump path using regex
const dumpPathMatch = messagesListText?.match(
/.*\[\[dyad-dump-path=([^\]]+)\]\]/,
);
if (!dumpPathMatch) {
throw new Error("No dump path found in messages list");
}
const dumpFilePath = dumpPathMatch[1];
// Read the JSON file
const dumpContent: string = (
fs.readFileSync(dumpFilePath, "utf-8") as any
).replaceAll(/\[\[dyad-dump-path=([^\]]+)\]\]/g, "[[dyad-dump-path=*]]");
// Perform snapshot comparison
const parsedDump = JSON.parse(dumpContent);
if (type === "request") {
expect(dumpContent).toMatchSnapshot(name);
return;
}
expect(
prettifyDump(parsedDump["body"]["messages"], {
onlyLastMessage: type === "last-message",
}),
).toMatchSnapshot(name);
}
async waitForChatCompletion() {
await expect(this.getRetryButton()).toBeVisible({
timeout: Timeout.MEDIUM,
});
}
async clickRetry() {
await this.getRetryButton().click();
}
async clickUndo() {
await this.getUndoButton().click();
}
private getRetryButton() {
return this.page.getByRole("button", { name: "Retry" });
}
private getUndoButton() {
return this.page.getByRole("button", { name: "Undo" });
}
getHomeChatInputContainer() {
return this.page.getByTestId("home-chat-input-container");
}
getChatInputContainer() {
return this.page.getByTestId("chat-input-container");
}
getChatInput() {
return this.page.getByRole("textbox", { name: "Ask Dyad to build..." });
}
clickNewChat({ index = 0 }: { index?: number } = {}) {
// There is two new chat buttons...
return this.page
.getByRole("button", { name: "New Chat" })
.nth(index)
.click();
}
async sendPrompt(prompt: string) {
await this.getChatInput().click();
await this.getChatInput().fill(prompt);
await this.page.getByRole("button", { name: "Send message" }).click();
await this.waitForChatCompletion();
}
async selectModel({ provider, model }: { provider: string; model: string }) {
await this.page.getByRole("button", { name: "Model: Auto" }).click();
await this.page.getByText(provider).click();
await this.page.getByText(model).click();
}
async selectTestModel() {
await this.page.getByRole("button", { name: "Model: Auto" }).click();
await this.page.getByText("test-provider").click();
await this.page.getByText("test-model").click();
}
async selectTestOllamaModel() {
await this.page.getByRole("button", { name: "Model: Auto" }).click();
await this.page.getByText("Local models").click();
await this.page.getByText("Ollama", { exact: true }).click();
await this.page.getByText("Testollama", { exact: true }).click();
}
async selectTestLMStudioModel() {
await this.page.getByRole("button", { name: "Model: Auto" }).click();
await this.page.getByText("Local models").click();
await this.page.getByText("LM Studio", { exact: true }).click();
// Both of the elements that match "lmstudio-model-1" are the same button, so we just pick the first.
await this.page
.getByText("lmstudio-model-1", { exact: true })
.first()
.click();
}
async setUpTestProvider() {
await this.page.getByText("Add custom providerConnect to").click();
// Fill out provider dialog
await this.page
.getByRole("textbox", { name: "Provider ID" })
.fill("testing");
await this.page.getByRole("textbox", { name: "Display Name" }).click();
await this.page
.getByRole("textbox", { name: "Display Name" })
.fill("test-provider");
await this.page.getByText("API Base URLThe base URL for").click();
await this.page
.getByRole("textbox", { name: "API Base URL" })
.fill("http://localhost:3500/v1");
await this.page.getByRole("button", { name: "Add Provider" }).click();
}
async setUpTestModel() {
await this.page
.getByRole("heading", { name: "test-provider Needs Setup" })
.click();
await this.page.getByRole("button", { name: "Add Custom Model" }).click();
await this.page
.getByRole("textbox", { name: "Model ID*" })
.fill("test-model");
await this.page.getByRole("textbox", { name: "Model ID*" }).press("Tab");
await this.page.getByRole("textbox", { name: "Name*" }).fill("test-model");
await this.page.getByRole("button", { name: "Add Model" }).click();
}
async goToSettingsTab() {
await this.page.getByRole("link", { name: "Settings" }).click();
}
getTitleBarAppNameButton() {
return this.page.getByTestId("title-bar-app-name-button");
}
getAppListItem({ appName }: { appName: string }) {
return this.page.getByTestId(`app-list-item-${appName}`);
}
async isCurrentAppNameNone() {
await expect(async () => {
await expect(this.getTitleBarAppNameButton()).toContainText(
"no app selected",
);
}).toPass();
}
async getCurrentAppName() {
// Make sure to wait for the app to be set to avoid a race condition.
await expect(async () => {
await expect(this.getTitleBarAppNameButton()).not.toContainText(
"no app selected",
);
}).toPass();
return (await this.getTitleBarAppNameButton().textContent())?.replace(
"App: ",
"",
);
}
async getCurrentAppPath() {
const currentAppName = await this.getCurrentAppName();
if (!currentAppName) {
throw new Error("No current app name found");
}
return this.getAppPath({ appName: currentAppName });
}
getAppPath({ appName }: { appName: string }) {
return path.join(this.userDataDir, "dyad-apps", appName);
}
async clickAppListItem({ appName }: { appName: string }) {
await this.page.getByTestId(`app-list-item-${appName}`).click();
}
async clickAppDetailsRenameAppButton() {
await this.page.getByTestId("app-details-rename-app-button").click();
}
async clickAppDetailsMoreOptions() {
await this.page.getByTestId("app-details-more-options-button").click();
}
////////////////////////////////
// Settings related
////////////////////////////////
async toggleAutoApprove() {
await this.page.getByRole("switch", { name: "Auto-approve" }).click();
}
async toggleNativeGit() {
await this.page.getByRole("switch", { name: "Enable Native Git" }).click();
}
async snapshotSettings() {
const settings = path.join(this.userDataDir, "user-settings.json");
const settingsContent = fs.readFileSync(settings, "utf-8");
// Sanitize the "telemetryUserId" since it's a UUID
const sanitizedSettingsContent = settingsContent.replace(
/"telemetryUserId": "[^"]*"/g,
'"telemetryUserId": "[UUID]"',
);
expect(sanitizedSettingsContent).toMatchSnapshot();
}
async clickTelemetryAccept() {
await this.page.getByTestId("telemetry-accept-button").click();
}
async clickTelemetryReject() {
await this.page.getByTestId("telemetry-reject-button").click();
}
async clickTelemetryLater() {
await this.page.getByTestId("telemetry-later-button").click();
}
async goToAppsTab() {
await this.page.getByRole("link", { name: "Apps" }).click();
}
async goToHubTab() {
await this.page.getByRole("link", { name: "Hub" }).click();
}
async selectTemplate(templateName: string) {
await this.page.getByRole("img", { name: templateName }).click();
}
////////////////////////////////
// Toast assertions
////////////////////////////////
async expectNoToast() {
await expect(this.page.locator("[data-sonner-toast]")).toHaveCount(0);
}
async waitForToast(
type?: "success" | "error" | "warning" | "info",
timeout = 5000,
) {
const selector = type
? `[data-sonner-toast][data-type="${type}"]`
: "[data-sonner-toast]";
await this.page.waitForSelector(selector, { timeout });
}
async waitForToastWithText(text: string, timeout = 5000) {
await this.page.waitForSelector(`[data-sonner-toast]:has-text("${text}")`, {
timeout,
});
}
async assertToastVisible(type?: "success" | "error" | "warning" | "info") {
const selector = type
? `[data-sonner-toast][data-type="${type}"]`
: "[data-sonner-toast]";
await expect(this.page.locator(selector)).toBeVisible();
}
async assertToastWithText(text: string) {
await expect(
this.page.locator(`[data-sonner-toast]:has-text("${text}")`),
).toBeVisible();
}
async dismissAllToasts() {
// Click all close buttons if they exist
const closeButtons = this.page.locator(
"[data-sonner-toast] button[data-close-button]",
);
const count = await closeButtons.count();
for (let i = 0; i < count; i++) {
await closeButtons.nth(i).click();
}
}
async sleep(ms: number) {
await new Promise((resolve) => setTimeout(resolve, ms));
}
}
// From https://github.com/microsoft/playwright/issues/8208#issuecomment-1435475930
//
// Note how we mark the fixture as { auto: true }.
// This way it is always instantiated, even if the test does not use it explicitly.
export const test = base.extend<{
attachScreenshotsToReport: void;
electronApp: ElectronApplication;
po: PageObject;
}>({
po: [
async ({ electronApp }, use) => {
const page = await electronApp.firstWindow();
const po = new PageObject(electronApp, page, {
userDataDir: (electronApp as any).$dyadUserDataDir,
});
await use(po);
},
{ auto: true },
],
attachScreenshotsToReport: [
async ({ electronApp }, use, testInfo) => {
await use();
// After the test we can check whether the test passed or failed.
if (testInfo.status !== testInfo.expectedStatus) {
const page = await electronApp.firstWindow();
try {
const screenshot = await page.screenshot({ timeout: 5_000 });
await testInfo.attach("screenshot", {
body: screenshot,
contentType: "image/png",
});
} catch (error) {
console.error("Error taking screenshot on failure", error);
}
}
},
{ auto: true },
],
electronApp: [
async ({}, use) => {
// find the latest build in the out directory
const latestBuild = findLatestBuild();
// parse the directory and find paths and other info
const appInfo = parseElectronApp(latestBuild);
process.env.OLLAMA_HOST = "http://localhost:3500/ollama";
process.env.LM_STUDIO_BASE_URL_FOR_TESTING =
"http://localhost:3500/lmstudio";
process.env.DYAD_LOCAL_ENGINE = "http://localhost:3500/engine/v1";
process.env.DYAD_GATEWAY_URL = "http://localhost:3500/gateway/v1";
process.env.E2E_TEST_BUILD = "true";
// This is just a hack to avoid the AI setup screen.
process.env.OPENAI_API_KEY = "sk-test";
const baseTmpDir = os.tmpdir();
const USER_DATA_DIR = path.join(
baseTmpDir,
`dyad-e2e-tests-${Date.now()}`,
);
const electronApp = await electron.launch({
args: [
appInfo.main,
"--enable-logging",
`--user-data-dir=${USER_DATA_DIR}`,
],
executablePath: appInfo.executable,
// Strong suspicion this is causing issues on Windows with tests hanging due to error:
// ffmpeg failed to write: Error [ERR_STREAM_WRITE_AFTER_END]: write after end
// recordVideo: {
// dir: "test-results",
// },
});
(electronApp as any).$dyadUserDataDir = USER_DATA_DIR;
console.log("electronApp launched!");
if (showDebugLogs) {
// Listen to main process output immediately
electronApp.process().stdout?.on("data", (data) => {
console.log(`MAIN_PROCESS_STDOUT: ${data.toString()}`);
});
electronApp.process().stderr?.on("data", (data) => {
console.error(`MAIN_PROCESS_STDERR: ${data.toString()}`);
});
}
electronApp.on("close", () => {
console.log(`Electron app closed listener:`);
});
electronApp.on("window", async (page) => {
const filename = page.url()?.split("/").pop();
console.log(`Window opened: ${filename}`);
// capture errors
page.on("pageerror", (error) => {
console.error(error);
});
// capture console messages
page.on("console", (msg) => {
console.log(msg.text());
});
});
await use(electronApp);
// Why are we doing a force kill on Windows?
//
// Otherwise, Playwright will just hang on the test cleanup
// because the electron app does NOT ever fully quit due to
// Windows' strict resource locking (e.g. file locking).
if (os.platform() === "win32") {
try {
console.log("[cleanup:start] Killing dyad.exe");
console.time("taskkill");
execSync("taskkill /f /t /im dyad.exe");
console.timeEnd("taskkill");
console.log("[cleanup:end] Killed dyad.exe");
} catch (error) {
console.warn(
"Failed to kill dyad.exe: (continuing with test cleanup)",
error,
);
}
} else {
await electronApp.close();
}
},
{ auto: true },
],
});
// Wrapper that skips tests on Windows platform
export const testSkipIfWindows = os.platform() === "win32" ? test.skip : test;
function prettifyDump(
allMessages: {
role: string;
content: string | Array<{}>;
}[],
{ onlyLastMessage = false }: { onlyLastMessage?: boolean } = {},
) {
const messages = onlyLastMessage ? allMessages.slice(-1) : allMessages;
return messages
.map((message) => {
const content = Array.isArray(message.content)
? JSON.stringify(message.content)
: message.content
// Normalize line endings to always use \n
.replace(/\r\n/g, "\n")
// We remove package.json because it's flaky.
// Depending on whether pnpm install is run, it will be modified,
// and the contents and timestamp (thus affecting order) will be affected.
.replace(
/\n<dyad-file path="package\.json">[\s\S]*?<\/dyad-file>\n/g,
"",
);
return `===\nrole: ${message.role}\nmessage: ${content}`;
})
.join("\n\n");
}
interface FileSnapshotData {
relativePath: string;
content: string;
}
function generateAppFilesSnapshotData(
currentPath: string,
basePath: string,
ignorePatterns: string[],
): FileSnapshotData[] {
const entries = fs.readdirSync(currentPath, { withFileTypes: true });
let files: FileSnapshotData[] = [];
// Sort entries for deterministic order
entries.sort((a, b) => a.name.localeCompare(b.name));
for (const entry of entries) {
const entryPath = path.join(currentPath, entry.name);
if (ignorePatterns.includes(entry.name)) {
continue;
}
if (entry.isDirectory()) {
files = files.concat(
generateAppFilesSnapshotData(entryPath, basePath, ignorePatterns),
);
} else if (entry.isFile()) {
const relativePath = path.relative(basePath, entryPath);
try {
const content = fs.readFileSync(entryPath, "utf-8");
files.push({ relativePath, content });
} catch (error) {
// Could be a binary file or permission issue, log and add a placeholder
const e = error as Error;
console.warn(`Could not read file ${entryPath}: ${e.message}`);
files.push({
relativePath,
content: `[Error reading file: ${e.message}]`,
});
}
}
}
return files;
}