295 lines
9.0 KiB
TypeScript
295 lines
9.0 KiB
TypeScript
import { test as base, Page, expect } from "@playwright/test";
|
|
import { findLatestBuild, parseElectronApp } from "electron-playwright-helpers";
|
|
import { ElectronApplication, _electron as electron } from "playwright";
|
|
import fs from "fs";
|
|
|
|
const showDebugLogs = process.env.DEBUG_LOGS === "true";
|
|
|
|
class PageObject {
|
|
constructor(private page: Page) {}
|
|
|
|
async setUp() {
|
|
await this.goToSettingsTab();
|
|
await this.setUpTestProvider();
|
|
await this.setUpTestModel();
|
|
await this.goToAppsTab();
|
|
await this.selectTestModel();
|
|
}
|
|
|
|
async snapshotMessages() {
|
|
await expect(this.page.getByTestId("messages-list")).toMatchAriaSnapshot();
|
|
}
|
|
|
|
async snapshotServerDump() {
|
|
// Get the text content of the messages list
|
|
const messagesListText = await this.page
|
|
.getByTestId("messages-list")
|
|
.textContent();
|
|
|
|
// Find the dump path using regex
|
|
const dumpPathMatch = messagesListText?.match(
|
|
/\[\[dyad-dump-path=([^\]]+)\]\]/,
|
|
);
|
|
|
|
if (!dumpPathMatch) {
|
|
throw new Error("No dump path found in messages list");
|
|
}
|
|
|
|
const dumpFilePath = dumpPathMatch[1];
|
|
|
|
// Read the JSON file
|
|
const dumpContent = fs.readFileSync(dumpFilePath, "utf-8");
|
|
|
|
// Perform snapshot comparison
|
|
expect(prettifyDump(dumpContent)).toMatchSnapshot("server-dump.txt");
|
|
}
|
|
|
|
async waitForChatCompletion() {
|
|
await expect(this.getRetryButton()).toBeVisible();
|
|
}
|
|
|
|
async clickRetry() {
|
|
await this.getRetryButton().click();
|
|
}
|
|
|
|
private getRetryButton() {
|
|
return this.page.getByRole("button", { name: "Retry" });
|
|
}
|
|
|
|
async sendPrompt(prompt: string) {
|
|
await this.page
|
|
.getByRole("textbox", { name: "Ask Dyad to build..." })
|
|
.click();
|
|
await this.page
|
|
.getByRole("textbox", { name: "Ask Dyad to build..." })
|
|
.fill(prompt);
|
|
await this.page.getByRole("button", { name: "Start new chat" }).click();
|
|
await this.waitForChatCompletion();
|
|
}
|
|
|
|
async selectTestModel() {
|
|
await this.page.getByRole("button", { name: "Model: Auto" }).click();
|
|
await this.page.getByText("test-provider").click();
|
|
await this.page.getByText("test-model").click();
|
|
}
|
|
|
|
async selectTestOllamaModel() {
|
|
await this.page.getByRole("button", { name: "Model: Auto" }).click();
|
|
await this.page.getByText("Local models").click();
|
|
await this.page.getByText("Ollama", { exact: true }).click();
|
|
await this.page.getByText("Testollama", { exact: true }).click();
|
|
}
|
|
|
|
async selectTestLMStudioModel() {
|
|
await this.page.getByRole("button", { name: "Model: Auto" }).click();
|
|
await this.page.getByText("Local models").click();
|
|
await this.page.getByText("LM Studio", { exact: true }).click();
|
|
// Both of the elements that match "lmstudio-model-1" are the same button, so we just pick the first.
|
|
await this.page
|
|
.getByText("lmstudio-model-1", { exact: true })
|
|
.first()
|
|
.click();
|
|
}
|
|
|
|
async setUpTestProvider() {
|
|
await this.page.getByText("Add custom providerConnect to").click();
|
|
// Fill out provider dialog
|
|
await this.page
|
|
.getByRole("textbox", { name: "Provider ID" })
|
|
.fill("testing");
|
|
await this.page.getByRole("textbox", { name: "Display Name" }).click();
|
|
await this.page
|
|
.getByRole("textbox", { name: "Display Name" })
|
|
.fill("test-provider");
|
|
await this.page.getByText("API Base URLThe base URL for").click();
|
|
await this.page
|
|
.getByRole("textbox", { name: "API Base URL" })
|
|
.fill("http://localhost:3500/v1");
|
|
await this.page.getByRole("button", { name: "Add Provider" }).click();
|
|
}
|
|
|
|
async setUpTestModel() {
|
|
await this.page
|
|
.getByRole("heading", { name: "test-provider Needs Setup" })
|
|
.click();
|
|
await this.page.getByRole("button", { name: "Add Custom Model" }).click();
|
|
await this.page
|
|
.getByRole("textbox", { name: "Model ID*" })
|
|
.fill("test-model");
|
|
await this.page.getByRole("textbox", { name: "Model ID*" }).press("Tab");
|
|
await this.page.getByRole("textbox", { name: "Name*" }).fill("test-model");
|
|
await this.page.getByRole("button", { name: "Add Model" }).click();
|
|
}
|
|
|
|
async goToSettingsTab() {
|
|
await this.page.getByRole("link", { name: "Settings" }).click();
|
|
}
|
|
|
|
async goToAppsTab() {
|
|
await this.page.getByRole("link", { name: "Apps" }).click();
|
|
}
|
|
|
|
////////////////////////////////
|
|
// Toast assertions
|
|
////////////////////////////////
|
|
|
|
async expectNoToast() {
|
|
await expect(this.page.locator("[data-sonner-toast]")).toHaveCount(0);
|
|
}
|
|
|
|
async waitForToast(
|
|
type?: "success" | "error" | "warning" | "info",
|
|
timeout = 5000,
|
|
) {
|
|
const selector = type
|
|
? `[data-sonner-toast][data-type="${type}"]`
|
|
: "[data-sonner-toast]";
|
|
|
|
await this.page.waitForSelector(selector, { timeout });
|
|
}
|
|
|
|
async waitForToastWithText(text: string, timeout = 5000) {
|
|
await this.page.waitForSelector(`[data-sonner-toast]:has-text("${text}")`, {
|
|
timeout,
|
|
});
|
|
}
|
|
|
|
async assertToastVisible(type?: "success" | "error" | "warning" | "info") {
|
|
const selector = type
|
|
? `[data-sonner-toast][data-type="${type}"]`
|
|
: "[data-sonner-toast]";
|
|
|
|
await expect(this.page.locator(selector)).toBeVisible();
|
|
}
|
|
|
|
async assertToastWithText(text: string) {
|
|
await expect(
|
|
this.page.locator(`[data-sonner-toast]:has-text("${text}")`),
|
|
).toBeVisible();
|
|
}
|
|
|
|
async dismissAllToasts() {
|
|
// Click all close buttons if they exist
|
|
const closeButtons = this.page.locator(
|
|
"[data-sonner-toast] button[data-close-button]",
|
|
);
|
|
const count = await closeButtons.count();
|
|
for (let i = 0; i < count; i++) {
|
|
await closeButtons.nth(i).click();
|
|
}
|
|
}
|
|
}
|
|
|
|
// From https://github.com/microsoft/playwright/issues/8208#issuecomment-1435475930
|
|
//
|
|
// Note how we mark the fixture as { auto: true }.
|
|
// This way it is always instantiated, even if the test does not use it explicitly.
|
|
export const test = base.extend<{
|
|
attachScreenshotsToReport: void;
|
|
electronApp: ElectronApplication;
|
|
po: PageObject;
|
|
}>({
|
|
po: [
|
|
async ({ electronApp }, use) => {
|
|
const page = await electronApp.firstWindow();
|
|
|
|
const po = new PageObject(page);
|
|
await use(po);
|
|
},
|
|
{ auto: true },
|
|
],
|
|
attachScreenshotsToReport: [
|
|
async ({ page }, use, testInfo) => {
|
|
await use();
|
|
|
|
// After the test we can check whether the test passed or failed.
|
|
if (testInfo.status !== testInfo.expectedStatus) {
|
|
const screenshot = await page.screenshot();
|
|
await testInfo.attach("screenshot", {
|
|
body: screenshot,
|
|
contentType: "image/png",
|
|
});
|
|
}
|
|
},
|
|
{ auto: true },
|
|
],
|
|
electronApp: [
|
|
async ({}, use) => {
|
|
// find the latest build in the out directory
|
|
const latestBuild = findLatestBuild();
|
|
// parse the directory and find paths and other info
|
|
const appInfo = parseElectronApp(latestBuild);
|
|
process.env.OLLAMA_HOST = "http://localhost:3500/ollama";
|
|
process.env.LM_STUDIO_BASE_URL_FOR_TESTING =
|
|
"http://localhost:3500/lmstudio";
|
|
process.env.E2E_TEST_BUILD = "true";
|
|
// This is just a hack to avoid the AI setup screen.
|
|
process.env.OPENAI_API_KEY = "sk-test";
|
|
const electronApp = await electron.launch({
|
|
args: [
|
|
appInfo.main,
|
|
"--enable-logging",
|
|
`--user-data-dir=/tmp/dyad-e2e-tests-${Date.now()}`,
|
|
],
|
|
executablePath: appInfo.executable,
|
|
recordVideo: {
|
|
dir: "test-results",
|
|
},
|
|
});
|
|
|
|
console.log("electronApp launched!");
|
|
if (showDebugLogs) {
|
|
// Listen to main process output immediately
|
|
electronApp.process().stdout?.on("data", (data) => {
|
|
console.log(`MAIN_PROCESS_STDOUT: ${data.toString()}`);
|
|
});
|
|
electronApp.process().stderr?.on("data", (data) => {
|
|
console.error(`MAIN_PROCESS_STDERR: ${data.toString()}`);
|
|
});
|
|
}
|
|
electronApp.on("close", () => {
|
|
console.log(`Electron app closed listener:`);
|
|
});
|
|
|
|
electronApp.on("window", async (page) => {
|
|
const filename = page.url()?.split("/").pop();
|
|
console.log(`Window opened: ${filename}`);
|
|
|
|
// capture errors
|
|
page.on("pageerror", (error) => {
|
|
console.error(error);
|
|
});
|
|
// capture console messages
|
|
page.on("console", (msg) => {
|
|
console.log(msg.text());
|
|
});
|
|
});
|
|
|
|
await use(electronApp);
|
|
await electronApp.close();
|
|
},
|
|
{ auto: true },
|
|
],
|
|
});
|
|
|
|
function prettifyDump(dumpContent: string) {
|
|
const parsedDump = JSON.parse(dumpContent) as Array<{
|
|
role: string;
|
|
content: string;
|
|
}>;
|
|
|
|
return parsedDump
|
|
.map((message) => {
|
|
const content = message.content
|
|
// We remove package.json because it's flaky.
|
|
// Depending on whether pnpm install is run, it will be modified,
|
|
// and the contents and timestamp (thus affecting order) will be affected.
|
|
.replace(
|
|
/\n<dyad-file path="package\.json">[\s\S]*?<\/dyad-file>\n/g,
|
|
"",
|
|
);
|
|
return `===\nrole: ${message.role}\nmessage: ${content}`;
|
|
})
|
|
.join("\n\n");
|
|
}
|