Annotator (#1861)
<!-- This is an auto-generated description by cubic. --> ## Summary by cubic Adds an in-app screenshot annotator to the Preview panel for Pro users so you can capture the current app view, draw or add text, and submit an annotated image to chat. - **New Features** - Pen button in PreviewIframe to toggle annotator; captures a screenshot via worker messaging and displays it in a Konva canvas. - Tools: select, freehand draw, and draggable text; supports undo/redo, delete, and resizing with Transformer. Canvas scales to the container. Includes a color picker. - Submit exports a PNG and attaches it to the chat via useAttachments; prefills the chat input; annotator auto-closes after submit. - Pro-only: non-Pro users see an upsell screen. - State atoms added: annotatorModeAtom, screenshotDataUrlAtom, attachmentsAtom; PreviewIframe now handles dyad-screenshot-response messages. - **Dependencies** - Added konva, react-konva, perfect-freehand, and html-to-image. - Proxy now injects html-to-image and the new dyad-screenshot-client.js for screenshot capture. <sup>Written for commit 580aca271c5993a0dc7426e36e34393e073bd67b. Summary will update automatically on new commits.</sup> <!-- End of auto-generated description by cubic. -->
This commit is contained in:
committed by
GitHub
parent
86e4005795
commit
a4ab1a7f84
75
e2e-tests/annotator.spec.ts
Normal file
75
e2e-tests/annotator.spec.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
import { testSkipIfWindows } from "./helpers/test_helper";
|
||||
import { expect } from "@playwright/test";
|
||||
import fs from "fs";
|
||||
|
||||
testSkipIfWindows(
|
||||
"annotator - capture and submit screenshot",
|
||||
async ({ po }) => {
|
||||
await po.setUpDyadPro({ autoApprove: true });
|
||||
|
||||
// Create a basic app
|
||||
await po.sendPrompt("basic");
|
||||
|
||||
// Click the annotator button to activate annotator mode
|
||||
await po.clickPreviewAnnotatorButton();
|
||||
|
||||
// Wait for annotator mode to be active
|
||||
await po.waitForAnnotatorMode();
|
||||
|
||||
// Submit the screenshot to chat
|
||||
await po.clickAnnotatorSubmit();
|
||||
|
||||
await expect(po.getChatInput()).toContainText(
|
||||
"Please update the UI based on these screenshots",
|
||||
);
|
||||
|
||||
// Verify the screenshot was attached to chat context
|
||||
await po.sendPrompt("[dump]");
|
||||
|
||||
// Wait for the LLM response containing the dump path to appear in the UI
|
||||
// before attempting to extract it from the messages list
|
||||
await po.page.waitForSelector("text=/\\[\\[dyad-dump-path=.*\\]\\]/");
|
||||
|
||||
// Get the dump file path from the messages list
|
||||
const messagesListText = await po.page
|
||||
.getByTestId("messages-list")
|
||||
.textContent();
|
||||
const dumpPathMatch = messagesListText?.match(
|
||||
/\[\[dyad-dump-path=([^\]]+)\]\]/,
|
||||
);
|
||||
|
||||
if (!dumpPathMatch) {
|
||||
throw new Error("No dump path found in messages list");
|
||||
}
|
||||
|
||||
const dumpFilePath = dumpPathMatch[1];
|
||||
const dumpContent = fs.readFileSync(dumpFilePath, "utf-8");
|
||||
const parsedDump = JSON.parse(dumpContent);
|
||||
|
||||
// Get the last message from the dump
|
||||
const messages = parsedDump.body.messages;
|
||||
const lastMessage = messages[messages.length - 1];
|
||||
|
||||
expect(lastMessage).toBeTruthy();
|
||||
expect(lastMessage.content).toBeTruthy();
|
||||
|
||||
// The content is an array with text and image parts
|
||||
expect(Array.isArray(lastMessage.content)).toBe(true);
|
||||
|
||||
// Find the text part and verify it mentions the PNG attachment
|
||||
const textPart = lastMessage.content.find(
|
||||
(part: any) => part.type === "text",
|
||||
);
|
||||
expect(textPart).toBeTruthy();
|
||||
expect(textPart.text).toMatch(/annotated-screenshot-.*\.png/);
|
||||
expect(textPart.text).toMatch(/image\/png/);
|
||||
|
||||
// Find the image part and verify it has the correct structure
|
||||
const imagePart = lastMessage.content.find(
|
||||
(part: any) => part.type === "image_url",
|
||||
);
|
||||
expect(imagePart).toBeTruthy();
|
||||
expect(imagePart.image_url).toBeTruthy();
|
||||
expect(imagePart.image_url.url).toMatch(/^data:image\/png;base64,/);
|
||||
},
|
||||
);
|
||||
Reference in New Issue
Block a user