<!-- This is an auto-generated description by cubic. -->
## Summary by cubic
Adds an in-app screenshot annotator to the Preview panel for Pro users
so you can capture the current app view, draw or add text, and submit an
annotated image to chat.

- **New Features**
- Pen button in PreviewIframe to toggle annotator; captures a screenshot
via worker messaging and displays it in a Konva canvas.
- Tools: select, freehand draw, and draggable text; supports undo/redo,
delete, and resizing with Transformer. Canvas scales to the container.
Includes a color picker.
- Submit exports a PNG and attaches it to the chat via useAttachments;
prefills the chat input; annotator auto-closes after submit.
  - Pro-only: non-Pro users see an upsell screen.
- State atoms added: annotatorModeAtom, screenshotDataUrlAtom,
attachmentsAtom; PreviewIframe now handles dyad-screenshot-response
messages.

- **Dependencies**
  - Added konva, react-konva, perfect-freehand, and html-to-image.
- Proxy now injects html-to-image and the new dyad-screenshot-client.js
for screenshot capture.

<sup>Written for commit 580aca271c5993a0dc7426e36e34393e073bd67b.
Summary will update automatically on new commits.</sup>

<!-- End of auto-generated description by cubic. -->
This commit is contained in:
Mohamed Aziz Mejri
2025-12-13 19:40:31 +01:00
committed by GitHub
parent 86e4005795
commit a4ab1a7f84
17 changed files with 1740 additions and 244 deletions

View File

@@ -23,6 +23,7 @@ import {
Monitor,
Tablet,
Smartphone,
Pen,
} from "lucide-react";
import { selectedChatIdAtom } from "@/atoms/chatAtoms";
import { CopyErrorMessage } from "@/components/CopyErrorMessage";
@@ -36,12 +37,13 @@ import {
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { useStreamChat } from "@/hooks/useStreamChat";
import { useUserBudgetInfo } from "@/hooks/useUserBudgetInfo";
import {
selectedComponentsPreviewAtom,
visualEditingSelectedComponentAtom,
currentComponentCoordinatesAtom,
previewIframeRefAtom,
annotatorModeAtom,
screenshotDataUrlAtom,
pendingVisualChangesAtom,
} from "@/atoms/previewAtoms";
import { ComponentSelection } from "@/ipc/ipc_types";
@@ -61,6 +63,11 @@ import { useRunApp } from "@/hooks/useRunApp";
import { useShortcut } from "@/hooks/useShortcut";
import { cn } from "@/lib/utils";
import { normalizePath } from "../../../shared/normalizePath";
import { showError } from "@/lib/toast";
import { AnnotatorOnlyForPro } from "./AnnotatorOnlyForPro";
import { useAttachments } from "@/hooks/useAttachments";
import { useUserBudgetInfo } from "@/hooks/useUserBudgetInfo";
import { Annotator } from "@/pro/ui/components/Annotator/Annotator";
import { VisualEditingToolbar } from "./VisualEditingToolbar";
interface ErrorBannerProps {
@@ -193,12 +200,32 @@ export const PreviewIframe = ({ loading }: { loading: boolean }) => {
const setPreviewIframeRef = useSetAtom(previewIframeRefAtom);
const iframeRef = useRef<HTMLIFrameElement>(null);
const [isPicking, setIsPicking] = useState(false);
const [annotatorMode, setAnnotatorMode] = useAtom(annotatorModeAtom);
const [screenshotDataUrl, setScreenshotDataUrl] = useAtom(
screenshotDataUrlAtom,
);
const { addAttachments } = useAttachments();
const setPendingChanges = useSetAtom(pendingVisualChangesAtom);
// AST Analysis State
const [isDynamicComponent, setIsDynamicComponent] = useState(false);
const [hasStaticText, setHasStaticText] = useState(false);
// Device mode state
type DeviceMode = "desktop" | "tablet" | "mobile";
const [deviceMode, setDeviceMode] = useState<DeviceMode>("desktop");
const [isDevicePopoverOpen, setIsDevicePopoverOpen] = useState(false);
// Device configurations
const deviceWidthConfig = {
tablet: 768,
mobile: 375,
};
//detect if the user is using Mac
const isMac = navigator.platform.toUpperCase().indexOf("MAC") >= 0;
const analyzeComponent = async (componentId: string) => {
if (!componentId || !selectedAppId) return;
@@ -283,21 +310,9 @@ export const PreviewIframe = ({ loading }: { loading: boolean }) => {
console.error("Failed to get element styles:", error);
}
};
// Device mode state
type DeviceMode = "desktop" | "tablet" | "mobile";
const [deviceMode, setDeviceMode] = useState<DeviceMode>("desktop");
const [isDevicePopoverOpen, setIsDevicePopoverOpen] = useState(false);
// Device configurations
const deviceWidthConfig = {
tablet: 768,
mobile: 375,
};
//detect if the user is using Mac
const isMac = navigator.platform.toUpperCase().indexOf("MAC") >= 0;
useEffect(() => {
setAnnotatorMode(false);
}, []);
// Reset visual editing state when app changes or component unmounts
useEffect(() => {
return () => {
@@ -423,6 +438,16 @@ export const PreviewIframe = ({ loading }: { loading: boolean }) => {
return;
}
if (event.data?.type === "dyad-screenshot-response") {
if (event.data.success && event.data.dataUrl) {
setScreenshotDataUrl(event.data.dataUrl);
setAnnotatorMode(true);
} else {
showError(event.data.error);
}
return;
}
const { type, payload } = event.data as {
type:
| "window-error"
@@ -558,6 +583,22 @@ export const PreviewIframe = ({ loading }: { loading: boolean }) => {
}
};
// Function to handle annotator button click
const handleAnnotatorClick = () => {
if (annotatorMode) {
setAnnotatorMode(false);
return;
}
if (iframeRef.current?.contentWindow) {
iframeRef.current.contentWindow.postMessage(
{
type: "dyad-take-screenshot",
},
"*",
);
}
};
// Activate component selector using a shortcut
useShortcut(
"c",
@@ -675,203 +716,239 @@ export const PreviewIframe = ({ loading }: { loading: boolean }) => {
return (
<div className="flex flex-col h-full">
{/* Browser-style header */}
<div className="flex items-center p-2 border-b space-x-2 ">
{/* Navigation Buttons */}
<div className="flex space-x-1">
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<button
onClick={handleActivateComponentSelector}
className={`p-1 rounded transition-colors duration-200 disabled:opacity-50 disabled:cursor-not-allowed ${
isPicking
? "bg-purple-500 text-white hover:bg-purple-600 dark:bg-purple-600 dark:hover:bg-purple-700"
: " text-purple-700 hover:bg-purple-200 dark:text-purple-300 dark:hover:bg-purple-900"
}`}
disabled={
loading || !selectedAppId || !isComponentSelectorInitialized
}
data-testid="preview-pick-element-button"
>
<MousePointerClick size={16} />
</button>
</TooltipTrigger>
<TooltipContent>
<p>
{isPicking
? "Deactivate component selector"
: "Select component"}
</p>
<p>{isMac ? "⌘ + ⇧ + C" : "Ctrl + ⇧ + C"}</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
<button
className="p-1 rounded hover:bg-gray-200 dark:hover:bg-gray-700 disabled:opacity-50 disabled:cursor-not-allowed dark:text-gray-300"
disabled={!canGoBack || loading || !selectedAppId}
onClick={handleNavigateBack}
data-testid="preview-navigate-back-button"
>
<ArrowLeft size={16} />
</button>
<button
className="p-1 rounded hover:bg-gray-200 dark:hover:bg-gray-700 disabled:opacity-50 disabled:cursor-not-allowed dark:text-gray-300"
disabled={!canGoForward || loading || !selectedAppId}
onClick={handleNavigateForward}
data-testid="preview-navigate-forward-button"
>
<ArrowRight size={16} />
</button>
<button
onClick={handleReload}
className="p-1 rounded hover:bg-gray-200 dark:hover:bg-gray-700 disabled:opacity-50 disabled:cursor-not-allowed dark:text-gray-300"
disabled={loading || !selectedAppId}
data-testid="preview-refresh-button"
>
<RefreshCw size={16} />
</button>
</div>
{/* Address Bar with Routes Dropdown - using shadcn/ui dropdown-menu */}
<div className="relative flex-grow min-w-20">
<DropdownMenu>
<DropdownMenuTrigger asChild>
<div className="flex items-center justify-between px-3 py-1 bg-gray-100 dark:bg-gray-700 rounded text-sm text-gray-700 dark:text-gray-200 cursor-pointer w-full min-w-0">
<span className="truncate flex-1 mr-2 min-w-0">
{navigationHistory[currentHistoryPosition]
? new URL(navigationHistory[currentHistoryPosition])
.pathname
: "/"}
</span>
<ChevronDown size={14} className="flex-shrink-0" />
</div>
</DropdownMenuTrigger>
<DropdownMenuContent className="w-full">
{availableRoutes.length > 0 ? (
availableRoutes.map((route) => (
<DropdownMenuItem
key={route.path}
onClick={() => navigateToRoute(route.path)}
className="flex justify-between"
>
<span>{route.label}</span>
<span className="text-gray-500 dark:text-gray-400 text-xs">
{route.path}
</span>
</DropdownMenuItem>
))
) : (
<DropdownMenuItem disabled>Loading routes...</DropdownMenuItem>
)}
</DropdownMenuContent>
</DropdownMenu>
</div>
{/* Action Buttons */}
<div className="flex space-x-1">
<button
onClick={onRestart}
className="flex items-center space-x-1 px-3 py-1 rounded-md text-sm hover:bg-[var(--background-darkest)] transition-colors"
title="Restart App"
>
<Power size={16} />
<span>Restart</span>
</button>
<button
data-testid="preview-open-browser-button"
onClick={() => {
if (originalUrl) {
IpcClient.getInstance().openExternalUrl(originalUrl);
}
}}
className="p-1 rounded hover:bg-gray-200 dark:hover:bg-gray-700 disabled:opacity-50 disabled:cursor-not-allowed dark:text-gray-300"
>
<ExternalLink size={16} />
</button>
{/* Device Mode Button */}
<Popover open={isDevicePopoverOpen} modal={false}>
<PopoverTrigger asChild>
<button
data-testid="device-mode-button"
onClick={() => {
// Toggle popover open/close
if (isDevicePopoverOpen) setDeviceMode("desktop");
setIsDevicePopoverOpen(!isDevicePopoverOpen);
}}
className={cn(
"p-1 rounded hover:bg-gray-200 dark:hover:bg-gray-700 dark:text-gray-300",
deviceMode !== "desktop" && "bg-gray-200 dark:bg-gray-700",
)}
title="Device Mode"
>
<MonitorSmartphone size={16} />
</button>
</PopoverTrigger>
<PopoverContent
className="w-auto p-2"
onOpenAutoFocus={(e) => e.preventDefault()}
onInteractOutside={(e) => e.preventDefault()}
>
<TooltipProvider>
<ToggleGroup
type="single"
value={deviceMode}
onValueChange={(value) => {
if (value) {
setDeviceMode(value as DeviceMode);
setIsDevicePopoverOpen(false);
{/* Browser-style header - hide when annotator is active */}
{!annotatorMode && (
<div className="flex items-center p-2 border-b space-x-2">
{/* Navigation Buttons */}
<div className="flex space-x-1">
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<button
onClick={handleActivateComponentSelector}
className={`p-1 rounded transition-colors duration-200 disabled:opacity-50 disabled:cursor-not-allowed ${
isPicking
? "bg-purple-500 text-white hover:bg-purple-600 dark:bg-purple-600 dark:hover:bg-purple-700"
: " text-purple-700 hover:bg-purple-200 dark:text-purple-300 dark:hover:bg-purple-900"
}`}
disabled={
loading ||
!selectedAppId ||
!isComponentSelectorInitialized
}
}}
variant="outline"
>
{/* Tooltips placed inside items instead of wrapping
to avoid asChild prop merging that breaks highlighting */}
<ToggleGroupItem value="desktop" aria-label="Desktop view">
<Tooltip>
<TooltipTrigger asChild>
<span className="flex items-center justify-center">
<Monitor size={16} />
</span>
</TooltipTrigger>
<TooltipContent>
<p>Desktop</p>
</TooltipContent>
</Tooltip>
</ToggleGroupItem>
<ToggleGroupItem value="tablet" aria-label="Tablet view">
<Tooltip>
<TooltipTrigger asChild>
<span className="flex items-center justify-center">
<Tablet size={16} className="scale-x-130" />
</span>
</TooltipTrigger>
<TooltipContent>
<p>Tablet</p>
</TooltipContent>
</Tooltip>
</ToggleGroupItem>
<ToggleGroupItem value="mobile" aria-label="Mobile view">
<Tooltip>
<TooltipTrigger asChild>
<span className="flex items-center justify-center">
<Smartphone size={16} />
</span>
</TooltipTrigger>
<TooltipContent>
<p>Mobile</p>
</TooltipContent>
</Tooltip>
</ToggleGroupItem>
</ToggleGroup>
</TooltipProvider>
</PopoverContent>
</Popover>
</div>
</div>
data-testid="preview-pick-element-button"
>
<MousePointerClick size={16} />
</button>
</TooltipTrigger>
<TooltipContent>
<p>
{isPicking
? "Deactivate component selector"
: "Select component"}
</p>
<p>{isMac ? "⌘ + ⇧ + C" : "Ctrl + ⇧ + C"}</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<button
onClick={handleAnnotatorClick}
className={`p-1 rounded transition-colors duration-200 disabled:opacity-50 disabled:cursor-not-allowed ${
annotatorMode
? "bg-purple-500 text-white hover:bg-purple-600 dark:bg-purple-600 dark:hover:bg-purple-700"
: " text-purple-700 hover:bg-purple-200 dark:text-purple-300 dark:hover:bg-purple-900"
}`}
disabled={
loading ||
!selectedAppId ||
isPicking ||
!isComponentSelectorInitialized
}
data-testid="preview-annotator-button"
>
<Pen size={16} />
</button>
</TooltipTrigger>
<TooltipContent>
<p>
{annotatorMode
? "Annotator mode active"
: "Activate annotator"}
</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
<button
className="p-1 rounded hover:bg-gray-200 dark:hover:bg-gray-700 disabled:opacity-50 disabled:cursor-not-allowed dark:text-gray-300"
disabled={!canGoBack || loading || !selectedAppId}
onClick={handleNavigateBack}
data-testid="preview-navigate-back-button"
>
<ArrowLeft size={16} />
</button>
<button
className="p-1 rounded hover:bg-gray-200 dark:hover:bg-gray-700 disabled:opacity-50 disabled:cursor-not-allowed dark:text-gray-300"
disabled={!canGoForward || loading || !selectedAppId}
onClick={handleNavigateForward}
data-testid="preview-navigate-forward-button"
>
<ArrowRight size={16} />
</button>
<button
onClick={handleReload}
className="p-1 rounded hover:bg-gray-200 dark:hover:bg-gray-700 disabled:opacity-50 disabled:cursor-not-allowed dark:text-gray-300"
disabled={loading || !selectedAppId}
data-testid="preview-refresh-button"
>
<RefreshCw size={16} />
</button>
</div>
<div className="relative flex-grow ">
{/* Address Bar with Routes Dropdown - using shadcn/ui dropdown-menu */}
<div className="relative flex-grow min-w-20">
<DropdownMenu>
<DropdownMenuTrigger asChild>
<div className="flex items-center justify-between px-3 py-1 bg-gray-100 dark:bg-gray-700 rounded text-sm text-gray-700 dark:text-gray-200 cursor-pointer w-full min-w-0">
<span className="truncate flex-1 mr-2 min-w-0">
{navigationHistory[currentHistoryPosition]
? new URL(navigationHistory[currentHistoryPosition])
.pathname
: "/"}
</span>
<ChevronDown size={14} className="flex-shrink-0" />
</div>
</DropdownMenuTrigger>
<DropdownMenuContent className="w-full">
{availableRoutes.length > 0 ? (
availableRoutes.map((route) => (
<DropdownMenuItem
key={route.path}
onClick={() => navigateToRoute(route.path)}
className="flex justify-between"
>
<span>{route.label}</span>
<span className="text-gray-500 dark:text-gray-400 text-xs">
{route.path}
</span>
</DropdownMenuItem>
))
) : (
<DropdownMenuItem disabled>
Loading routes...
</DropdownMenuItem>
)}
</DropdownMenuContent>
</DropdownMenu>
</div>
{/* Action Buttons */}
<div className="flex space-x-1">
<button
onClick={onRestart}
className="flex items-center space-x-1 px-3 py-1 rounded-md text-sm hover:bg-[var(--background-darkest)] transition-colors"
title="Restart App"
>
<Power size={16} />
<span>Restart</span>
</button>
<button
data-testid="preview-open-browser-button"
onClick={() => {
if (originalUrl) {
IpcClient.getInstance().openExternalUrl(originalUrl);
}
}}
className="p-1 rounded hover:bg-gray-200 dark:hover:bg-gray-700 disabled:opacity-50 disabled:cursor-not-allowed dark:text-gray-300"
>
<ExternalLink size={16} />
</button>
{/* Device Mode Button */}
<Popover open={isDevicePopoverOpen} modal={false}>
<PopoverTrigger asChild>
<button
data-testid="device-mode-button"
onClick={() => {
// Toggle popover open/close
if (isDevicePopoverOpen) setDeviceMode("desktop");
setIsDevicePopoverOpen(!isDevicePopoverOpen);
}}
className={cn(
"p-1 rounded hover:bg-gray-200 dark:hover:bg-gray-700 dark:text-gray-300",
deviceMode !== "desktop" && "bg-gray-200 dark:bg-gray-700",
)}
title="Device Mode"
>
<MonitorSmartphone size={16} />
</button>
</PopoverTrigger>
<PopoverContent
className="w-auto p-2"
onOpenAutoFocus={(e) => e.preventDefault()}
onInteractOutside={(e) => e.preventDefault()}
>
<TooltipProvider>
<ToggleGroup
type="single"
value={deviceMode}
onValueChange={(value) => {
if (value) {
setDeviceMode(value as DeviceMode);
setIsDevicePopoverOpen(false);
}
}}
variant="outline"
>
{/* Tooltips placed inside items instead of wrapping
to avoid asChild prop merging that breaks highlighting */}
<ToggleGroupItem value="desktop" aria-label="Desktop view">
<Tooltip>
<TooltipTrigger asChild>
<span className="flex items-center justify-center">
<Monitor size={16} />
</span>
</TooltipTrigger>
<TooltipContent>
<p>Desktop</p>
</TooltipContent>
</Tooltip>
</ToggleGroupItem>
<ToggleGroupItem value="tablet" aria-label="Tablet view">
<Tooltip>
<TooltipTrigger asChild>
<span className="flex items-center justify-center">
<Tablet size={16} className="scale-x-130" />
</span>
</TooltipTrigger>
<TooltipContent>
<p>Tablet</p>
</TooltipContent>
</Tooltip>
</ToggleGroupItem>
<ToggleGroupItem value="mobile" aria-label="Mobile view">
<Tooltip>
<TooltipTrigger asChild>
<span className="flex items-center justify-center">
<Smartphone size={16} />
</span>
</TooltipTrigger>
<TooltipContent>
<p>Mobile</p>
</TooltipContent>
</Tooltip>
</ToggleGroupItem>
</ToggleGroup>
</TooltipProvider>
</PopoverContent>
</Popover>
</div>
</div>
)}
<div className="relative flex-grow overflow-hidden">
<ErrorBanner
error={errorMessage}
onDismiss={() => setErrorMessage(undefined)}
@@ -899,32 +976,59 @@ export const PreviewIframe = ({ loading }: { loading: boolean }) => {
deviceMode !== "desktop" && "flex justify-center",
)}
>
<iframe
sandbox="allow-same-origin allow-scripts allow-forms allow-popups allow-modals allow-orientation-lock allow-pointer-lock allow-presentation allow-downloads"
data-testid="preview-iframe-element"
onLoad={() => {
setErrorMessage(undefined);
}}
ref={iframeRef}
key={reloadKey}
title={`Preview for App ${selectedAppId}`}
className="w-full h-full border-none bg-white dark:bg-gray-950"
style={
deviceMode == "desktop"
? {}
: { width: `${deviceWidthConfig[deviceMode]}px` }
}
src={appUrl}
allow="clipboard-read; clipboard-write; fullscreen; microphone; camera; display-capture; geolocation; autoplay; picture-in-picture"
/>
{/* Visual Editing Toolbar */}
{isProMode && visualEditingSelectedComponent && selectedAppId && (
<VisualEditingToolbar
selectedComponent={visualEditingSelectedComponent}
iframeRef={iframeRef}
isDynamic={isDynamicComponent}
hasStaticText={hasStaticText}
/>
{annotatorMode && screenshotDataUrl ? (
<div
className="w-full h-full bg-white dark:bg-gray-950"
style={
deviceMode == "desktop"
? {}
: { width: `${deviceWidthConfig[deviceMode]}px` }
}
>
{userBudget ? (
<Annotator
screenshotUrl={screenshotDataUrl}
onSubmit={addAttachments}
handleAnnotatorClick={handleAnnotatorClick}
/>
) : (
<AnnotatorOnlyForPro
onGoBack={() => setAnnotatorMode(false)}
/>
)}
</div>
) : (
<>
<iframe
sandbox="allow-same-origin allow-scripts allow-forms allow-popups allow-modals allow-orientation-lock allow-pointer-lock allow-presentation allow-downloads"
data-testid="preview-iframe-element"
onLoad={() => {
setErrorMessage(undefined);
}}
ref={iframeRef}
key={reloadKey}
title={`Preview for App ${selectedAppId}`}
className="w-full h-full border-none bg-white dark:bg-gray-950"
style={
deviceMode == "desktop"
? {}
: { width: `${deviceWidthConfig[deviceMode]}px` }
}
src={appUrl}
allow="clipboard-read; clipboard-write; fullscreen; microphone; camera; display-capture; geolocation; autoplay; picture-in-picture"
/>
{/* Visual Editing Toolbar */}
{isProMode &&
visualEditingSelectedComponent &&
selectedAppId && (
<VisualEditingToolbar
selectedComponent={visualEditingSelectedComponent}
iframeRef={iframeRef}
isDynamic={isDynamicComponent}
hasStaticText={hasStaticText}
/>
)}
</>
)}
</div>
)}