kunthawat/moreminimore-vibe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upload image via chat (#686)

Browse Source
This commit is contained in:
Will Chen
2025-07-22 15:45:30 -07:00
committed by GitHub
parent de73445766
commit 9edd0fa80f
16 changed files with 509 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
src/ipc/handlers/chat_stream_handlers.ts
View File

@@ -57,6 +57,7 @@ import {
getDyadRenameTags,
} from "../utils/dyad_tag_parser";
import { fileExists } from "../utils/file_utils";
import { FileUploadsState } from "../utils/file_uploads_state";
type AsyncIterableStream<T> = AsyncIterable<T> & ReadableStream<T>;
@@ -161,6 +162,9 @@ async function processStreamChunks({
export function registerChatStreamHandlers() {
ipcMain.handle("chat:stream", async (event, req: ChatStreamParams) => {
try {
const fileUploadsState = FileUploadsState.getInstance();
fileUploadsState.initialize({ chatId: req.chatId });
// Create an AbortController for this stream
const abortController = new AbortController();
activeStreams.set(req.chatId, abortController);
@@ -221,7 +225,7 @@ export function registerChatStreamHandlers() {
if (req.attachments && req.attachments.length > 0) {
attachmentInfo = "\n\nAttachments:\n";
for (const attachment of req.attachments) {
for (const [index, attachment] of req.attachments.entries()) {
// Generate a unique filename
const hash = crypto
.createHash("md5")
@@ -236,15 +240,30 @@ export function registerChatStreamHandlers() {
await writeFile(filePath, Buffer.from(base64Data, "base64"));
attachmentPaths.push(filePath);
attachmentInfo += `- ${attachment.name} (${attachment.type})\n`;
// If it's a text-based file, try to include the content
if (await isTextFile(filePath)) {
try {
attachmentInfo += `<dyad-text-attachment filename="${attachment.name}" type="${attachment.type}" path="${filePath}">
</dyad-text-attachment>
\n\n`;
} catch (err) {
logger.error(`Error reading file content: ${err}`);
if (attachment.attachmentType === "upload-to-codebase") {
// For upload-to-codebase, create a unique file ID and store the mapping
const fileId = `DYAD_ATTACHMENT_${index}`;
fileUploadsState.addFileUpload(fileId, {
filePath,
originalName: attachment.name,
});
// Add instruction for AI to use dyad-write tag
attachmentInfo += `\n\nFile to upload to codebase: ${attachment.name} (file id: ${fileId})\n`;
} else {
// For chat-context, use the existing logic
attachmentInfo += `- ${attachment.name} (${attachment.type})\n`;
// If it's a text-based file, try to include the content
if (await isTextFile(filePath)) {
try {
attachmentInfo += `<dyad-text-attachment filename="${attachment.name}" type="${attachment.type}" path="${filePath}">
</dyad-text-attachment>
\n\n`;
} catch (err) {
logger.error(`Error reading file content: ${err}`);
}
}
}
}
@@ -454,10 +473,35 @@ ${componentSnippet}
attachment.type.startsWith("image/"),
);
if (hasImageAttachments) {
const hasUploadedAttachments =
req.attachments &&
req.attachments.some(
(attachment) => attachment.attachmentType === "upload-to-codebase",
);
// If there's mixed attachments (e.g. some upload to codebase attachments and some upload images as chat context attachemnts)
// we will just include the file upload system prompt, otherwise the AI gets confused and doesn't reliably
// print out the dyad-write tags.
// Usually, AI models will want to use the image as reference to generate code (e.g. UI mockups) anyways, so
// it's not that critical to include the image analysis instructions.
if (hasUploadedAttachments) {
systemPrompt += `
When files are attached to this conversation, upload them to the codebase using this exact format:
<dyad-write path="path/to/destination/filename.ext" description="Upload file to codebase">
DYAD_ATTACHMENT_X
</dyad-write>
Example for file with id of DYAD_ATTACHMENT_0:
<dyad-write path="src/components/Button.jsx" description="Upload file to codebase">
DYAD_ATTACHMENT_0
</dyad-write>
`;
} else if (hasImageAttachments) {
systemPrompt += `
# Image Analysis Capabilities
# Image Analysis Instructions
This conversation includes one or more image attachments. When the user uploads images:
1. If the user explicitly asks for analysis, description, or information about the image, please analyze the image content.
2. Describe what you see in the image if asked.
@@ -857,7 +901,10 @@ ${problemReport.problems
const status = await processFullResponseActions(
fullResponse,
req.chatId,
{ chatSummary, messageId: placeholderAssistantMessage.id }, // Use placeholder ID
{
chatSummary,
messageId: placeholderAssistantMessage.id,
}, // Use placeholder ID
);
const chat = await db.query.chats.findFirst({
@@ -929,6 +976,8 @@ ${problemReport.problems
);
// Clean up the abort controller
activeStreams.delete(req.chatId);
// Clean up file uploads state on error
FileUploadsState.getInstance().clear();
return "error";
}
});

41
src/ipc/ipc_client.ts
View File

@@ -50,6 +50,7 @@ import type {
SaveVercelAccessTokenParams,
VercelProject,
UpdateChatParams,
FileAttachment,
} from "./ipc_types";
import type { AppChatContext, ProposalResult } from "@/lib/schemas";
import { showError } from "@/lib/toast";
@@ -258,7 +259,7 @@ export class IpcClient {
selectedComponent: ComponentSelection | null;
chatId: number;
redo?: boolean;
attachments?: File[];
attachments?: FileAttachment[];
onUpdate: (messages: Message[]) => void;
onEnd: (response: ChatResponseEnd) => void;
onError: (error: string) => void;
@@ -278,24 +279,28 @@ export class IpcClient {
// Handle file attachments if provided
if (attachments && attachments.length > 0) {
// Process each file and convert to base64
// Process each file attachment and convert to base64
Promise.all(
attachments.map(async (file) => {
return new Promise<{ name: string; type: string; data: string }>(
(resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
resolve({
name: file.name,
type: file.type,
data: reader.result as string,
});
};
reader.onerror = () =>
reject(new Error(`Failed to read file: ${file.name}`));
reader.readAsDataURL(file);
},
);
attachments.map(async (attachment) => {
return new Promise<{
name: string;
type: string;
data: string;
attachmentType: "upload-to-codebase" | "chat-context";
}>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
resolve({
name: attachment.file.name,
type: attachment.file.type,
data: reader.result as string,
attachmentType: attachment.type,
});
};
reader.onerror = () =>
reject(new Error(`Failed to read file: ${attachment.file.name}`));
reader.readAsDataURL(attachment.file);
});
}),
)
.then((fileDataArray) => {

18
src/ipc/ipc_types.ts
View File

@@ -22,6 +22,7 @@ export interface ChatStreamParams {
name: string;
type: string;
data: string; // Base64 encoded file data
attachmentType: "upload-to-codebase" | "chat-context"; // FileAttachment type
}>;
selectedComponent: ComponentSelection | null;
}
@@ -321,3 +322,20 @@ export interface UpdateChatParams {
chatId: number;
title: string;
}
export interface UploadFileToCodebaseParams {
appId: number;
filePath: string;
fileData: string; // Base64 encoded file data
fileName: string;
}
export interface UploadFileToCodebaseResult {
success: boolean;
filePath: string;
}
export interface FileAttachment {
file: File;
type: "upload-to-codebase" | "chat-context";
}

37
src/ipc/processors/response_processor.ts
View File

@@ -26,6 +26,7 @@ import {
getDyadAddDependencyTags,
getDyadExecuteSqlTags,
} from "../utils/dyad_tag_parser";
import { FileUploadsState } from "../utils/file_uploads_state";
const readFile = fs.promises.readFile;
const logger = log.scope("response_processor");
@@ -53,13 +54,19 @@ export async function processFullResponseActions(
{
chatSummary,
messageId,
}: { chatSummary: string | undefined; messageId: number },
}: {
chatSummary: string | undefined;
messageId: number;
},
): Promise<{
updatedFiles?: boolean;
error?: string;
extraFiles?: string[];
extraFilesError?: string;
}> {
const fileUploadsState = FileUploadsState.getInstance();
const fileUploadsMap = fileUploadsState.getFileUploadsForChat(chatId);
fileUploadsState.clear();
logger.log("processFullResponseActions for chatId", chatId);
// Get the app associated with the chat
const chatWithApp = await db.query.chats.findFirst({
@@ -289,9 +296,33 @@ export async function processFullResponseActions(
// Process all file writes
for (const tag of dyadWriteTags) {
const filePath = tag.path;
const content = tag.content;
let content: string | Buffer = tag.content;
const fullFilePath = safeJoin(appPath, filePath);
// Check if content (stripped of whitespace) exactly matches a file ID and replace with actual file content
if (fileUploadsMap) {
const trimmedContent = tag.content.trim();
const fileInfo = fileUploadsMap.get(trimmedContent);
if (fileInfo) {
try {
const fileContent = await readFile(fileInfo.filePath);
content = fileContent;
logger.log(
`Replaced file ID ${trimmedContent} with content from ${fileInfo.originalName}`,
);
} catch (error) {
logger.error(
`Failed to read uploaded file ${fileInfo.originalName}:`,
error,
);
errors.push({
message: `Failed to read uploaded file: ${fileInfo.originalName}`,
error: error,
});
}
}
}
// Ensure directory exists
const dirPath = path.dirname(fullFilePath);
fs.mkdirSync(dirPath, { recursive: true });
@@ -300,7 +331,7 @@ export async function processFullResponseActions(
fs.writeFileSync(fullFilePath, content);
logger.log(`Successfully wrote file: ${fullFilePath}`);
writtenFiles.push(filePath);
if (isServerFunction(filePath)) {
if (isServerFunction(filePath) && typeof content === "string") {
try {
await deploySupabaseFunctions({
supabaseProjectId: chatWithApp.app.supabaseProjectId!,

66
src/ipc/utils/file_uploads_state.ts Normal file
View File

@@ -0,0 +1,66 @@
import log from "electron-log";
const logger = log.scope("file_uploads_state");
export interface FileUploadInfo {
filePath: string;
originalName: string;
}
export class FileUploadsState {
private static instance: FileUploadsState;
private currentChatId: number | null = null;
private fileUploadsMap = new Map<string, FileUploadInfo>();
private constructor() {}
public static getInstance(): FileUploadsState {
if (!FileUploadsState.instance) {
FileUploadsState.instance = new FileUploadsState();
}
return FileUploadsState.instance;
}
/**
* Initialize file uploads state for a specific chat and message
*/
public initialize({ chatId }: { chatId: number }): void {
this.currentChatId = chatId;
this.fileUploadsMap.clear();
logger.debug(`Initialized file uploads state for chat ${chatId}`);
}
/**
* Add a file upload mapping
*/
public addFileUpload(fileId: string, fileInfo: FileUploadInfo): void {
this.fileUploadsMap.set(fileId, fileInfo);
logger.log(`Added file upload: ${fileId} -> ${fileInfo.originalName}`);
}
/**
* Get the current file uploads map
*/
public getFileUploadsForChat(chatId: number): Map<string, FileUploadInfo> {
if (this.currentChatId !== chatId) {
return new Map();
}
return new Map(this.fileUploadsMap);
}
/**
* Get current chat ID
*/
public getCurrentChatId(): number | null {
return this.currentChatId;
}
/**
* Clear the current state
*/
public clear(): void {
this.currentChatId = null;
this.fileUploadsMap.clear();
logger.debug("Cleared file uploads state");
}
}