diff --git a/src/ipc/handlers/chat_stream_handlers.ts b/src/ipc/handlers/chat_stream_handlers.ts index ae50789..a8813ec 100644 --- a/src/ipc/handlers/chat_stream_handlers.ts +++ b/src/ipc/handlers/chat_stream_handlers.ts @@ -1,5 +1,5 @@ import { ipcMain } from "electron"; -import { CoreMessage, TextPart, ImagePart, streamText } from "ai"; +import { CoreMessage, TextPart, ImagePart } from "ai"; import { db } from "../../db"; import { chats, messages } from "../../db/schema"; import { and, eq, isNull } from "drizzle-orm"; @@ -29,6 +29,7 @@ import * as crypto from "crypto"; import { readFile, writeFile, unlink } from "fs/promises"; import { getMaxTokens } from "../utils/token_utils"; import { MAX_CHAT_TURNS_IN_CONTEXT } from "@/constants/settings_constants"; +import { streamTextWithBackup } from "../utils/stream_utils"; const logger = log.scope("chat_stream_handlers"); @@ -214,7 +215,7 @@ export function registerChatStreamHandlers() { } else { // Normal AI processing for non-test prompts const settings = readSettings(); - const modelClient = await getModelClient( + const { modelClient, backupModelClients } = await getModelClient( settings.selectedModel, settings, ); @@ -372,13 +373,14 @@ This conversation includes one or more image attachments. When the user uploads } // When calling streamText, the messages need to be properly formatted for mixed content - const { textStream } = streamText({ + const { textStream } = streamTextWithBackup({ maxTokens: await getMaxTokens(settings.selectedModel), temperature: 0, model: modelClient, + backupModelClients: backupModelClients, system: systemPrompt, messages: chatMessages.filter((m) => m.content), - onError: (error) => { + onError: (error: any) => { logger.error("Error streaming text:", error); const message = (error as any)?.error?.message || JSON.stringify(error); diff --git a/src/ipc/handlers/safe_handle.ts b/src/ipc/handlers/safe_handle.ts index 4357ee8..0a2f14c 100644 --- a/src/ipc/handlers/safe_handle.ts +++ b/src/ipc/handlers/safe_handle.ts @@ -12,7 +12,9 @@ export function createLoggedHandler(logger: log.LogFunctions) { logger.log(`IPC: ${channel} called with args: ${JSON.stringify(args)}`); try { const result = await fn(event, ...args); - logger.log(`IPC: ${channel} returned: ${JSON.stringify(result)}`); + logger.log( + `IPC: ${channel} returned: ${JSON.stringify(result).slice(0, 100)}...`, + ); return result; } catch (error) { logger.error( diff --git a/src/ipc/utils/get_model_client.ts b/src/ipc/utils/get_model_client.ts index e7a9a0f..20fe6a4 100644 --- a/src/ipc/utils/get_model_client.ts +++ b/src/ipc/utils/get_model_client.ts @@ -1,3 +1,4 @@ +import { LanguageModelV1 } from "ai"; import { createOpenAI } from "@ai-sdk/openai"; import { createGoogleGenerativeAI as createGoogle } from "@ai-sdk/google"; import { createAnthropic } from "@ai-sdk/anthropic"; @@ -8,6 +9,8 @@ import type { LargeLanguageModel, UserSettings } from "../../lib/schemas"; import { getEnvVar } from "./read_env"; import log from "electron-log"; import { getLanguageModelProviders } from "../shared/language_model_helpers"; +import { LanguageModelProvider } from "../ipc_types"; +import { llmErrorStore } from "@/main/llm_error_store"; const AUTO_MODELS = [ { @@ -24,11 +27,19 @@ const AUTO_MODELS = [ }, ]; +export interface ModelClient { + model: LanguageModelV1; + builtinProviderId?: string; +} + const logger = log.scope("getModelClient"); export async function getModelClient( model: LargeLanguageModel, settings: UserSettings, -) { +): Promise<{ + modelClient: ModelClient; + backupModelClients: ModelClient[]; +}> { const allProviders = await getLanguageModelProviders(); const dyadApiKey = settings.providerSettings?.auto?.apiKey?.value; @@ -83,7 +94,44 @@ export async function getModelClient( logger.info("Using Dyad Pro API key via Gateway"); // Do not use free variant (for openrouter). const modelName = model.name.split(":free")[0]; - return provider(`${providerConfig.gatewayPrefix}${modelName}`); + const autoModelClient = { + model: provider(`${providerConfig.gatewayPrefix}${modelName}`), + builtinProviderId: "auto", + }; + const googleSettings = settings.providerSettings?.google; + + // Budget saver mode logic (all must be true): + // 1. Pro Saver Mode is enabled + // 2. Provider is Google + // 3. API Key is set + // 4. Has no recent errors + if ( + settings.enableProSaverMode && + providerConfig.id === "google" && + googleSettings && + googleSettings.apiKey?.value && + llmErrorStore.modelHasNoRecentError({ + model: model.name, + provider: providerConfig.id, + }) + ) { + return { + modelClient: getRegularModelClient( + { + provider: providerConfig.id, + name: model.name, + }, + settings, + providerConfig, + ).modelClient, + backupModelClients: [autoModelClient], + }; + } else { + return { + modelClient: autoModelClient, + backupModelClients: [], + }; + } } else { logger.warn( `Dyad Pro enabled, but provider ${model.provider} does not have a gateway prefix defined. Falling back to direct provider connection.`, @@ -91,7 +139,14 @@ export async function getModelClient( // Fall through to regular provider logic if gateway prefix is missing } } + return getRegularModelClient(model, settings, providerConfig); +} +function getRegularModelClient( + model: LargeLanguageModel, + settings: UserSettings, + providerConfig: LanguageModelProvider, +) { // Get API key for the specific provider const apiKey = settings.providerSettings?.[model.provider]?.apiKey?.value || @@ -99,30 +154,60 @@ export async function getModelClient( ? getEnvVar(providerConfig.envVarName) : undefined); + const providerId = providerConfig.id; // Create client based on provider ID or type - switch (providerConfig.id) { + switch (providerId) { case "openai": { const provider = createOpenAI({ apiKey }); - return provider(model.name); + return { + modelClient: { + model: provider(model.name), + builtinProviderId: providerId, + }, + backupModelClients: [], + }; } case "anthropic": { const provider = createAnthropic({ apiKey }); - return provider(model.name); + return { + modelClient: { + model: provider(model.name), + builtinProviderId: providerId, + }, + backupModelClients: [], + }; } case "google": { const provider = createGoogle({ apiKey }); - return provider(model.name); + return { + modelClient: { + model: provider(model.name), + builtinProviderId: providerId, + }, + backupModelClients: [], + }; } case "openrouter": { const provider = createOpenRouter({ apiKey }); - return provider(model.name); + return { + modelClient: { + model: provider(model.name), + builtinProviderId: providerId, + }, + backupModelClients: [], + }; } case "ollama": { // Ollama typically runs locally and doesn't require an API key in the same way const provider = createOllama({ baseURL: providerConfig.apiBaseUrl, }); - return provider(model.name); + return { + modelClient: { + model: provider(model.name), + }, + backupModelClients: [], + }; } case "lmstudio": { // LM Studio uses OpenAI compatible API @@ -131,7 +216,12 @@ export async function getModelClient( name: "lmstudio", baseURL, }); - return provider(model.name); + return { + modelClient: { + model: provider(model.name), + }, + backupModelClients: [], + }; } default: { // Handle custom providers @@ -147,7 +237,12 @@ export async function getModelClient( baseURL: providerConfig.apiBaseUrl, apiKey: apiKey, }); - return provider(model.name); + return { + modelClient: { + model: provider(model.name), + }, + backupModelClients: [], + }; } // If it's not a known ID and not type 'custom', it's unsupported throw new Error(`Unsupported model provider: ${model.provider}`); diff --git a/src/ipc/utils/stream_utils.ts b/src/ipc/utils/stream_utils.ts new file mode 100644 index 0000000..1256ca1 --- /dev/null +++ b/src/ipc/utils/stream_utils.ts @@ -0,0 +1,123 @@ +import { streamText } from "ai"; +import log from "electron-log"; +import { ModelClient } from "./get_model_client"; +import { llmErrorStore } from "@/main/llm_error_store"; +const logger = log.scope("stream_utils"); + +export interface StreamTextWithBackupParams + extends Omit[0], "model"> { + model: ModelClient; // primary client + backupModelClients?: ModelClient[]; // ordered fall-backs +} + +export function streamTextWithBackup(params: StreamTextWithBackupParams): { + textStream: AsyncIterable; +} { + const { + model: primaryModel, + backupModelClients = [], + onError: callerOnError, + abortSignal: callerAbort, + ...rest + } = params; + + const modelClients: ModelClient[] = [primaryModel, ...backupModelClients]; + + async function* combinedGenerator(): AsyncIterable { + let lastErr: { error: unknown } | undefined = undefined; + + for (let i = 0; i < modelClients.length; i++) { + const currentModelClient = modelClients[i]; + + /* Local abort controller for this single attempt */ + const attemptAbort = new AbortController(); + if (callerAbort) { + if (callerAbort.aborted) { + // Already aborted, trigger immediately + attemptAbort.abort(); + } else { + callerAbort.addEventListener("abort", () => attemptAbort.abort(), { + once: true, + }); + } + } + + let errorFromCurrent: { error: unknown } | undefined = undefined; // set when onError fires + const providerId = currentModelClient.builtinProviderId; + if (providerId) { + llmErrorStore.clearModelError({ + model: currentModelClient.model.modelId, + provider: providerId, + }); + } + logger.info( + "Streaming text with model", + currentModelClient.model.modelId, + "provider", + currentModelClient.model.provider, + "builtinProviderId", + currentModelClient.builtinProviderId, + ); + const { textStream } = streamText({ + ...rest, + maxRetries: 0, + model: currentModelClient.model, + abortSignal: attemptAbort.signal, + onError: (error) => { + const providerId = currentModelClient.builtinProviderId; + if (providerId) { + llmErrorStore.recordModelError({ + model: currentModelClient.model.modelId, + provider: providerId, + }); + } + logger.error( + `Error streaming text with ${providerId} and model ${currentModelClient.model.modelId}: ${error}`, + error, + ); + errorFromCurrent = error; + attemptAbort.abort(); // kill fetch / SSE + }, + }); + + try { + for await (const chunk of textStream) { + /* If onError fired during streaming, bail out immediately. */ + if (errorFromCurrent) throw errorFromCurrent; + yield chunk; + } + + /* Stream ended – check if it actually failed */ + if (errorFromCurrent) throw errorFromCurrent; + + /* Completed successfully – stop trying more models. */ + return; + } catch (err) { + if (typeof err === "object" && err !== null && "error" in err) { + lastErr = err as { error: unknown }; + } else { + lastErr = { error: err }; + } + logger.warn( + `[streamTextWithBackup] model #${i} failed – ${ + i < modelClients.length - 1 + ? "switching to backup" + : "no backups left" + }`, + err, + ); + /* loop continues to next model (if any) */ + } + } + + /* Every model failed */ + if (!lastErr) { + throw new Error("Invariant in StreamTextWithbackup failed!"); + } + callerOnError?.(lastErr); + logger.error("All model invocations failed", lastErr); + // throw lastErr ?? new Error("All model invocations failed"); + } + + return { textStream: combinedGenerator() }; +} diff --git a/src/main/llm_error_store.ts b/src/main/llm_error_store.ts new file mode 100644 index 0000000..31fd4e6 --- /dev/null +++ b/src/main/llm_error_store.ts @@ -0,0 +1,35 @@ +class LlmErrorStore { + private modelErrorToTimestamp: Record = {}; + + constructor() {} + + recordModelError({ model, provider }: { model: string; provider: string }) { + this.modelErrorToTimestamp[this.getKey({ model, provider })] = Date.now(); + } + + clearModelError({ model, provider }: { model: string; provider: string }) { + delete this.modelErrorToTimestamp[this.getKey({ model, provider })]; + } + + modelHasNoRecentError({ + model, + provider, + }: { + model: string; + provider: string; + }): boolean { + const key = this.getKey({ model, provider }); + const timestamp = this.modelErrorToTimestamp[key]; + if (!timestamp) { + return true; + } + const oneHourAgo = Date.now() - 1000 * 60 * 60; + return timestamp < oneHourAgo; + } + + private getKey({ model, provider }: { model: string; provider: string }) { + return `${provider}::${model}`; + } +} + +export const llmErrorStore = new LlmErrorStore(); diff --git a/testing/fake-llm-server/README.md b/testing/fake-llm-server/README.md new file mode 100644 index 0000000..e15b684 --- /dev/null +++ b/testing/fake-llm-server/README.md @@ -0,0 +1,116 @@ +# Fake LLM Server + +A simple server that mimics the OpenAI streaming chat completions API for testing purposes. + +## Features + +- Implements a basic version of the OpenAI chat completions API +- Supports both streaming and non-streaming responses +- Always responds with "hello world" message +- Simulates a 429 rate limit error when the last message is "[429]" +- Configurable through environment variables + +## Installation + +```bash +npm install +``` + +## Usage + +Start the server: + +```bash +# Development mode +npm run dev + +# Production mode +npm run build +npm start +``` + +### Example usage + +``` +curl -X POST http://localhost:3500/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"messages":[{"role":"user","content":"Say something"}],"model":"any-model","stream":true}' +``` + +The server will be available at http://localhost:3500 by default. + +## API Endpoints + +### POST /v1/chat/completions + +This endpoint mimics OpenAI's chat completions API. + +#### Request Format + +```json +{ + "messages": [{ "role": "user", "content": "Your prompt here" }], + "model": "any-model", + "stream": true +} +``` + +- Set `stream: true` to receive a streaming response +- Set `stream: false` or omit it for a regular JSON response + +#### Response + +For non-streaming requests, you'll get a standard JSON response: + +```json +{ + "id": "chatcmpl-123456789", + "object": "chat.completion", + "created": 1699000000, + "model": "fake-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "hello world" + }, + "finish_reason": "stop" + } + ] +} +``` + +For streaming requests, you'll receive a series of server-sent events (SSE), each containing a chunk of the response. + +### Simulating Rate Limit Errors + +To test how your application handles rate limiting, send a message with content exactly equal to `[429]`: + +```json +{ + "messages": [{ "role": "user", "content": "[429]" }], + "model": "any-model" +} +``` + +This will return a 429 status code with the following response: + +```json +{ + "error": { + "message": "Too many requests. Please try again later.", + "type": "rate_limit_error", + "param": null, + "code": "rate_limit_exceeded" + } +} +``` + +## Configuration + +You can configure the server by modifying the `PORT` variable in the code. + +## Use Case + +This server is primarily intended for testing applications that integrate with OpenAI's API, allowing you to develop and test without making actual API calls to OpenAI. diff --git a/testing/fake-llm-server/dist/index.js b/testing/fake-llm-server/dist/index.js new file mode 100644 index 0000000..0b3bc49 --- /dev/null +++ b/testing/fake-llm-server/dist/index.js @@ -0,0 +1,90 @@ +"use strict"; +var __importDefault = + (this && this.__importDefault) || + function (mod) { + return mod && mod.__esModule ? mod : { default: mod }; + }; +Object.defineProperty(exports, "__esModule", { value: true }); +const express_1 = __importDefault(require("express")); +const http_1 = require("http"); +const cors_1 = __importDefault(require("cors")); +// Create Express app +const app = (0, express_1.default)(); +app.use((0, cors_1.default)()); +app.use(express_1.default.json()); +const PORT = 3500; +// Helper function to create OpenAI-like streaming response chunks +function createStreamChunk(content, role = "assistant", isLast = false) { + const chunk = { + id: `chatcmpl-${Date.now()}`, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: "fake-model", + choices: [ + { + index: 0, + delta: isLast ? {} : { content, role }, + finish_reason: isLast ? "stop" : null, + }, + ], + }; + return `data: ${JSON.stringify(chunk)}\n\n${isLast ? "data: [DONE]\n\n" : ""}`; +} +// Handle POST requests to /v1/chat/completions +app.post("/v1/chat/completions", (req, res) => { + const { stream = false } = req.body; + // Non-streaming response + if (!stream) { + return res.json({ + id: `chatcmpl-${Date.now()}`, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: "fake-model", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: "hello world", + }, + finish_reason: "stop", + }, + ], + }); + } + // Streaming response + res.setHeader("Content-Type", "text/event-stream"); + res.setHeader("Cache-Control", "no-cache"); + res.setHeader("Connection", "keep-alive"); + // Split the "hello world" message into characters to simulate streaming + const message = "hello world"; + const messageChars = message.split(""); + // Stream each character with a delay + let index = 0; + // Send role first + res.write(createStreamChunk("", "assistant")); + const interval = setInterval(() => { + if (index < messageChars.length) { + res.write(createStreamChunk(messageChars[index])); + index++; + } else { + // Send the final chunk + res.write(createStreamChunk("", "assistant", true)); + clearInterval(interval); + res.end(); + } + }, 100); +}); +// Start the server +const server = (0, http_1.createServer)(app); +server.listen(PORT, () => { + console.log(`Fake LLM server running on http://localhost:${PORT}`); +}); +// Handle SIGINT (Ctrl+C) +process.on("SIGINT", () => { + console.log("Shutting down fake LLM server"); + server.close(() => { + console.log("Server closed"); + process.exit(0); + }); +}); diff --git a/testing/fake-llm-server/index.ts b/testing/fake-llm-server/index.ts new file mode 100644 index 0000000..b1fa7c6 --- /dev/null +++ b/testing/fake-llm-server/index.ts @@ -0,0 +1,113 @@ +import express from "express"; +import { createServer } from "http"; +import cors from "cors"; + +// Create Express app +const app = express(); +app.use(cors()); +app.use(express.json()); + +const PORT = 3500; + +// Helper function to create OpenAI-like streaming response chunks +function createStreamChunk( + content: string, + role: string = "assistant", + isLast: boolean = false, +) { + const chunk = { + id: `chatcmpl-${Date.now()}`, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: "fake-model", + choices: [ + { + index: 0, + delta: isLast ? {} : { content, role }, + finish_reason: isLast ? "stop" : null, + }, + ], + }; + + return `data: ${JSON.stringify(chunk)}\n\n${isLast ? "data: [DONE]\n\n" : ""}`; +} + +// Handle POST requests to /v1/chat/completions +app.post("/v1/chat/completions", (req, res) => { + const { stream = false, messages = [] } = req.body; + + // Check if the last message contains "[429]" to simulate rate limiting + const lastMessage = messages[messages.length - 1]; + if (lastMessage && lastMessage.content === "[429]") { + return res.status(429).json({ + error: { + message: "Too many requests. Please try again later.", + type: "rate_limit_error", + param: null, + code: "rate_limit_exceeded", + }, + }); + } + + // Non-streaming response + if (!stream) { + return res.json({ + id: `chatcmpl-${Date.now()}`, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: "fake-model", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: "hello world", + }, + finish_reason: "stop", + }, + ], + }); + } + + // Streaming response + res.setHeader("Content-Type", "text/event-stream"); + res.setHeader("Cache-Control", "no-cache"); + res.setHeader("Connection", "keep-alive"); + + // Split the "hello world" message into characters to simulate streaming + const message = "hello world"; + const messageChars = message.split(""); + + // Stream each character with a delay + let index = 0; + + // Send role first + res.write(createStreamChunk("", "assistant")); + + const interval = setInterval(() => { + if (index < messageChars.length) { + res.write(createStreamChunk(messageChars[index])); + index++; + } else { + // Send the final chunk + res.write(createStreamChunk("", "assistant", true)); + clearInterval(interval); + res.end(); + } + }, 100); +}); + +// Start the server +const server = createServer(app); +server.listen(PORT, () => { + console.log(`Fake LLM server running on http://localhost:${PORT}`); +}); + +// Handle SIGINT (Ctrl+C) +process.on("SIGINT", () => { + console.log("Shutting down fake LLM server"); + server.close(() => { + console.log("Server closed"); + process.exit(0); + }); +}); diff --git a/testing/fake-llm-server/package-lock.json b/testing/fake-llm-server/package-lock.json new file mode 100644 index 0000000..ff460fd --- /dev/null +++ b/testing/fake-llm-server/package-lock.json @@ -0,0 +1,999 @@ +{ + "name": "fake-llm-server", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "fake-llm-server", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "cors": "^2.8.5", + "express": "^4.18.2", + "stream": "0.0.2" + }, + "devDependencies": { + "@types/cors": "^2.8.18", + "@types/express": "^4.17.21", + "@types/node": "^20.17.46", + "ts-node": "^10.9.2", + "typescript": "^5.8.3" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.11", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/body-parser": { + "version": "1.19.5", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/cors": { + "version": "2.8.18", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/express": { + "version": "4.17.21", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.19.6", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/http-errors": { + "version": "2.0.4", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/mime": { + "version": "1.3.5", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "20.17.46", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.19.2" + } + }, + "node_modules/@types/qs": { + "version": "6.9.18", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/send": { + "version": "0.17.4", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.7", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "*" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.14.1", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "dev": true, + "license": "MIT" + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "1.20.3", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.13.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.1", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "license": "MIT" + }, + "node_modules/cors": { + "version": "2.8.5", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/create-require": { + "version": "1.1.1", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "2.6.9", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/diff": { + "version": "4.0.2", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "license": "MIT" + }, + "node_modules/emitter-component": { + "version": "1.1.2", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.21.2", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.3", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.7.1", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.3.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.12", + "proxy-addr": "~2.0.7", + "qs": "6.13.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.19.0", + "serve-static": "1.16.2", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "1.3.1", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.0", + "license": "MIT", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "dev": true, + "license": "ISC" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.12", + "license": "MIT" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.13.0", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.2", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "license": "MIT" + }, + "node_modules/send": { + "version": "0.19.0", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.2", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.19.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.1", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/stream": { + "version": "0.0.2", + "license": "MIT", + "dependencies": { + "emitter-component": "^1.1.1" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/ts-node": { + "version": "10.9.2", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/typescript": { + "version": "5.8.3", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.19.8", + "dev": true, + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "dev": true, + "license": "MIT" + }, + "node_modules/vary": { + "version": "1.1.2", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/yn": { + "version": "3.1.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + } + } +} diff --git a/testing/fake-llm-server/package.json b/testing/fake-llm-server/package.json new file mode 100644 index 0000000..ab7cc63 --- /dev/null +++ b/testing/fake-llm-server/package.json @@ -0,0 +1,27 @@ +{ + "name": "fake-llm-server", + "version": "1.0.0", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "start": "node dist/index.js", + "dev": "ts-node index.ts", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "description": "Fake OpenAI API server for testing", + "dependencies": { + "cors": "^2.8.5", + "express": "^4.18.2", + "stream": "0.0.2" + }, + "devDependencies": { + "@types/cors": "^2.8.18", + "@types/express": "^4.17.21", + "@types/node": "^20.17.46", + "ts-node": "^10.9.2", + "typescript": "^5.8.3" + } +} diff --git a/testing/fake-llm-server/tsconfig.json b/testing/fake-llm-server/tsconfig.json new file mode 100644 index 0000000..d1da21a --- /dev/null +++ b/testing/fake-llm-server/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "outDir": "dist", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["*.ts"], + "exclude": ["node_modules"] +}