Set explicit max output tokens to avoid truncated responses (#31)

src/ipc/handlers/chat_stream_handlers.ts

@@ -15,7 +15,7 @@ import { extractCodebase } from "../../utils/codebase";
 import { processFullResponseActions } from "../processors/response_processor";
 import { streamTestResponse } from "./testing_chat_handlers";
 import { getTestResponse } from "./testing_chat_handlers";
-import { getModelClient } from "../utils/get_model_client";
+import { getMaxTokens, getModelClient } from "../utils/get_model_client";
 import log from "electron-log";
 import {
   getSupabaseContext,
@@ -165,9 +165,8 @@ export function registerChatStreamHandlers() {
         } else {
           systemPrompt += "\n\n" + SUPABASE_NOT_AVAILABLE_SYSTEM_PROMPT;
         }
-
         const { textStream } = streamText({
-          maxTokens: 8_000,
+          maxTokens: getMaxTokens(settings.selectedModel),
           temperature: 0,
           model: modelClient,
           system: systemPrompt,

src/ipc/utils/get_model_client.ts

@@ -9,6 +9,7 @@ import {
   PROVIDER_TO_ENV_VAR,
   AUTO_MODELS,
   PROVIDERS,
+  MODEL_OPTIONS,
 } from "../../constants/models";
 import { getEnvVar } from "./read_env";
 import log from "electron-log";
@@ -27,7 +28,7 @@ export function getModelClient(
         getEnvVar(PROVIDER_TO_ENV_VAR[autoModel.provider]);
 
       if (apiKey) {
-        console.log(
+        logger.log(
           `Using provider: ${autoModel.provider} model: ${autoModel.name}`
         );
         // Use the first model that has an API key
@@ -89,3 +90,19 @@ export function getModelClient(
     }
   }
 }
+
+// Most models support at least 8000 output tokens so we use it as a default value.
+const DEFAULT_MAX_TOKENS = 8_000;
+
+export function getMaxTokens(model: LargeLanguageModel) {
+  if (!MODEL_OPTIONS[model.provider as keyof typeof MODEL_OPTIONS]) {
+    logger.warn(
+      `Model provider ${model.provider} not found in MODEL_OPTIONS. Using default max tokens.`
+    );
+    return DEFAULT_MAX_TOKENS;
+  }
+  const modelOption = MODEL_OPTIONS[
+    model.provider as keyof typeof MODEL_OPTIONS
+  ].find((m) => m.name === model.name);
+  return modelOption?.maxOutputTokens || DEFAULT_MAX_TOKENS;
+}