kunthawat/ALwrity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge_PR_416_fix_textgen_ai_models_mapping

Browse Source
This commit is contained in:
ajaysi
2026-03-12 16:05:47 +05:30
parent acc5e1f72c 4b7f443509
commit 8d421a158f
5 changed files with 509 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

304
backend/services/llm_providers/main_text_generation.py
View File

@@ -10,10 +10,124 @@ from typing import Optional, Dict, Any, List
from datetime import datetime
from loguru import logger
from fastapi import HTTPException
from ..onboarding.api_key_manager import APIKeyManager
from .gemini_provider import gemini_text_response, gemini_structured_json_response
from .huggingface_provider import huggingface_text_response, huggingface_structured_json_response
from .tenant_provider_config import get_available_text_providers, get_tenant_api_key
from .routing_observability import emit_routing_event
def _normalize_provider(provider: Optional[str]) -> Optional[str]:
if not provider:
return None
provider_aliases = {
"gemini": "google",
"google": "google",
"hf": "huggingface",
"hf_response_api": "huggingface",
"huggingface": "huggingface",
"wavespeed": "huggingface",
}
value = str(provider).strip().lower()
return provider_aliases.get(value, value)
def _parse_csv_env(value: Optional[str]) -> List[str]:
if not value:
return []
return [v.strip() for v in str(value).split(",") if v.strip()]
def _resolve_provider_sequence(
preferred_provider: Optional[str],
env_provider_raw: str,
available_providers: List[str],
) -> List[str]:
configured = _parse_csv_env(preferred_provider) if preferred_provider else _parse_csv_env(env_provider_raw)
normalized = [_normalize_provider(p) for p in configured if _normalize_provider(p)]
if not normalized:
if "google" in available_providers:
return ["google"]
if "huggingface" in available_providers:
return ["huggingface"]
return []
# preserve order and keep only available providers
sequence = []
for provider in normalized:
if provider in available_providers:
sequence.append(provider)
# strict mode for single configured provider: no silent remap
if len(normalized) == 1:
return sequence
# multi-provider mode: append any other available providers as tail only if none configured are available
if not sequence:
return [p for p in ["huggingface", "google"] if p in available_providers]
return sequence
def _map_logical_model_to_provider_model(provider: str, model_name: str) -> str:
"""Map logical model aliases/full names to provider-specific model IDs."""
raw = (model_name or "").strip()
if not raw:
return raw
# Full provider path supplied explicitly; use as-is.
if "/" in raw:
return raw
key = raw.lower()
hf_map = {
"gpt-oss": "openai/gpt-oss-120b:cerebras",
"gpt-oss-120b": "openai/gpt-oss-120b:cerebras",
"gpt-oss-20b": "openai/gpt-oss-20b:cerebras",
"mistral": "mistralai/Mistral-7B-Instruct-v0.3:cerebras",
"mistral-7b": "mistralai/Mistral-7B-Instruct-v0.3:cerebras",
"llama": "meta-llama/Llama-3.1-8B-Instruct:groq",
"llama-8b": "meta-llama/Llama-3.1-8B-Instruct:groq",
"llama-70b": "meta-llama/Llama-3.1-70B-Instruct:groq",
}
wavespeed_map = {
"gpt-oss": "openai/gpt-oss-120b",
"gpt-oss-120b": "openai/gpt-oss-120b",
"gpt-oss-20b": "openai/gpt-oss-20b",
"mistral": "mistralai/Mistral-7B-Instruct-v0.3",
"mistral-7b": "mistralai/Mistral-7B-Instruct-v0.3",
"llama": "meta-llama/Llama-3.1-8B-Instruct",
"llama-8b": "meta-llama/Llama-3.1-8B-Instruct",
"llama-70b": "meta-llama/Llama-3.1-70B-Instruct",
}
if provider in {"huggingface", "hf", "hf_response_api"}:
return hf_map.get(key, raw)
if provider == "wavespeed":
return wavespeed_map.get(key, raw)
return raw
def _resolve_model_sequence(provider: str, preferred_hf_models: Optional[List[str]] = None) -> List[str]:
models_env = _parse_csv_env(os.getenv("TEXTGEN_AI_MODELS", ""))
if provider == "google":
return ["gemini-2.0-flash-001"]
if preferred_hf_models:
return [_map_logical_model_to_provider_model(provider, m) for m in preferred_hf_models if m]
if not models_env:
return ["openai/gpt-oss-120b:groq"]
resolved = [_map_logical_model_to_provider_model(provider, m) for m in models_env if m.strip()]
return resolved or ["openai/gpt-oss-120b:groq"]
def llm_text_gen(
@@ -23,7 +137,11 @@ def llm_text_gen(
user_id: str = None,
preferred_hf_models: Optional[List[str]] = None,
preferred_provider: Optional[str] = None,
<<<<<<< HEAD
flow_type: Optional[str] = None,
=======
flow_type: str = "default",
>>>>>>> pr-416
) -> str:
"""
Generate text using Language Model (LLM) based on the provided prompt.
@@ -49,12 +167,18 @@ def llm_text_gen(
logger.debug(f"[llm_text_gen] Prompt length: {len(prompt)} characters")
# Set default values for LLM parameters
<<<<<<< HEAD
gpt_provider = "huggingface" # Default to premium HF route for ALwrity AI tools
model = "openai/gpt-oss-120b:cerebras"
=======
gpt_provider = "google"
model = "gemini-2.0-flash-001"
>>>>>>> pr-416
temperature = 0.7
max_tokens = 4000
top_p = 0.9
n = 1
<<<<<<< HEAD
fp = 16
frequency_penalty = 0.0
presence_penalty = 0.0
@@ -143,6 +267,13 @@ def llm_text_gen(
elif gpt_provider == "google":
model = "gemini-2.0-flash-001" # Google has fewer options
=======
env_provider_raw = os.getenv('GPT_PROVIDER', '').lower()
env_provider = _normalize_provider(env_provider_raw)
preferred_provider_normalized = _normalize_provider(preferred_provider)
>>>>>>> pr-416
# Default blog characteristics
blog_tone = "Professional"
blog_demographic = "Professional"
@@ -151,6 +282,7 @@ def llm_text_gen(
blog_output_format = "markdown"
blog_length = 2000
<<<<<<< HEAD
# Check which providers have API keys available using APIKeyManager
api_key_manager = APIKeyManager()
available_providers = []
@@ -230,12 +362,47 @@ def llm_text_gen(
model = "openai/gpt-oss-120b"
else:
raise RuntimeError("No supported providers available.")
=======
available_providers = get_available_text_providers(user_id)
provider_sequence = _resolve_provider_sequence(preferred_provider, env_provider_raw, available_providers)
>>>>>>> pr-416
if gpt_provider == "huggingface" and preferred_hf_models:
model = preferred_hf_models[0]
logger.info(f"[llm_text_gen] Using preferred low-cost HF model: {model}")
if not provider_sequence:
logger.error("[llm_text_gen] No configured providers available for tenant.")
raise RuntimeError("No LLM providers available for tenant.")
# strict mode if single configured provider; multi-provider fallback if comma-separated providers
pinned_provider = len(_parse_csv_env(preferred_provider or env_provider_raw)) == 1 and bool(preferred_provider or env_provider_raw)
gpt_provider = provider_sequence[0]
model_sequence = _resolve_model_sequence(gpt_provider, preferred_hf_models)
model = model_sequence[0]
hf_api_key = get_tenant_api_key(user_id, "huggingface") if gpt_provider == "huggingface" else None
logger.info(
"[llm_text_gen] Mode | providers={} | models={} | env_models={} | strict_provider={} | strict_model={}",
provider_sequence,
model_sequence,
_parse_csv_env(os.getenv("TEXTGEN_AI_MODELS", "")),
pinned_provider,
len(model_sequence) == 1,
)
<<<<<<< HEAD
logger.info(f"[llm_text_gen][{flow_tag}] Using provider={gpt_provider}, model={model}")
=======
logger.debug(f"[llm_text_gen] Using provider: {gpt_provider}, model: {model}")
emit_routing_event(
logger,
"text_route_selected",
user_id=user_id,
flow_type=flow_type,
provider_selected=gpt_provider,
model_selected=model,
env_provider=env_provider_raw or "auto",
fallback_count=0,
)
>>>>>>> pr-416
# Map provider name to APIProvider enum (define at function scope for usage tracking)
from models.subscription_models import APIProvider
@@ -291,6 +458,13 @@ def llm_text_gen(
estimated_output_tokens = int(input_tokens * 1.5)
estimated_total_tokens = input_tokens + estimated_output_tokens
logger.info(
"[llm_text_gen][subscription_preflight] start | user_id={} | provider={} | tokens_requested={}",
user_id,
actual_provider_name or provider_enum.value,
estimated_total_tokens,
)
# Check limits using sync method from pricing service (strict enforcement)
can_proceed, message, usage_info = pricing_service.check_usage_limits(
user_id=user_id,
@@ -315,7 +489,14 @@ def llm_text_gen(
'usage_info': usage_info if usage_info else {}
}
raise HTTPException(status_code=429, detail=error_detail)
logger.info(
"[llm_text_gen][subscription_preflight] pass | user_id={} | provider={} | tokens_requested={}",
user_id,
actual_provider_name or provider_enum.value,
estimated_total_tokens,
)
# Get current usage for limit checking only
current_period = pricing_service.get_current_billing_period(user_id) or datetime.now().strftime("%Y-%m")
usage = db.query(UsageSummary).filter(
@@ -361,6 +542,7 @@ def llm_text_gen(
else:
system_instructions = system_prompt
<<<<<<< HEAD
# HF behavior: fail fast on selected model; no intra-provider model fallback chain.
hf_fallback_models: List[str] = []
@@ -463,23 +645,27 @@ def llm_text_gen(
logger.info(
f"[llm_text_gen][{flow_tag}] ✅ API call successful, tracking usage for user {user_id}, provider {provider_enum.value}"
)
=======
# Generate response based on provider/model sequence
response_text = None
errors: List[str] = []
for provider_idx, provider_name in enumerate(provider_sequence):
candidate_models = _resolve_model_sequence(provider_name, preferred_hf_models)
for model_idx, candidate_model in enumerate(candidate_models):
>>>>>>> pr-416
try:
from services.intelligence.agents.agent_usage_tracking import track_agent_usage_sync
# Estimate tokens
tokens_input = int(len(prompt.split()) * 1.3)
# Calculate duration (mocking it since we didn't track start time explicitly in this function)
# Ideally we should track start_time at beginning of function
duration = 0.5
track_agent_usage_sync(
emit_routing_event(
logger,
"text_route_attempt",
user_id=user_id,
model_name=model,
prompt=prompt,
response_text=response_text,
duration=duration
flow_type=flow_type,
provider_selected=provider_name,
model_selected=candidate_model,
provider_attempt=provider_idx + 1,
model_attempt=model_idx + 1,
)
<<<<<<< HEAD
except Exception as usage_error:
# Non-blocking: log error but don't fail the request
@@ -535,6 +721,10 @@ def llm_text_gen(
fallback_model = "openai/gpt-oss-120b"
if fallback_provider == "google":
=======
if provider_name == "google":
>>>>>>> pr-416
if json_struct:
response_text = gemini_structured_json_response(
prompt=prompt,
@@ -543,7 +733,7 @@ def llm_text_gen(
top_p=top_p,
top_k=n,
max_tokens=max_tokens,
system_prompt=system_instructions
system_prompt=system_instructions,
)
else:
response_text = gemini_text_response(
@@ -552,22 +742,29 @@ def llm_text_gen(
top_p=top_p,
n=n,
max_tokens=max_tokens,
system_prompt=system_instructions
system_prompt=system_instructions,
)
elif fallback_provider == "huggingface":
elif provider_name == "huggingface":
hf_api_key_current = get_tenant_api_key(user_id, "huggingface")
if json_struct:
response_text = huggingface_structured_json_response(
prompt=prompt,
schema=json_struct,
<<<<<<< HEAD
model=fallback_model,
fallback_models=hf_fallback_models,
=======
model=candidate_model,
>>>>>>> pr-416
temperature=temperature,
max_tokens=max_tokens,
system_prompt=system_instructions
system_prompt=system_instructions,
api_key=hf_api_key_current,
)
else:
response_text = huggingface_text_response(
prompt=prompt,
<<<<<<< HEAD
model=fallback_model,
fallback_models=hf_fallback_models,
temperature=temperature,
@@ -592,31 +789,37 @@ def llm_text_gen(
prompt=prompt,
model=fallback_model,
fallback_models=None,
=======
model=candidate_model,
>>>>>>> pr-416
temperature=temperature,
max_tokens=max_tokens,
top_p=top_p,
system_prompt=system_instructions
system_prompt=system_instructions,
api_key=hf_api_key_current,
)
# TRACK USAGE after successful fallback call
else:
raise RuntimeError(f"Unknown provider {provider_name}")
if response_text:
<<<<<<< HEAD
logger.info(
f"[llm_text_gen][{flow_tag}] ✅ Fallback API call successful, tracking usage for user {user_id}, provider {provider_enum.value}"
)
=======
logger.info(f"[llm_text_gen] ✅ API call successful, tracking usage for user {user_id}, provider {provider_enum.value}")
>>>>>>> pr-416
try:
from services.intelligence.agents.agent_usage_tracking import track_agent_usage_sync
# Estimate tokens
tokens_input = int(len(prompt.split()) * 1.3)
track_agent_usage_sync(
user_id=user_id,
model_name=fallback_model,
model_name=candidate_model,
prompt=prompt,
response_text=response_text,
duration=0.5 # Approximate duration
duration=0.5,
)
except Exception as usage_error:
<<<<<<< HEAD
logger.error(f"[llm_text_gen] ❌ Failed to track fallback usage: {usage_error}", exc_info=True)
return response_text
@@ -626,6 +829,22 @@ def llm_text_gen(
# CIRCUIT BREAKER: Stop immediately to prevent expensive API calls
logger.error(f"[llm_text_gen][{flow_tag}] CIRCUIT BREAKER: Stopping to prevent expensive API calls.")
raise RuntimeError("All LLM providers failed to generate a response.")
=======
logger.error(f"[llm_text_gen] ❌ Failed to track usage: {usage_error}", exc_info=True)
return response_text
except Exception as provider_error:
err = f"provider={provider_name},model={candidate_model},error={provider_error}"
errors.append(err)
logger.error("[llm_text_gen] Attempt failed: {}", err)
continue
# strict provider mode: single configured provider should not switch
if pinned_provider and len(provider_sequence) == 1:
break
logger.error("[llm_text_gen] CIRCUIT BREAKER: All configured provider/model attempts failed. {}", errors)
raise RuntimeError("All configured LLM provider/model attempts failed.")
>>>>>>> pr-416
except Exception as e:
logger.error(f"[llm_text_gen][{flow_tag}] Error during text generation: {str(e)}")
@@ -633,12 +852,21 @@ def llm_text_gen(
def check_gpt_provider(gpt_provider: str) -> bool:
"""Check if the specified GPT provider is supported."""
<<<<<<< HEAD
supported_providers = ["google", "huggingface", "wavespeed"]
return gpt_provider in supported_providers
=======
providers = [_normalize_provider(p) for p in _parse_csv_env(gpt_provider)]
if not providers:
providers = [_normalize_provider(gpt_provider)]
supported_providers = {"google", "huggingface"}
return all(p in supported_providers for p in providers if p)
>>>>>>> pr-416
def get_api_key(gpt_provider: str) -> Optional[str]:
"""Get API key for the specified provider."""
def get_api_key(gpt_provider: str, user_id: Optional[str] = None) -> Optional[str]:
"""Get API key for the specified provider, preferring tenant-scoped keys."""
try:
<<<<<<< HEAD
api_key_manager = APIKeyManager()
provider_mapping = {
"google": "gemini",
@@ -648,6 +876,10 @@ def get_api_key(gpt_provider: str) -> Optional[str]:
mapped_provider = provider_mapping.get(gpt_provider, gpt_provider)
return api_key_manager.get_api_key(mapped_provider)
=======
return get_tenant_api_key(user_id, gpt_provider)
>>>>>>> pr-416
except Exception as e:
logger.error(f"[get_api_key] Error getting API key for {gpt_provider}: {str(e)}")
return None
return None