Merge_PR_418_refine_hf_fallback_policy_and_sif_low_cost_routing

backend/services/llm_providers/huggingface_provider.py

@@ -49,8 +49,12 @@ Last Updated: January 2025
 import os
 import json
 import re
+<<<<<<< HEAD
 from functools import lru_cache
 from typing import Optional, Dict, Any
+=======
+from typing import Optional, Dict, Any, List, Iterable
+>>>>>>> pr-418
 
 from loguru import logger
 from utils.logger_utils import get_service_logger
@@ -92,7 +96,7 @@ HF_FALLBACK_MODELS = [
 ]
 
 
-def _candidate_model_variants(model: str):
+def _candidate_model_variants(model: str, allow_model_variant_fallback: bool = True):
     """Yield model ids to try for a single logical model preference."""
     if not model:
         return
@@ -101,12 +105,13 @@ def _candidate_model_variants(model: str):
     yield model
 
     # Fallback to base repo id when provider suffix is not recognized by the router
-    if ":" in model:
+    if allow_model_variant_fallback and ":" in model:
         base_model = model.split(":", 1)[0]
         if base_model:
             yield base_model
 
 
+<<<<<<< HEAD
 def _fallback_model_sequence(model: str, fallback_models: Optional[List[str]] = None):
     # IMPORTANT: Do not apply implicit global fallback chains.
     # Callers must explicitly provide fallback_models when they want multi-model retries.
@@ -114,9 +119,27 @@ def _fallback_model_sequence(model: str, fallback_models: Optional[List[str]] =
         sequence = [model] + fallback_models
     else:
         sequence = [model]
+=======
+def _fallback_model_sequence(
+    model: str,
+    fallback_models: Optional[List[str]] = None,
+    allow_model_variant_fallback: bool = True,
+):
+    sequence: Iterable[str]
+    if fallback_models is None:
+        # Safe default only when caller doesn't provide explicit policy.
+        sequence = [model] + HF_FALLBACK_MODELS
+    else:
+        # Caller owns fallback policy fully. Empty list means only requested model.
+        sequence = [model] + list(fallback_models)
+
+>>>>>>> pr-418
     seen = set()
     for preferred_model in sequence:
-        for candidate in _candidate_model_variants(preferred_model):
+        for candidate in _candidate_model_variants(
+            preferred_model,
+            allow_model_variant_fallback=allow_model_variant_fallback,
+        ):
             if candidate and candidate not in seen:
                 seen.add(candidate)
                 yield candidate
@@ -237,7 +260,12 @@ def huggingface_text_response(
     max_tokens: int = 2048,
     top_p: float = 0.9,
     system_prompt: Optional[str] = None,
+<<<<<<< HEAD
     api_key: Optional[str] = None,
+=======
+    fallback_models: Optional[List[str]] = None,
+    allow_model_variant_fallback: bool = True,
+>>>>>>> pr-418
 ) -> str:
     """
     Generate text response using Hugging Face Inference Providers API.
@@ -333,6 +361,7 @@ def huggingface_text_response(
         import time
         time.sleep(1)  # 1 second delay between API calls
         
+<<<<<<< HEAD
         # Call exactly the requested model; no retries, no fallbacks, no variants
 =======
 >>>>>>> pr-416
@@ -343,6 +372,33 @@ def huggingface_text_response(
             top_p=top_p,
             max_tokens=max_tokens
         )
+=======
+        response = None
+        last_error = None
+        for candidate_model in _fallback_model_sequence(
+            model=model,
+            fallback_models=fallback_models,
+            allow_model_variant_fallback=allow_model_variant_fallback,
+        ):
+            try:
+                response = client.chat.completions.create(
+                    model=candidate_model,
+                    messages=messages,
+                    temperature=temperature,
+                    top_p=top_p,
+                    max_tokens=max_tokens
+                )
+                if candidate_model != model:
+                    logger.warning("HF text generation switched to fallback model: {}", candidate_model)
+                break
+            except NotFoundError as nf_err:
+                last_error = nf_err
+                logger.warning("HF model not found: {}. Trying fallback model.", candidate_model)
+                continue
+
+        if response is None:
+            raise last_error or Exception("Hugging Face text generation failed: all fallback models failed")
+>>>>>>> pr-418
         
         # Extract text from response
         generated_text = response.choices[0].message.content
@@ -394,7 +450,12 @@ def huggingface_structured_json_response(
     temperature: float = 0.7,
     max_tokens: int = 8192,
     system_prompt: Optional[str] = None,
+<<<<<<< HEAD
     api_key: Optional[str] = None,
+=======
+    fallback_models: Optional[List[str]] = None,
+    allow_model_variant_fallback: bool = True,
+>>>>>>> pr-418
 ) -> Dict[str, Any]:
     """
     Generate structured JSON response using Hugging Face Inference Providers API.
@@ -505,7 +566,15 @@ def huggingface_structured_json_response(
 <<<<<<< HEAD
             response = None
             last_error = None
+<<<<<<< HEAD
             for candidate_model in _fallback_model_sequence(model, fallback_models):
+=======
+            for candidate_model in _fallback_model_sequence(
+                model=model,
+                fallback_models=fallback_models,
+                allow_model_variant_fallback=allow_model_variant_fallback,
+            ):
+>>>>>>> pr-418
                 try:
                     response = client.chat.completions.create(
                         model=candidate_model,
@@ -562,7 +631,15 @@ def huggingface_structured_json_response(
                 logger.info("Retrying without response_format...")
                 response = None
                 last_error = None
+<<<<<<< HEAD
                 for candidate_model in _fallback_model_sequence(model, fallback_models):
+=======
+                for candidate_model in _fallback_model_sequence(
+                    model=model,
+                    fallback_models=fallback_models,
+                    allow_model_variant_fallback=allow_model_variant_fallback,
+                ):
+>>>>>>> pr-418
                     try:
                         response = client.chat.completions.create(
                             model=candidate_model,