kunthawat/ALwrity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Brainstorm Topics with GSC + Issue #518 fixes + Blog Editor enhancements

Browse Source 
Issue #518 - Subscription not updating after checkout:
- Fix stale closure in SubscriptionContext checkout polling (use subscriptionRef)
- Move checkout success polling from InitialRouteHandler into SubscriptionContext
- Remove redundant polling code from InitialRouteHandler
- Fix plan label: 'Free' instead of 'No Plan', proper capitalization
- Add plan refresh button in UserBadge
- Add 'View Costing Details' to UserBadge dropdown
- Rename 'ALwrity Podcast Maker' to 'Podcast Creator' across UI
- Clean subscription=success URL param after verification

Blog Writer WYSIWYG Editor enhancements:
- Per-section preview toggle (view/edit icons)
- Enhanced hover-based toolbar
- Circular SVG progress stats bar with detailed tooltip
- Research tool chips in stats bar footer
- Per-section TTS with useTextToSpeech hook (browser native)
- Full blog preview modal with print/PDF support
- PlayAllTTSButton: sequential playback with progress bar
- OnThisPageNav: floating sidebar with scroll tracking
- Section data attributes for scroll anchoring

GSC Brainstorm Topics feature:
- Backend: gsc_brainstorm_service.py (rule-based + LLM recommendations)
- Backend: POST /gsc/brainstorm endpoint with 3-word minimum validation
- Frontend: gscBrainstorm.ts API client
- Frontend: useGSCBrainstormConnection hook (popup OAuth, no /onboarding redirect)
- Frontend: useGSCBrainstorm hook (connect check + brainstorm call)
- Frontend: GSCBrainstormModal (3-tab results: Opportunities, Gaps, AI Recs)
- Frontend: BrainstormButton (visible at 3+ words, GSC connect overlay)
- Wire BrainstormButton into ManualResearchForm and ResearchAction
- Add blog_writer to gsc_auth router features for ALWRITY_ENABLED_FEATURES
This commit is contained in:
ajaysi
2026-05-20 22:34:37 +05:30
parent 68190dedb3
commit 644e72d289
98 changed files with 16137 additions and 2501 deletions
Download Patch File Download Diff File Expand all files Collapse all files

583
backend/services/hallucination_detector.py
View File

@@ -1,9 +1,9 @@
"""
Hallucination Detector Service
This service implements fact-checking functionality using Exa.ai API
to detect and verify claims in AI-generated content, similar to the
Exa.ai demo implementation.
Implements fact-checking using Exa.ai for evidence search and the
configured LLM provider (via GPT_PROVIDER) for claim extraction and assessment.
Respects GPT_PROVIDER env var: google, wavespeed, openai, huggingface.
"""
import json
@@ -11,15 +11,9 @@ import logging
from typing import List, Dict, Any, Optional
from dataclasses import dataclass
from datetime import datetime
import requests
import os
import asyncio
import concurrent.futures
try:
from google import genai
GOOGLE_GENAI_AVAILABLE = True
except Exception:
GOOGLE_GENAI_AVAILABLE = False
logger = logging.getLogger(__name__)
@@ -44,70 +38,121 @@ class HallucinationResult:
insufficient_claims: int
timestamp: str
def _get_llm_provider_info() -> Dict[str, str]:
"""Determine the LLM provider from GPT_PROVIDER env var."""
provider_env = os.getenv('GPT_PROVIDER', 'google').lower().strip()
provider = provider_env.split(',')[0].strip() if provider_env else 'google'
if provider in ('wavespeed', 'wave'):
return {'provider': 'wavespeed', 'name': 'WaveSpeed'}
elif provider in ('gemini', 'google'):
return {'provider': 'google', 'name': 'Gemini'}
elif provider in ('openai', 'gpt'):
return {'provider': 'openai', 'name': 'OpenAI'}
elif provider in ('hf_response_api', 'huggingface', 'hf'):
return {'provider': 'huggingface', 'name': 'HuggingFace'}
else:
return {'provider': provider, 'name': provider.capitalize()}
class HallucinationDetector:
"""
Hallucination detector using Exa.ai for fact-checking.
Implements the three-step process from Exa.ai demo:
Hallucination detector using Exa.ai for evidence search
and the configured LLM provider (GPT_PROVIDER) for claim extraction/assessment.
Implements the three-step process:
1. Extract verifiable claims from text
2. Search for evidence using Exa.ai
3. Verify claims against sources
"""
def __init__(self):
self.exa_api_key = os.getenv('EXA_API_KEY')
self.gemini_api_key = os.getenv('GEMINI_API_KEY')
if not self.exa_api_key:
logger.warning("EXA_API_KEY not found. Hallucination detection will be limited.")
if not self.gemini_api_key:
logger.warning("GEMINI_API_KEY not found. Falling back to heuristic claim extraction.")
# Initialize Gemini client for claim extraction and assessment
self.gemini_client = genai.Client(api_key=self.gemini_api_key) if (GOOGLE_GENAI_AVAILABLE and self.gemini_api_key) else None
# Rate limiting to prevent API abuse
self._llm_provider_info = _get_llm_provider_info()
# Check that at least one LLM key is available for the configured provider
self._check_provider_keys()
# Rate limiting
self.daily_api_calls = 0
self.daily_limit = 20 # Max 20 API calls per day for fact checking
self.daily_limit = 20
self.last_reset_date = None
def _check_provider_keys(self):
"""Check that API keys for the configured provider are available."""
provider = self._llm_provider_info['provider']
if provider == 'google':
key = os.getenv('GEMINI_API_KEY')
if not key:
logger.warning(f"GEMINI_API_KEY not found. Hallucination detection will fail for provider '{provider}'.")
elif provider == 'wavespeed':
key = os.getenv('WAVESPEED_API_KEY')
if not key:
logger.warning(f"WAVESPEED_API_KEY not found. Hallucination detection will fail for provider '{provider}'.")
elif provider == 'openai':
key = os.getenv('OPENAI_API_KEY')
if not key:
logger.warning(f"OPENAI_API_KEY not found. Hallucination detection will fail for provider '{provider}'.")
# huggingface uses serverless endpoint or HF token
@property
def provider_name(self) -> str:
return self._llm_provider_info['name']
@property
def provider_key(self) -> str:
return self._llm_provider_info['provider']
def _check_rate_limit(self) -> bool:
"""Check if we're within daily API usage limits."""
from datetime import date
today = date.today()
# Reset counter if it's a new day
if self.last_reset_date != today:
self.daily_api_calls = 0
self.last_reset_date = today
# Check if we've exceeded the limit
if self.daily_api_calls >= self.daily_limit:
logger.warning(f"Daily API limit reached ({self.daily_limit} calls). Fact checking disabled for today.")
return False
# Increment counter for this API call
self.daily_api_calls += 1
logger.info(f"Fact check API call #{self.daily_api_calls}/{self.daily_limit} today")
return True
async def detect_hallucinations(self, text: str) -> HallucinationResult:
def _generate_text(self, prompt: str, system_prompt: Optional[str] = None, user_id: str = None) -> str:
"""Generate text using the configured LLM provider (respects GPT_PROVIDER)."""
from services.llm_providers.main_text_generation import llm_text_gen
result = llm_text_gen(
prompt=prompt,
system_prompt=system_prompt or "You are a precise fact-checking assistant. Respond only with valid JSON as instructed.",
max_tokens=4000,
user_id=user_id,
)
return result
async def _generate_text_async(self, prompt: str, system_prompt: Optional[str] = None, user_id: str = None) -> str:
"""Async wrapper for _generate_text."""
loop = asyncio.get_event_loop()
with concurrent.futures.ThreadPoolExecutor() as executor:
result = await loop.run_in_executor(
executor,
lambda: self._generate_text(prompt, system_prompt, user_id)
)
return result
async def detect_hallucinations(self, text: str, user_id: str = None) -> HallucinationResult:
"""
Main method to detect hallucinations in the given text.
Args:
text: The text to analyze for factual accuracy
Returns:
HallucinationResult with claims analysis and confidence scores
"""
try:
logger.info(f"Starting hallucination detection for text of length: {len(text)}")
logger.info(f"Text sample: {text[:200]}...")
# Check rate limits first
if not self._check_rate_limit():
return HallucinationResult(
claims=[],
@@ -118,17 +163,11 @@ class HallucinationDetector:
insufficient_claims=0,
timestamp=datetime.now().isoformat()
)
# Validate required API keys
if not self.gemini_api_key:
raise Exception("GEMINI_API_KEY not configured. Cannot perform hallucination detection.")
if not self.exa_api_key:
raise Exception("EXA_API_KEY not configured. Cannot search for evidence.")
# Step 1: Extract claims from text
claims_texts = await self._extract_claims(text)
claims_texts = await self._extract_claims(text, user_id=user_id)
logger.info(f"Extracted {len(claims_texts)} claims from text: {claims_texts}")
if not claims_texts:
logger.warning("No verifiable claims found in text")
return HallucinationResult(
@@ -140,22 +179,18 @@ class HallucinationDetector:
insufficient_claims=0,
timestamp=datetime.now().isoformat()
)
# Step 2 & 3: Verify claims in batch to reduce API calls
verified_claims = await self._verify_claims_batch(claims_texts)
# Step 2 & 3: Verify claims in batch
verified_claims = await self._verify_claims_batch(claims_texts, user_id=user_id)
# Calculate overall metrics
total_claims = len(verified_claims)
supported_claims = sum(1 for c in verified_claims if c.assessment == "supported")
refuted_claims = sum(1 for c in verified_claims if c.assessment == "refuted")
insufficient_claims = sum(1 for c in verified_claims if c.assessment == "insufficient_information")
# Calculate overall confidence (weighted average)
if total_claims > 0:
overall_confidence = sum(c.confidence for c in verified_claims) / total_claims
else:
overall_confidence = 0.0
overall_confidence = sum(c.confidence for c in verified_claims) / total_claims if total_claims > 0 else 0.0
result = HallucinationResult(
claims=verified_claims,
overall_confidence=overall_confidence,
@@ -165,120 +200,67 @@ class HallucinationDetector:
insufficient_claims=insufficient_claims,
timestamp=datetime.now().isoformat()
)
logger.info(f"Hallucination detection completed. Overall confidence: {overall_confidence:.2f}")
return result
except Exception as e:
logger.error(f"Error in hallucination detection: {str(e)}")
raise Exception(f"Hallucination detection failed: {str(e)}")
async def _extract_claims(self, text: str) -> List[str]:
"""
Extract verifiable claims from text using LLM.
Args:
text: Input text to extract claims from
Returns:
List of claim strings
"""
if not self.gemini_client:
raise Exception("Gemini client not available. Cannot extract claims without AI provider.")
async def _extract_claims(self, text: str, user_id: str = None) -> List[str]:
"""Extract verifiable claims from text using LLM."""
try:
prompt = (
"Extract verifiable factual claims from the following text. "
"A verifiable claim is a statement that can be checked against external sources for accuracy.\n\n"
"Return ONLY a valid JSON array of strings, where each string is a single verifiable claim.\n\n"
"Examples of GOOD verifiable claims:\n"
"- \"The company was founded in 2020\"\n"
"- \"Sales increased by 25% last quarter\"\n"
"- \"The product has 10,000 users\"\n"
"- \"The market size is $50 billion\"\n"
"- \"The software supports 15 languages\"\n"
"- \"The company has offices in 5 countries\"\n\n"
'- "The company was founded in 2020"\n'
'- "Sales increased by 25% last quarter"\n'
'- "The product has 10,000 users"\n\n'
"Examples of BAD claims (opinions, subjective statements):\n"
"- \"This is the best product\"\n"
"- \"Customers love our service\"\n"
"- \"We are innovative\"\n"
"- \"The future looks bright\"\n\n"
'- "This is the best product"\n'
'- "Customers love our service"\n\n'
"IMPORTANT: Extract at least 2-3 verifiable claims if possible. "
"Look for specific facts, numbers, dates, locations, and measurable statements.\n\n"
f"Text to analyze: {text}\n\n"
"Return only the JSON array of verifiable claims:"
)
loop = asyncio.get_event_loop()
with concurrent.futures.ThreadPoolExecutor() as executor:
resp = await loop.run_in_executor(executor, lambda: self.gemini_client.models.generate_content(
model="gemini-1.5-flash",
contents=prompt
))
if not resp or not resp.text:
raise Exception("Empty response from Gemini API")
claims_text = resp.text.strip()
logger.info(f"Raw Gemini response for claims: {claims_text[:200]}...")
# Try to extract JSON from the response
try:
claims = json.loads(claims_text)
except json.JSONDecodeError:
# Try to find JSON array in the response (handle markdown code blocks)
import re
# First try to extract from markdown code blocks
code_block_match = re.search(r'```(?:json)?\s*(\[.*?\])\s*```', claims_text, re.DOTALL)
if code_block_match:
claims = json.loads(code_block_match.group(1))
else:
# Try to find JSON array directly
json_match = re.search(r'\[.*?\]', claims_text, re.DOTALL)
if json_match:
claims = json.loads(json_match.group())
else:
raise Exception(f"Could not parse JSON from Gemini response: {claims_text[:100]}")
result_text = await self._generate_text_async(prompt, user_id=user_id)
logger.info(f"Raw LLM response for claims: {result_text[:200]}...")
claims = self._parse_json_from_response(result_text, expect_array=True)
if isinstance(claims, list):
valid_claims = [claim for claim in claims if isinstance(claim, str) and claim.strip()]
logger.info(f"Successfully extracted {len(valid_claims)} claims")
return valid_claims
else:
raise Exception(f"Expected JSON array, got: {type(claims)}")
except Exception as e:
logger.error(f"Error extracting claims: {str(e)}")
raise Exception(f"Failed to extract claims: {str(e)}")
async def _verify_claims_batch(self, claims: List[str]) -> List[Claim]:
"""
Verify multiple claims in batch to reduce API calls.
Args:
claims: List of claims to verify
Returns:
List of Claim objects with verification results
"""
async def _verify_claims_batch(self, claims: List[str], user_id: str = None) -> List[Claim]:
"""Verify multiple claims in batch to reduce API calls."""
try:
logger.info(f"Starting batch verification of {len(claims)} claims")
# Limit to maximum 3 claims to prevent excessive API usage
max_claims = min(len(claims), 3)
claims_to_verify = claims[:max_claims]
if len(claims) > max_claims:
logger.warning(f"Limited verification to {max_claims} claims to prevent API rate limits")
# Step 1: Search for evidence for all claims in one batch
all_sources = await self._search_evidence_batch(claims_to_verify)
# Step 2: Assess all claims against sources in one API call
verified_claims = await self._assess_claims_batch(claims_to_verify, all_sources)
# Add any remaining claims as insufficient information
# Step 1: Search for evidence
all_sources = await self._search_evidence_batch(claims_to_verify, user_id=user_id)
# Step 2: Assess claims against sources
verified_claims = await self._assess_claims_batch(claims_to_verify, all_sources, user_id=user_id)
# Add remaining claims as insufficient information
for i in range(max_claims, len(claims)):
verified_claims.append(Claim(
text=claims[i],
@@ -288,13 +270,12 @@ class HallucinationDetector:
refuting_sources=[],
reasoning="Not verified due to API rate limit protection"
))
logger.info(f"Batch verification completed for {len(verified_claims)} claims")
return verified_claims
except Exception as e:
logger.error(f"Error in batch verification: {str(e)}")
# Return all claims as insufficient information
return [
Claim(
text=claim,
@@ -307,20 +288,11 @@ class HallucinationDetector:
for claim in claims
]
async def _verify_claim(self, claim: str) -> Claim:
"""
Verify a single claim using Exa.ai search.
Args:
claim: The claim to verify
Returns:
Claim object with verification results
"""
async def _verify_claim(self, claim: str, user_id: str = None) -> Claim:
"""Verify a single claim using Exa.ai search."""
try:
# Search for evidence using Exa.ai
sources = await self._search_evidence(claim)
sources = await self._search_evidence(claim, user_id=user_id)
if not sources:
return Claim(
text=claim,
@@ -330,10 +302,9 @@ class HallucinationDetector:
refuting_sources=[],
reasoning="No sources found for verification"
)
# Verify claim against sources using LLM
verification_result = await self._assess_claim_against_sources(claim, sources)
verification_result = await self._assess_claim_against_sources(claim, sources, user_id=user_id)
return Claim(
text=claim,
confidence=verification_result.get('confidence', 0.5),
@@ -342,7 +313,7 @@ class HallucinationDetector:
refuting_sources=verification_result.get('refuting_sources', []),
reasoning=verification_result.get('reasoning', '')
)
except Exception as e:
logger.error(f"Error verifying claim '{claim}': {str(e)}")
return Claim(
@@ -353,68 +324,40 @@ class HallucinationDetector:
refuting_sources=[],
reasoning=f"Error during verification: {str(e)}"
)
async def _search_evidence_batch(self, claims: List[str]) -> List[Dict[str, Any]]:
"""
Search for evidence for multiple claims in one API call.
Args:
claims: List of claims to search for
Returns:
List of sources relevant to the claims
"""
async def _search_evidence_batch(self, claims: List[str], user_id: str = None) -> List[Dict[str, Any]]:
"""Search for evidence for multiple claims in one API call."""
try:
# Combine all claims into one search query
combined_query = " ".join(claims[:2]) # Use first 2 claims to avoid query length limits
combined_query = " ".join(claims[:2])
logger.info(f"Searching for evidence for {len(claims)} claims with combined query")
# Use the existing search method with combined query
sources = await self._search_evidence(combined_query)
# Limit sources to prevent excessive processing
sources = await self._search_evidence(combined_query, user_id=user_id)
max_sources = 5
if len(sources) > max_sources:
sources = sources[:max_sources]
logger.info(f"Limited sources to {max_sources} to prevent API rate limits")
return sources
except Exception as e:
logger.error(f"Error in batch evidence search: {str(e)}")
return []
async def _assess_claims_batch(self, claims: List[str], sources: List[Dict[str, Any]]) -> List[Claim]:
"""
Assess multiple claims against sources in one API call.
Args:
claims: List of claims to assess
sources: List of sources to assess against
Returns:
List of Claim objects with assessment results
"""
if not self.gemini_client:
raise Exception("Gemini client not available. Cannot assess claims without AI provider.")
async def _assess_claims_batch(self, claims: List[str], sources: List[Dict[str, Any]], user_id: str = None) -> List[Claim]:
"""Assess multiple claims against sources in one LLM call."""
try:
# Limit to 3 claims to prevent excessive API usage
claims_to_assess = claims[:3]
# Prepare sources text
combined_sources = "\n\n".join([
f"Source {i+1}: {src.get('url','')}\nText: {src.get('text','')[:1000]}"
for i, src in enumerate(sources)
])
# Prepare claims text
claims_text = "\n".join([
f"Claim {i+1}: {claim}"
for i, claim in enumerate(claims_to_assess)
])
prompt = (
"You are a strict fact-checker. Analyze each claim against the provided sources.\n\n"
"Return ONLY a valid JSON object with this exact structure:\n"
@@ -434,63 +377,36 @@ class HallucinationDetector:
f"Sources:\n{combined_sources}\n\n"
"Return only the JSON object:"
)
loop = asyncio.get_event_loop()
with concurrent.futures.ThreadPoolExecutor() as executor:
resp = await loop.run_in_executor(executor, lambda: self.gemini_client.models.generate_content(
model="gemini-1.5-flash",
contents=prompt
))
if not resp or not resp.text:
raise Exception("Empty response from Gemini API for batch assessment")
result_text = resp.text.strip()
logger.info(f"Raw Gemini response for batch assessment: {result_text[:200]}...")
# Try to extract JSON from the response
try:
result = json.loads(result_text)
except json.JSONDecodeError:
# Try to find JSON object in the response (handle markdown code blocks)
import re
code_block_match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', result_text, re.DOTALL)
if code_block_match:
result = json.loads(code_block_match.group(1))
else:
json_match = re.search(r'\{.*?\}', result_text, re.DOTALL)
if json_match:
result = json.loads(json_match.group())
else:
raise Exception(f"Could not parse JSON from Gemini response: {result_text[:100]}")
# Process assessments
result_text = await self._generate_text_async(prompt, user_id=user_id)
logger.info(f"Raw LLM response for batch assessment: {result_text[:200]}...")
result = self._parse_json_from_response(result_text, expect_array=False)
assessments = result.get('assessments', [])
verified_claims = []
for i, claim in enumerate(claims_to_assess):
# Find assessment for this claim
assessment = None
for a in assessments:
if a.get('claim_index') == i:
assessment = a
break
if assessment:
# Process supporting and refuting sources
supporting_sources = []
refuting_sources = []
if isinstance(assessment.get('supporting_sources'), list):
for idx in assessment['supporting_sources']:
if isinstance(idx, int) and 0 <= idx < len(sources):
supporting_sources.append(sources[idx])
if isinstance(assessment.get('refuting_sources'), list):
for idx in assessment['refuting_sources']:
if isinstance(idx, int) and 0 <= idx < len(sources):
refuting_sources.append(sources[idx])
verified_claims.append(Claim(
text=claim,
confidence=float(assessment.get('confidence', 0.5)),
@@ -500,7 +416,6 @@ class HallucinationDetector:
reasoning=assessment.get('reasoning', '')
))
else:
# No assessment found for this claim
verified_claims.append(Claim(
text=claim,
confidence=0.0,
@@ -509,13 +424,12 @@ class HallucinationDetector:
refuting_sources=[],
reasoning="No assessment provided"
))
logger.info(f"Successfully assessed {len(verified_claims)} claims in batch")
return verified_claims
except Exception as e:
logger.error(f"Error in batch assessment: {str(e)}")
# Return all claims as insufficient information
return [
Claim(
text=claim,
@@ -528,88 +442,32 @@ class HallucinationDetector:
for claim in claims_to_assess
]
async def _search_evidence(self, claim: str) -> List[Dict[str, Any]]:
"""
Search for evidence using Exa.ai API.
Args:
claim: The claim to search evidence for
Returns:
List of source documents with evidence
"""
if not self.exa_api_key:
raise Exception("Exa API key not available. Cannot search for evidence without Exa.ai access.")
async def _search_evidence(self, claim: str, user_id: str = None) -> List[Dict[str, Any]]:
"""Search for evidence using ExaResearchProvider with subscription checks."""
try:
headers = {
'x-api-key': self.exa_api_key,
'Content-Type': 'application/json'
}
payload = {
'query': claim,
'numResults': 5,
'text': True,
'useAutoprompt': True
}
response = requests.post(
'https://api.exa.ai/search',
headers=headers,
json=payload,
timeout=15
from services.blog_writer.research.exa_provider import ExaResearchProvider
provider = ExaResearchProvider()
sources = await provider.simple_search(
query=claim,
num_results=5,
user_id=user_id,
)
if response.status_code == 200:
data = response.json()
results = data.get('results', [])
if not results:
raise Exception(f"No search results found for claim: {claim}")
sources = []
for result in results:
source = {
'title': result.get('title', 'Untitled'),
'url': result.get('url', ''),
'text': result.get('text', ''),
'publishedDate': result.get('publishedDate', ''),
'author': result.get('author', ''),
'score': result.get('score', 0.5)
}
sources.append(source)
logger.info(f"Found {len(sources)} sources for claim: {claim[:50]}...")
return sources
else:
raise Exception(f"Exa API error: {response.status_code} - {response.text}")
if not sources:
raise Exception(f"No search results found for claim: {claim}")
logger.info(f"Found {len(sources)} sources for claim: {claim[:50]}...")
return sources
except Exception as e:
logger.error(f"Error searching evidence with Exa: {str(e)}")
raise Exception(f"Failed to search evidence: {str(e)}")
async def _assess_claim_against_sources(self, claim: str, sources: List[Dict[str, Any]]) -> Dict[str, Any]:
"""
Assess whether sources support or refute the claim using LLM.
Args:
claim: The claim to assess
sources: List of source documents
Returns:
Dictionary with assessment results
"""
if not self.gemini_client:
raise Exception("Gemini client not available. Cannot assess claims without AI provider.")
async def _assess_claim_against_sources(self, claim: str, sources: List[Dict[str, Any]], user_id: str = None) -> Dict[str, Any]:
"""Assess whether sources support or refute the claim using LLM."""
try:
combined_sources = "\n\n".join([
f"Source {i+1}: {src.get('url','')}\nText: {src.get('text','')[:2000]}"
for i, src in enumerate(sources)
])
prompt = (
"You are a strict fact-checker. Analyze the claim against the provided sources.\n\n"
"Return ONLY a valid JSON object with this exact structure:\n"
@@ -624,70 +482,44 @@ class HallucinationDetector:
f"Sources:\n{combined_sources}\n\n"
"Return only the JSON object:"
)
loop = asyncio.get_event_loop()
with concurrent.futures.ThreadPoolExecutor() as executor:
resp = await loop.run_in_executor(executor, lambda: self.gemini_client.models.generate_content(
model="gemini-1.5-flash",
contents=prompt
))
if not resp or not resp.text:
raise Exception("Empty response from Gemini API for claim assessment")
result_text = resp.text.strip()
logger.info(f"Raw Gemini response for assessment: {result_text[:200]}...")
# Try to extract JSON from the response
try:
result = json.loads(result_text)
except json.JSONDecodeError:
# Try to find JSON object in the response (handle markdown code blocks)
import re
# First try to extract from markdown code blocks
code_block_match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', result_text, re.DOTALL)
if code_block_match:
result = json.loads(code_block_match.group(1))
else:
# Try to find JSON object directly
json_match = re.search(r'\{.*?\}', result_text, re.DOTALL)
if json_match:
result = json.loads(json_match.group())
else:
raise Exception(f"Could not parse JSON from Gemini response: {result_text[:100]}")
result_text = await self._generate_text_async(prompt, user_id=user_id)
logger.info(f"Raw LLM response for assessment: {result_text[:200]}...")
result = self._parse_json_from_response(result_text, expect_array=False)
# Validate required fields
required_fields = ['assessment', 'confidence', 'supporting_sources', 'refuting_sources', 'reasoning']
for field in required_fields:
if field not in result:
raise Exception(f"Missing required field '{field}' in assessment response")
# Process supporting and refuting sources
supporting_sources = []
refuting_sources = []
if isinstance(result.get('supporting_sources'), list):
for idx in result['supporting_sources']:
if isinstance(idx, int) and 0 <= idx < len(sources):
supporting_sources.append(sources[idx])
if isinstance(result.get('refuting_sources'), list):
for idx in result['refuting_sources']:
if isinstance(idx, int) and 0 <= idx < len(sources):
refuting_sources.append(sources[idx])
# Validate assessment value
valid_assessments = ['supported', 'refuted', 'insufficient_information']
if result['assessment'] not in valid_assessments:
raise Exception(f"Invalid assessment value: {result['assessment']}")
# Validate confidence value
confidence = float(result['confidence'])
if not (0.0 <= confidence <= 1.0):
raise Exception(f"Invalid confidence value: {confidence}")
logger.info(f"Successfully assessed claim: {result['assessment']} (confidence: {confidence})")
return {
'assessment': result['assessment'],
'confidence': confidence,
@@ -695,8 +527,39 @@ class HallucinationDetector:
'refuting_sources': refuting_sources,
'reasoning': result['reasoning']
}
except Exception as e:
logger.error(f"Error assessing claim against sources: {str(e)}")
raise Exception(f"Failed to assess claim: {str(e)}")
def _parse_json_from_response(self, text: str, expect_array: bool = False):
"""Extract and parse JSON from LLM response, handling markdown code blocks."""
text = text.strip()
# Try direct parse first
try:
result = json.loads(text)
return result
except json.JSONDecodeError:
pass
import re
# Try to extract from markdown code blocks
if expect_array:
code_block_match = re.search(r'```(?:json)?\s*(\[.*?\])\s*```', text, re.DOTALL)
if code_block_match:
return json.loads(code_block_match.group(1))
# Try to find JSON array directly
json_match = re.search(r'\[.*\]', text, re.DOTALL)
if json_match:
return json.loads(json_match.group())
else:
code_block_match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', text, re.DOTALL)
if code_block_match:
return json.loads(code_block_match.group(1))
# Try to find JSON object directly
json_match = re.search(r'\{.*\}', text, re.DOTALL)
if json_match:
return json.loads(json_match.group())
raise Exception(f"Could not parse JSON from LLM response: {text[:100]}")