419 lines
20 KiB
Python
419 lines
20 KiB
Python
"""
|
|
Research Service - Core research functionality for AI Blog Writer.
|
|
|
|
Handles Google Search grounding, caching, and research orchestration.
|
|
"""
|
|
|
|
from typing import Dict, Any, List
|
|
from loguru import logger
|
|
|
|
from models.blog_models import (
|
|
BlogResearchRequest,
|
|
BlogResearchResponse,
|
|
ResearchSource,
|
|
GroundingMetadata,
|
|
GroundingChunk,
|
|
GroundingSupport,
|
|
Citation,
|
|
)
|
|
|
|
from .keyword_analyzer import KeywordAnalyzer
|
|
from .competitor_analyzer import CompetitorAnalyzer
|
|
from .content_angle_generator import ContentAngleGenerator
|
|
from .data_filter import ResearchDataFilter
|
|
|
|
|
|
class ResearchService:
|
|
"""Service for conducting comprehensive research using Google Search grounding."""
|
|
|
|
def __init__(self):
|
|
self.keyword_analyzer = KeywordAnalyzer()
|
|
self.competitor_analyzer = CompetitorAnalyzer()
|
|
self.content_angle_generator = ContentAngleGenerator()
|
|
self.data_filter = ResearchDataFilter()
|
|
|
|
async def research(self, request: BlogResearchRequest) -> BlogResearchResponse:
|
|
"""
|
|
Stage 1: Research & Strategy (AI Orchestration)
|
|
Uses ONLY Gemini's native Google Search grounding - ONE API call for everything.
|
|
Follows LinkedIn service pattern for efficiency and cost optimization.
|
|
Includes intelligent caching for exact keyword matches.
|
|
"""
|
|
try:
|
|
from services.llm_providers.gemini_grounded_provider import GeminiGroundedProvider
|
|
from services.cache.research_cache import research_cache
|
|
|
|
topic = request.topic or ", ".join(request.keywords)
|
|
industry = request.industry or (request.persona.industry if request.persona and request.persona.industry else "General")
|
|
target_audience = getattr(request.persona, 'target_audience', 'General') if request.persona else 'General'
|
|
|
|
# Check cache first for exact keyword match
|
|
cached_result = research_cache.get_cached_result(
|
|
keywords=request.keywords,
|
|
industry=industry,
|
|
target_audience=target_audience
|
|
)
|
|
|
|
if cached_result:
|
|
logger.info(f"Returning cached research result for keywords: {request.keywords}")
|
|
return BlogResearchResponse(**cached_result)
|
|
|
|
# Cache miss - proceed with API call
|
|
logger.info(f"Cache miss - making API call for keywords: {request.keywords}")
|
|
gemini = GeminiGroundedProvider()
|
|
|
|
# Single comprehensive research prompt - Gemini handles Google Search automatically
|
|
research_prompt = f"""
|
|
Research the topic "{topic}" in the {industry} industry for {target_audience} audience. Provide a comprehensive analysis including:
|
|
|
|
1. Current trends and insights (2024-2025)
|
|
2. Key statistics and data points with sources
|
|
3. Industry expert opinions and quotes
|
|
4. Recent developments and news
|
|
5. Market analysis and forecasts
|
|
6. Best practices and case studies
|
|
7. Keyword analysis: primary, secondary, and long-tail opportunities
|
|
8. Competitor analysis: top players and content gaps
|
|
9. Content angle suggestions: 5 compelling angles for blog posts
|
|
|
|
Focus on factual, up-to-date information from credible sources.
|
|
Include specific data points, percentages, and recent developments.
|
|
Structure your response with clear sections for each analysis area.
|
|
"""
|
|
|
|
# Single Gemini call with native Google Search grounding - no fallbacks
|
|
gemini_result = await gemini.generate_grounded_content(
|
|
prompt=research_prompt,
|
|
content_type="research",
|
|
max_tokens=2000
|
|
)
|
|
|
|
# Extract sources from grounding metadata
|
|
sources = self._extract_sources_from_grounding(gemini_result)
|
|
|
|
# Extract grounding metadata for detailed UI display
|
|
grounding_metadata = self._extract_grounding_metadata(gemini_result)
|
|
|
|
# Extract search widget and queries for UI display
|
|
search_widget = gemini_result.get("search_widget", "") or ""
|
|
search_queries = gemini_result.get("search_queries", []) or []
|
|
|
|
# Parse the comprehensive response for different analysis components
|
|
content = gemini_result.get("content", "")
|
|
keyword_analysis = self.keyword_analyzer.analyze(content, request.keywords)
|
|
competitor_analysis = self.competitor_analyzer.analyze(content)
|
|
suggested_angles = self.content_angle_generator.generate(content, topic, industry)
|
|
|
|
logger.info(f"Research completed successfully with {len(sources)} sources and {len(search_queries)} search queries")
|
|
|
|
# Create the response
|
|
response = BlogResearchResponse(
|
|
success=True,
|
|
sources=sources,
|
|
keyword_analysis=keyword_analysis,
|
|
competitor_analysis=competitor_analysis,
|
|
suggested_angles=suggested_angles,
|
|
# Add search widget and queries for UI display
|
|
search_widget=search_widget if 'search_widget' in locals() else "",
|
|
search_queries=search_queries if 'search_queries' in locals() else [],
|
|
# Add grounding metadata for detailed UI display
|
|
grounding_metadata=grounding_metadata,
|
|
)
|
|
|
|
# Filter and clean research data for optimal AI processing
|
|
filtered_response = self.data_filter.filter_research_data(response)
|
|
logger.info("Research data filtering completed successfully")
|
|
|
|
# Cache the successful result for future exact keyword matches (both caches)
|
|
persistent_research_cache.cache_result(
|
|
keywords=request.keywords,
|
|
industry=industry,
|
|
target_audience=target_audience,
|
|
result=filtered_response.dict()
|
|
)
|
|
|
|
# Also cache in memory for faster access
|
|
research_cache.cache_result(
|
|
keywords=request.keywords,
|
|
industry=industry,
|
|
target_audience=target_audience,
|
|
result=filtered_response.dict()
|
|
)
|
|
|
|
return filtered_response
|
|
|
|
except Exception as e:
|
|
error_message = str(e)
|
|
logger.error(f"Research failed: {error_message}")
|
|
|
|
# Return a graceful failure response instead of raising
|
|
return BlogResearchResponse(
|
|
success=False,
|
|
sources=[],
|
|
keyword_analysis={},
|
|
competitor_analysis={},
|
|
suggested_angles=[],
|
|
search_widget="",
|
|
search_queries=[],
|
|
error_message=error_message
|
|
)
|
|
|
|
async def research_with_progress(self, request: BlogResearchRequest, task_id: str) -> BlogResearchResponse:
|
|
"""
|
|
Research method with progress updates for real-time feedback.
|
|
"""
|
|
try:
|
|
from services.llm_providers.gemini_grounded_provider import GeminiGroundedProvider
|
|
from services.cache.research_cache import research_cache
|
|
from services.cache.persistent_research_cache import persistent_research_cache
|
|
from api.blog_writer.task_manager import task_manager
|
|
|
|
topic = request.topic or ", ".join(request.keywords)
|
|
industry = request.industry or (request.persona.industry if request.persona and request.persona.industry else "General")
|
|
target_audience = getattr(request.persona, 'target_audience', 'General') if request.persona else 'General'
|
|
|
|
# Check cache first for exact keyword match (try both caches)
|
|
await task_manager.update_progress(task_id, "🔍 Checking cache for existing research...")
|
|
|
|
# Try persistent cache first (survives restarts)
|
|
cached_result = persistent_research_cache.get_cached_result(
|
|
keywords=request.keywords,
|
|
industry=industry,
|
|
target_audience=target_audience
|
|
)
|
|
|
|
# Fallback to in-memory cache
|
|
if not cached_result:
|
|
cached_result = research_cache.get_cached_result(
|
|
keywords=request.keywords,
|
|
industry=industry,
|
|
target_audience=target_audience
|
|
)
|
|
|
|
if cached_result:
|
|
await task_manager.update_progress(task_id, "✅ Found cached research results! Returning instantly...")
|
|
logger.info(f"Returning cached research result for keywords: {request.keywords}")
|
|
return BlogResearchResponse(**cached_result)
|
|
|
|
# Cache miss - proceed with API call
|
|
await task_manager.update_progress(task_id, "🌐 Cache miss - connecting to Google Search grounding...")
|
|
logger.info(f"Cache miss - making API call for keywords: {request.keywords}")
|
|
gemini = GeminiGroundedProvider()
|
|
|
|
# Single comprehensive research prompt - Gemini handles Google Search automatically
|
|
research_prompt = f"""
|
|
Research the topic "{topic}" in the {industry} industry for {target_audience} audience. Provide a comprehensive analysis including:
|
|
|
|
1. Current trends and insights (2024-2025)
|
|
2. Key statistics and data points with sources
|
|
3. Industry expert opinions and quotes
|
|
4. Recent developments and news
|
|
5. Market analysis and forecasts
|
|
6. Best practices and case studies
|
|
7. Keyword analysis: primary, secondary, and long-tail opportunities
|
|
8. Competitor analysis: top players and content gaps
|
|
9. Content angle suggestions: 5 compelling angles for blog posts
|
|
|
|
Focus on factual, up-to-date information from credible sources.
|
|
Include specific data points, percentages, and recent developments.
|
|
Structure your response with clear sections for each analysis area.
|
|
"""
|
|
|
|
await task_manager.update_progress(task_id, "🤖 Making AI request to Gemini with Google Search grounding...")
|
|
# Single Gemini call with native Google Search grounding - no fallbacks
|
|
gemini_result = await gemini.generate_grounded_content(
|
|
prompt=research_prompt,
|
|
content_type="research",
|
|
max_tokens=2000
|
|
)
|
|
|
|
await task_manager.update_progress(task_id, "📊 Processing research results and extracting insights...")
|
|
# Extract sources from grounding metadata
|
|
sources = self._extract_sources_from_grounding(gemini_result)
|
|
|
|
# Extract grounding metadata for detailed UI display
|
|
grounding_metadata = self._extract_grounding_metadata(gemini_result)
|
|
|
|
# Extract search widget and queries for UI display
|
|
search_widget = gemini_result.get("search_widget", "") or ""
|
|
search_queries = gemini_result.get("search_queries", []) or []
|
|
|
|
await task_manager.update_progress(task_id, "🔍 Analyzing keywords and content angles...")
|
|
# Parse the comprehensive response for different analysis components
|
|
content = gemini_result.get("content", "")
|
|
keyword_analysis = self.keyword_analyzer.analyze(content, request.keywords)
|
|
competitor_analysis = self.competitor_analyzer.analyze(content)
|
|
suggested_angles = self.content_angle_generator.generate(content, topic, industry)
|
|
|
|
await task_manager.update_progress(task_id, "💾 Caching results for future use...")
|
|
logger.info(f"Research completed successfully with {len(sources)} sources and {len(search_queries)} search queries")
|
|
|
|
# Create the response
|
|
response = BlogResearchResponse(
|
|
success=True,
|
|
sources=sources,
|
|
keyword_analysis=keyword_analysis,
|
|
competitor_analysis=competitor_analysis,
|
|
suggested_angles=suggested_angles,
|
|
# Add search widget and queries for UI display
|
|
search_widget=search_widget if 'search_widget' in locals() else "",
|
|
search_queries=search_queries if 'search_queries' in locals() else [],
|
|
# Add grounding metadata for detailed UI display
|
|
grounding_metadata=grounding_metadata,
|
|
# Preserve original user keywords for caching
|
|
original_keywords=request.keywords,
|
|
)
|
|
|
|
# Filter and clean research data for optimal AI processing
|
|
await task_manager.update_progress(task_id, "🔍 Filtering and cleaning research data...")
|
|
filtered_response = self.data_filter.filter_research_data(response)
|
|
logger.info("Research data filtering completed successfully")
|
|
|
|
# Cache the successful result for future exact keyword matches (both caches)
|
|
persistent_research_cache.cache_result(
|
|
keywords=request.keywords,
|
|
industry=industry,
|
|
target_audience=target_audience,
|
|
result=filtered_response.dict()
|
|
)
|
|
|
|
# Also cache in memory for faster access
|
|
research_cache.cache_result(
|
|
keywords=request.keywords,
|
|
industry=industry,
|
|
target_audience=target_audience,
|
|
result=filtered_response.dict()
|
|
)
|
|
|
|
return filtered_response
|
|
|
|
except Exception as e:
|
|
error_message = str(e)
|
|
logger.error(f"Research failed: {error_message}")
|
|
|
|
# Return a graceful failure response instead of raising
|
|
return BlogResearchResponse(
|
|
success=False,
|
|
sources=[],
|
|
keyword_analysis={},
|
|
competitor_analysis={},
|
|
suggested_angles=[],
|
|
search_widget="",
|
|
search_queries=[],
|
|
error_message=error_message
|
|
)
|
|
|
|
def _extract_sources_from_grounding(self, gemini_result: Dict[str, Any]) -> List[ResearchSource]:
|
|
"""Extract sources from Gemini grounding metadata."""
|
|
sources = []
|
|
|
|
# The Gemini grounded provider already extracts sources and puts them in the 'sources' field
|
|
raw_sources = gemini_result.get("sources", [])
|
|
for src in raw_sources:
|
|
source = ResearchSource(
|
|
title=src.get("title", "Untitled"),
|
|
url=src.get("url", ""),
|
|
excerpt=src.get("content", "")[:500] if src.get("content") else f"Source from {src.get('title', 'web')}",
|
|
credibility_score=float(src.get("credibility_score", 0.8)),
|
|
published_at=str(src.get("publication_date", "2024-01-01")),
|
|
index=src.get("index"),
|
|
source_type=src.get("type", "web")
|
|
)
|
|
sources.append(source)
|
|
|
|
return sources
|
|
|
|
def _extract_grounding_metadata(self, gemini_result: Dict[str, Any]) -> GroundingMetadata:
|
|
"""Extract detailed grounding metadata from Gemini result."""
|
|
grounding_chunks = []
|
|
grounding_supports = []
|
|
citations = []
|
|
|
|
# Extract grounding chunks from the raw grounding metadata
|
|
raw_grounding = gemini_result.get("grounding_metadata", {})
|
|
|
|
# Handle case where grounding_metadata might be a GroundingMetadata object
|
|
if hasattr(raw_grounding, 'grounding_chunks'):
|
|
raw_chunks = raw_grounding.grounding_chunks
|
|
else:
|
|
raw_chunks = raw_grounding.get("grounding_chunks", [])
|
|
|
|
for chunk in raw_chunks:
|
|
if "web" in chunk:
|
|
web_data = chunk["web"]
|
|
grounding_chunk = GroundingChunk(
|
|
title=web_data.get("title", "Untitled"),
|
|
url=web_data.get("uri", ""),
|
|
confidence_score=None # Will be set from supports
|
|
)
|
|
grounding_chunks.append(grounding_chunk)
|
|
|
|
# Extract grounding supports with confidence scores
|
|
if hasattr(raw_grounding, 'grounding_supports'):
|
|
raw_supports = raw_grounding.grounding_supports
|
|
else:
|
|
raw_supports = raw_grounding.get("grounding_supports", [])
|
|
for support in raw_supports:
|
|
# Handle both dictionary and GroundingSupport object formats
|
|
if hasattr(support, 'confidence_scores'):
|
|
confidence_scores = support.confidence_scores
|
|
chunk_indices = support.grounding_chunk_indices
|
|
segment_text = getattr(support, 'segment_text', '')
|
|
start_index = getattr(support, 'start_index', None)
|
|
end_index = getattr(support, 'end_index', None)
|
|
else:
|
|
confidence_scores = support.get("confidence_scores", [])
|
|
chunk_indices = support.get("grounding_chunk_indices", [])
|
|
segment = support.get("segment", {})
|
|
segment_text = segment.get("text", "")
|
|
start_index = segment.get("start_index")
|
|
end_index = segment.get("end_index")
|
|
|
|
grounding_support = GroundingSupport(
|
|
confidence_scores=confidence_scores,
|
|
grounding_chunk_indices=chunk_indices,
|
|
segment_text=segment_text,
|
|
start_index=start_index,
|
|
end_index=end_index
|
|
)
|
|
grounding_supports.append(grounding_support)
|
|
|
|
# Update confidence scores for chunks
|
|
if confidence_scores and chunk_indices:
|
|
avg_confidence = sum(confidence_scores) / len(confidence_scores)
|
|
for idx in chunk_indices:
|
|
if idx < len(grounding_chunks):
|
|
grounding_chunks[idx].confidence_score = avg_confidence
|
|
|
|
# Extract citations from the raw result
|
|
raw_citations = gemini_result.get("citations", [])
|
|
for citation in raw_citations:
|
|
citation_obj = Citation(
|
|
citation_type=citation.get("type", "inline"),
|
|
start_index=citation.get("start_index", 0),
|
|
end_index=citation.get("end_index", 0),
|
|
text=citation.get("text", ""),
|
|
source_indices=citation.get("source_indices", []),
|
|
reference=citation.get("reference", "")
|
|
)
|
|
citations.append(citation_obj)
|
|
|
|
# Extract search entry point and web search queries
|
|
if hasattr(raw_grounding, 'search_entry_point'):
|
|
search_entry_point = getattr(raw_grounding.search_entry_point, 'rendered_content', '') if raw_grounding.search_entry_point else ''
|
|
else:
|
|
search_entry_point = raw_grounding.get("search_entry_point", {}).get("rendered_content", "")
|
|
|
|
if hasattr(raw_grounding, 'web_search_queries'):
|
|
web_search_queries = raw_grounding.web_search_queries
|
|
else:
|
|
web_search_queries = raw_grounding.get("web_search_queries", [])
|
|
|
|
return GroundingMetadata(
|
|
grounding_chunks=grounding_chunks,
|
|
grounding_supports=grounding_supports,
|
|
citations=citations,
|
|
search_entry_point=search_entry_point,
|
|
web_search_queries=web_search_queries
|
|
)
|