Added enhanced linguistic analyzer and persona quality improver

This commit is contained in:
ajaysi
2025-09-14 09:53:27 +05:30
parent c63148e1ce
commit 1460ce3cb6
35 changed files with 4446 additions and 118 deletions

View File

@@ -0,0 +1,152 @@
"""
ContextMemory - maintains intelligent continuity context across sections using LLM-enhanced summarization.
Stores smart per-section summaries and thread keywords for use in prompts with cost optimization.
"""
from __future__ import annotations
from typing import Dict, List, Optional, Tuple
from collections import deque
from loguru import logger
import hashlib
# Import the common gemini provider
from services.llm_providers.gemini_provider import gemini_text_response
class ContextMemory:
"""In-memory continuity store for recent sections with LLM-enhanced summarization.
Notes:
- Keeps an ordered deque of recent (section_id, summary) pairs
- Uses LLM for intelligent summarization when content is substantial
- Provides utilities to build a compact previous-sections summary
- Implements caching to minimize LLM calls
"""
def __init__(self, max_entries: int = 10):
self.max_entries = max_entries
self._recent: deque[Tuple[str, str]] = deque(maxlen=max_entries)
# Cache for LLM-generated summaries
self._summary_cache: Dict[str, str] = {}
logger.info("✅ ContextMemory initialized with LLM-enhanced summarization")
def update_with_section(self, section_id: str, full_text: str, use_llm: bool = True) -> None:
"""Create a compact summary and store it for continuity usage."""
summary = self._summarize_text_intelligently(full_text, use_llm=use_llm)
self._recent.append((section_id, summary))
def get_recent_summaries(self, limit: int = 2) -> List[str]:
"""Return the last N stored summaries (most recent first)."""
return [s for (_sid, s) in list(self._recent)[-limit:]]
def build_previous_sections_summary(self, limit: int = 2) -> str:
"""Join recent summaries for prompt injection."""
recents = self.get_recent_summaries(limit=limit)
if not recents:
return ""
return "\n\n".join(recents)
def _summarize_text_intelligently(self, text: str, target_words: int = 80, use_llm: bool = True) -> str:
"""Create intelligent summary using LLM when appropriate, fallback to truncation."""
# Create cache key
cache_key = self._get_cache_key(text)
# Check cache first
if cache_key in self._summary_cache:
logger.debug("Summary cache hit")
return self._summary_cache[cache_key]
# Determine if we should use LLM
should_use_llm = use_llm and self._should_use_llm_summarization(text)
if should_use_llm:
try:
summary = self._llm_summarize_text(text, target_words)
self._summary_cache[cache_key] = summary
logger.info("LLM-based summarization completed")
return summary
except Exception as e:
logger.warning(f"LLM summarization failed, using fallback: {e}")
# Fall through to local summarization
# Local fallback
summary = self._summarize_text_locally(text, target_words)
self._summary_cache[cache_key] = summary
return summary
def _should_use_llm_summarization(self, text: str) -> bool:
"""Determine if content is substantial enough to warrant LLM summarization."""
word_count = len(text.split())
# Use LLM for substantial content (>150 words) or complex structure
has_complex_structure = any(marker in text for marker in ['##', '###', '**', '*', '-', '1.', '2.'])
return word_count > 150 or has_complex_structure
def _llm_summarize_text(self, text: str, target_words: int = 80) -> str:
"""Use Gemini API for intelligent text summarization."""
# Truncate text to minimize tokens while keeping key content
truncated_text = text[:800] # First 800 chars usually contain the main points
prompt = f"""
Summarize the following content in approximately {target_words} words, focusing on key concepts and main points.
Content: {truncated_text}
Requirements:
- Capture the main ideas and key concepts
- Maintain the original tone and style
- Keep it concise but informative
- Focus on what's most important for continuity
Generate only the summary, no explanations or formatting.
"""
try:
result = gemini_text_response(
prompt=prompt,
temperature=0.3, # Low temperature for consistent summarization
max_tokens=500, # Increased tokens for better summaries
system_prompt="You are an expert at creating concise, informative summaries."
)
if result and result.strip():
summary = result.strip()
# Ensure it's not too long
words = summary.split()
if len(words) > target_words + 20: # Allow some flexibility
summary = " ".join(words[:target_words]) + "..."
return summary
else:
logger.warning("LLM summary response empty, using fallback")
return self._summarize_text_locally(text, target_words)
except Exception as e:
logger.error(f"LLM summarization error: {e}")
return self._summarize_text_locally(text, target_words)
def _summarize_text_locally(self, text: str, target_words: int = 80) -> str:
"""Very lightweight, deterministic truncation-based summary.
This deliberately avoids extra LLM calls. It collects the first
sentences up to approximately target_words.
"""
words = text.split()
if len(words) <= target_words:
return text.strip()
return " ".join(words[:target_words]).strip() + ""
def _get_cache_key(self, text: str) -> str:
"""Generate cache key from text hash."""
# Use first 200 chars for cache key to balance uniqueness vs memory
return hashlib.md5(text[:200].encode()).hexdigest()[:12]
def clear_cache(self):
"""Clear summary cache (useful for testing or memory management)."""
self._summary_cache.clear()
logger.info("ContextMemory cache cleared")

View File

@@ -0,0 +1,74 @@
"""
EnhancedContentGenerator - thin orchestrator combining URL selection and Gemini provider.
Provides Draft vs Polished modes and optional URL Context usage.
"""
from typing import Any, Dict
from services.llm_providers.gemini_grounded_provider import GeminiGroundedProvider
from .source_url_manager import SourceURLManager
from .context_memory import ContextMemory
from .transition_generator import TransitionGenerator
from .flow_analyzer import FlowAnalyzer
class EnhancedContentGenerator:
def __init__(self):
self.provider = GeminiGroundedProvider()
self.url_manager = SourceURLManager()
self.memory = ContextMemory(max_entries=12)
self.transitioner = TransitionGenerator()
self.flow = FlowAnalyzer()
async def generate_section(self, section: Any, research: Any, mode: str = "polished") -> Dict[str, Any]:
urls = self.url_manager.pick_relevant_urls(section, research)
prev_summary = self.memory.build_previous_sections_summary(limit=2)
prompt = self._build_prompt(section, research, prev_summary)
result = await self.provider.generate_grounded_content(
prompt=prompt,
content_type="linkedin_article",
temperature=0.6 if mode == "polished" else 0.8,
max_tokens=2048,
urls=urls,
mode=mode,
)
# Generate transition and compute intelligent flow metrics
previous_text = prev_summary
current_text = result.get("content", "")
transition = self.transitioner.generate_transition(previous_text, getattr(section, 'heading', 'This section'), use_llm=True)
metrics = self.flow.assess_flow(previous_text, current_text, use_llm=True)
# Update memory for subsequent sections and store continuity snapshot
if current_text:
self.memory.update_with_section(getattr(section, 'id', 'unknown'), current_text, use_llm=True)
# Return enriched result
result["transition"] = transition
result["continuity_metrics"] = metrics
# Persist a lightweight continuity snapshot for API access
try:
sid = getattr(section, 'id', 'unknown')
if not hasattr(self, "_last_continuity"):
self._last_continuity = {}
self._last_continuity[sid] = metrics
except Exception:
pass
return result
def _build_prompt(self, section: Any, research: Any, prev_summary: str) -> str:
heading = getattr(section, 'heading', 'Section')
key_points = getattr(section, 'key_points', [])
keywords = getattr(section, 'keywords', [])
target_words = getattr(section, 'target_words', 300)
return (
f"You are writing the blog section '{heading}'.\n\n"
f"Context summary: {prev_summary}\n"
f"Key points: {', '.join(key_points)}\n"
f"Keywords: {', '.join(keywords)}\n"
f"Target word count: {target_words}.\n"
"Use only factual info from provided sources; add short transition, then body."
)

View File

@@ -0,0 +1,162 @@
"""
FlowAnalyzer - evaluates narrative flow using LLM-based analysis with cost optimization.
Uses Gemini API for intelligent analysis while minimizing API calls through caching and smart triggers.
"""
from typing import Dict, Optional
from loguru import logger
import hashlib
import json
# Import the common gemini provider
from services.llm_providers.gemini_provider import gemini_structured_json_response
class FlowAnalyzer:
def __init__(self):
# Simple in-memory cache to avoid redundant LLM calls
self._cache: Dict[str, Dict[str, float]] = {}
# Cache for rule-based fallback when LLM analysis isn't needed
self._rule_cache: Dict[str, Dict[str, float]] = {}
logger.info("✅ FlowAnalyzer initialized with LLM-based analysis")
def assess_flow(self, previous_text: str, current_text: str, use_llm: bool = True) -> Dict[str, float]:
"""
Return flow metrics in range 0..1.
Args:
previous_text: Previous section content
current_text: Current section content
use_llm: Whether to use LLM analysis (default: True for significant content)
"""
if not current_text:
return {"flow": 0.0, "consistency": 0.0, "progression": 0.0}
# Create cache key from content hashes
cache_key = self._get_cache_key(previous_text, current_text)
# Check cache first
if cache_key in self._cache:
logger.debug("Flow analysis cache hit")
return self._cache[cache_key]
# Determine if we should use LLM analysis
should_use_llm = use_llm and self._should_use_llm_analysis(previous_text, current_text)
if should_use_llm:
try:
metrics = self._llm_flow_analysis(previous_text, current_text)
self._cache[cache_key] = metrics
logger.info("LLM-based flow analysis completed")
return metrics
except Exception as e:
logger.warning(f"LLM flow analysis failed, falling back to rules: {e}")
# Fall through to rule-based analysis
# Rule-based fallback (cached separately)
if cache_key in self._rule_cache:
return self._rule_cache[cache_key]
metrics = self._rule_based_analysis(previous_text, current_text)
self._rule_cache[cache_key] = metrics
return metrics
def _should_use_llm_analysis(self, previous_text: str, current_text: str) -> bool:
"""Determine if content is significant enough to warrant LLM analysis."""
# Use LLM for substantial content or when previous context exists
word_count = len(current_text.split())
has_previous = bool(previous_text and len(previous_text.strip()) > 50)
# Use LLM if: substantial content (>100 words) OR has meaningful previous context
return word_count > 100 or has_previous
def _llm_flow_analysis(self, previous_text: str, current_text: str) -> Dict[str, float]:
"""Use Gemini API for intelligent flow analysis."""
# Truncate content to minimize tokens while keeping context
prev_truncated = (previous_text[-300:] if previous_text else "") if previous_text else ""
curr_truncated = current_text[:500] # First 500 chars usually contain the key content
prompt = f"""
Analyze the narrative flow between these two content sections. Rate each aspect from 0.0 to 1.0.
PREVIOUS SECTION (end): {prev_truncated}
CURRENT SECTION (start): {curr_truncated}
Evaluate:
1. Flow Quality (0.0-1.0): How smoothly does the content transition? Are there logical connections?
2. Consistency (0.0-1.0): Do key themes, terminology, and tone remain consistent?
3. Progression (0.0-1.0): Does the content logically build upon previous ideas?
Return ONLY a JSON object with these exact keys: flow, consistency, progression
"""
schema = {
"type": "object",
"properties": {
"flow": {"type": "number", "minimum": 0.0, "maximum": 1.0},
"consistency": {"type": "number", "minimum": 0.0, "maximum": 1.0},
"progression": {"type": "number", "minimum": 0.0, "maximum": 1.0}
},
"required": ["flow", "consistency", "progression"]
}
try:
result = gemini_structured_json_response(
prompt=prompt,
schema=schema,
temperature=0.2, # Low temperature for consistent scoring
max_tokens=1000 # Increased tokens for better analysis
)
if result.parsed:
return {
"flow": float(result.parsed.get("flow", 0.6)),
"consistency": float(result.parsed.get("consistency", 0.6)),
"progression": float(result.parsed.get("progression", 0.6))
}
else:
logger.warning("LLM response parsing failed, using fallback")
return self._rule_based_analysis(previous_text, current_text)
except Exception as e:
logger.error(f"LLM flow analysis error: {e}")
return self._rule_based_analysis(previous_text, current_text)
def _rule_based_analysis(self, previous_text: str, current_text: str) -> Dict[str, float]:
"""Fallback rule-based analysis for cost efficiency."""
flow = 0.6
consistency = 0.6
progression = 0.6
# Enhanced heuristics
if previous_text and previous_text[-1] in ".!?":
flow += 0.1
if any(k in current_text.lower() for k in ["therefore", "next", "building on", "as a result", "furthermore", "additionally"]):
progression += 0.2
if len(current_text.split()) > 120:
consistency += 0.1
if any(k in current_text.lower() for k in ["however", "but", "although", "despite"]):
flow += 0.1 # Good use of contrast words
return {
"flow": min(flow, 1.0),
"consistency": min(consistency, 1.0),
"progression": min(progression, 1.0),
}
def _get_cache_key(self, previous_text: str, current_text: str) -> str:
"""Generate cache key from content hashes."""
# Use first 100 chars of each for cache key to balance uniqueness vs memory
prev_hash = hashlib.md5((previous_text[:100] if previous_text else "").encode()).hexdigest()[:8]
curr_hash = hashlib.md5(current_text[:100].encode()).hexdigest()[:8]
return f"{prev_hash}_{curr_hash}"
def clear_cache(self):
"""Clear analysis cache (useful for testing or memory management)."""
self._cache.clear()
self._rule_cache.clear()
logger.info("FlowAnalyzer cache cleared")

View File

@@ -0,0 +1,42 @@
"""
SourceURLManager - selects the most relevant source URLs for a section.
Low-effort heuristic using keywords and titles; safe defaults if no research.
"""
from typing import List, Dict, Any
class SourceURLManager:
def pick_relevant_urls(self, section: Any, research: Any, limit: int = 5) -> List[str]:
if not research or not getattr(research, 'sources', None):
return []
section_keywords = set([k.lower() for k in getattr(section, 'keywords', [])])
scored: List[tuple[float, str]] = []
for s in research.sources:
url = getattr(s, 'url', None) or getattr(s, 'uri', None) or s.get('url') if isinstance(s, dict) else None
title = getattr(s, 'title', None) or s.get('title') if isinstance(s, dict) else ''
if not url or not isinstance(url, str):
continue
title_l = (title or '').lower()
# simple overlap score
score = 0.0
for kw in section_keywords:
if kw and kw in title_l:
score += 1.0
# prefer https and reputable domains lightly
if url.startswith('https://'):
score += 0.2
scored.append((score, url))
scored.sort(key=lambda x: x[0], reverse=True)
dedup: List[str] = []
for _, u in scored:
if u not in dedup:
dedup.append(u)
if len(dedup) >= limit:
break
return dedup

View File

@@ -0,0 +1,143 @@
"""
TransitionGenerator - produces intelligent transitions between sections using LLM analysis.
Uses Gemini API for natural transitions while maintaining cost efficiency through smart caching.
"""
from typing import Optional, Dict
from loguru import logger
import hashlib
# Import the common gemini provider
from services.llm_providers.gemini_provider import gemini_text_response
class TransitionGenerator:
def __init__(self):
# Simple cache to avoid redundant LLM calls for similar transitions
self._cache: Dict[str, str] = {}
logger.info("✅ TransitionGenerator initialized with LLM-based generation")
def generate_transition(self, previous_text: str, current_heading: str, use_llm: bool = True) -> str:
"""
Return a 12 sentence bridge from previous_text into current_heading.
Args:
previous_text: Previous section content
current_heading: Current section heading
use_llm: Whether to use LLM generation (default: True for substantial content)
"""
prev = (previous_text or "").strip()
if not prev:
return f"Let's explore {current_heading.lower()} next."
# Create cache key
cache_key = self._get_cache_key(prev, current_heading)
# Check cache first
if cache_key in self._cache:
logger.debug("Transition generation cache hit")
return self._cache[cache_key]
# Determine if we should use LLM
should_use_llm = use_llm and self._should_use_llm_generation(prev, current_heading)
if should_use_llm:
try:
transition = self._llm_generate_transition(prev, current_heading)
self._cache[cache_key] = transition
logger.info("LLM-based transition generated")
return transition
except Exception as e:
logger.warning(f"LLM transition generation failed, using fallback: {e}")
# Fall through to heuristic generation
# Heuristic fallback
transition = self._heuristic_transition(prev, current_heading)
self._cache[cache_key] = transition
return transition
def _should_use_llm_generation(self, previous_text: str, current_heading: str) -> bool:
"""Determine if content is substantial enough to warrant LLM generation."""
# Use LLM for substantial previous content (>100 words) or complex headings
word_count = len(previous_text.split())
complex_heading = len(current_heading.split()) > 2 or any(char in current_heading for char in [':', '-', '&'])
return word_count > 100 or complex_heading
def _llm_generate_transition(self, previous_text: str, current_heading: str) -> str:
"""Use Gemini API for intelligent transition generation."""
# Truncate previous text to minimize tokens while keeping context
prev_truncated = previous_text[-200:] # Last 200 chars usually contain the conclusion
prompt = f"""
Create a smooth, natural 1-2 sentence transition from the previous content to the new section.
PREVIOUS CONTENT (ending): {prev_truncated}
NEW SECTION HEADING: {current_heading}
Requirements:
- Write exactly 1-2 sentences
- Create a logical bridge between the topics
- Use natural, engaging language
- Avoid repetition of the previous content
- Lead smoothly into the new section topic
Generate only the transition text, no explanations or formatting.
"""
try:
result = gemini_text_response(
prompt=prompt,
temperature=0.6, # Balanced creativity and consistency
max_tokens=300, # Increased tokens for better transitions
system_prompt="You are an expert content writer creating smooth transitions between sections."
)
if result and result.strip():
# Clean up the response
transition = result.strip()
# Ensure it's 1-2 sentences
sentences = transition.split('. ')
if len(sentences) > 2:
transition = '. '.join(sentences[:2]) + '.'
return transition
else:
logger.warning("LLM transition response empty, using fallback")
return self._heuristic_transition(previous_text, current_heading)
except Exception as e:
logger.error(f"LLM transition generation error: {e}")
return self._heuristic_transition(previous_text, current_heading)
def _heuristic_transition(self, previous_text: str, current_heading: str) -> str:
"""Fallback heuristic-based transition generation."""
tail = previous_text[-240:]
# Enhanced heuristics based on content patterns
if any(word in tail.lower() for word in ["problem", "issue", "challenge"]):
return f"Now that we've identified the challenges, let's explore {current_heading.lower()} to find solutions."
elif any(word in tail.lower() for word in ["solution", "approach", "method"]):
return f"Building on this approach, {current_heading.lower()} provides the next step in our analysis."
elif any(word in tail.lower() for word in ["important", "crucial", "essential"]):
return f"Given this importance, {current_heading.lower()} becomes our next focus area."
else:
return (
f"Building on the discussion above, this leads us into {current_heading.lower()}, "
f"where we focus on practical implications and what to do next."
)
def _get_cache_key(self, previous_text: str, current_heading: str) -> str:
"""Generate cache key from content hashes."""
# Use last 100 chars of previous text and heading for cache key
prev_hash = hashlib.md5(previous_text[-100:].encode()).hexdigest()[:8]
heading_hash = hashlib.md5(current_heading.encode()).hexdigest()[:8]
return f"{prev_hash}_{heading_hash}"
def clear_cache(self):
"""Clear transition cache (useful for testing or memory management)."""
self._cache.clear()
logger.info("TransitionGenerator cache cleared")

View File

@@ -28,6 +28,7 @@ from models.blog_models import (
from ..research import ResearchService
from ..outline import OutlineService
from ..content.enhanced_content_generator import EnhancedContentGenerator
class BlogWriterService:
@@ -36,6 +37,7 @@ class BlogWriterService:
def __init__(self):
self.research_service = ResearchService()
self.outline_service = OutlineService()
self.content_generator = EnhancedContentGenerator()
# Research Methods
async def research(self, request: BlogResearchRequest) -> BlogResearchResponse:
@@ -71,12 +73,37 @@ class BlogWriterService:
"""Rebalance word count distribution across sections."""
return self.outline_service.rebalance_word_counts(outline, target_words)
# Content Generation Methods (TODO: Extract to content module)
# Content Generation Methods
async def generate_section(self, request: BlogSectionRequest) -> BlogSectionResponse:
"""Generate section content from outline."""
# TODO: Move to content module
md = f"## {request.section.heading}\n\nThis section content will be generated here.\n"
return BlogSectionResponse(success=True, markdown=md, citations=request.section.references)
# Compose research-lite object with minimal continuity summary if available
research_ctx: Any = getattr(request, 'research', None)
try:
ai_result = await self.content_generator.generate_section(
section=request.section,
research=research_ctx,
mode=(request.mode or "polished"),
)
markdown = ai_result.get('content') or ai_result.get('markdown') or ''
citations = []
# Map basic citations from sources if present
for s in ai_result.get('sources', [])[:5]:
citations.append({
"title": s.get('title') if isinstance(s, dict) else getattr(s, 'title', ''),
"url": s.get('url') if isinstance(s, dict) else getattr(s, 'url', ''),
})
if not markdown:
markdown = f"## {request.section.heading}\n\n(Generated content was empty.)"
return BlogSectionResponse(
success=True,
markdown=markdown,
citations=citations,
continuity_metrics=ai_result.get('continuity_metrics')
)
except Exception as e:
logger.error(f"Section generation failed: {e}")
fallback = f"## {request.section.heading}\n\nThis section will cover: {', '.join(request.section.key_points)}."
return BlogSectionResponse(success=False, markdown=fallback, citations=[])
async def optimize_section(self, request: BlogOptimizeRequest) -> BlogOptimizeResponse:
"""Optimize section content for readability and SEO."""

View File

@@ -59,13 +59,15 @@ class CompetitorAnalyzer:
prompt=competitor_prompt,
schema=competitor_schema,
temperature=0.3,
max_tokens=1000
max_tokens=4000
)
if isinstance(competitor_analysis, dict) and 'error' not in competitor_analysis:
logger.info("✅ AI competitor analysis completed successfully")
return competitor_analysis
else:
# Fail gracefully - no fallback data
logger.error(f"AI competitor analysis failed: {competitor_analysis}")
raise ValueError(f"Competitor analysis failed: {competitor_analysis.get('error', 'Unknown error')}")
error_msg = competitor_analysis.get('error', 'Unknown error') if isinstance(competitor_analysis, dict) else str(competitor_analysis)
logger.error(f"AI competitor analysis failed: {error_msg}")
raise ValueError(f"Competitor analysis failed: {error_msg}")

View File

@@ -67,13 +67,15 @@ class ContentAngleGenerator:
prompt=angles_prompt,
schema=angles_schema,
temperature=0.7,
max_tokens=800
max_tokens=4000
)
if isinstance(angles_result, dict) and 'content_angles' in angles_result:
logger.info("✅ AI content angles generation completed successfully")
return angles_result['content_angles'][:7]
else:
# Fail gracefully - no fallback data
logger.error(f"AI content angles generation failed: {angles_result}")
raise ValueError(f"Content angles generation failed: {angles_result.get('error', 'Unknown error')}")
error_msg = angles_result.get('error', 'Unknown error') if isinstance(angles_result, dict) else str(angles_result)
logger.error(f"AI content angles generation failed: {error_msg}")
raise ValueError(f"Content angles generation failed: {error_msg}")

View File

@@ -66,13 +66,15 @@ class KeywordAnalyzer:
prompt=keyword_prompt,
schema=keyword_schema,
temperature=0.3,
max_tokens=1000
max_tokens=4000
)
if isinstance(keyword_analysis, dict) and 'error' not in keyword_analysis:
logger.info("✅ AI keyword analysis completed successfully")
return keyword_analysis
else:
# Fail gracefully - no fallback data
logger.error(f"AI keyword analysis failed: {keyword_analysis}")
raise ValueError(f"Keyword analysis failed: {keyword_analysis.get('error', 'Unknown error')}")
error_msg = keyword_analysis.get('error', 'Unknown error') if isinstance(keyword_analysis, dict) else str(keyword_analysis)
logger.error(f"AI keyword analysis failed: {error_msg}")
raise ValueError(f"Keyword analysis failed: {error_msg}")