Added enhanced linguistic analyzer and persona quality improver

backend/services/persona/enhanced_linguistic_analyzer.py

@@ -0,0 +1,629 @@
+"""
+Enhanced Linguistic Analysis Service
+Advanced analysis for better writing style mimicry and persona quality.
+"""
+
+import re
+import json
+from typing import Dict, Any, List, Tuple
+from collections import Counter, defaultdict
+from loguru import logger
+import nltk
+from nltk.tokenize import sent_tokenize, word_tokenize
+from nltk.corpus import stopwords
+from nltk.tag import pos_tag
+from textstat import flesch_reading_ease, flesch_kincaid_grade
+import spacy
+
+class EnhancedLinguisticAnalyzer:
+    """Advanced linguistic analysis for persona creation and improvement."""
+    
+    def __init__(self):
+        """Initialize the linguistic analyzer."""
+        self.nlp = None
+        try:
+            # Try to load spaCy model
+            self.nlp = spacy.load("en_core_web_sm")
+        except OSError:
+            logger.warning("spaCy model not found. Install with: python -m spacy download en_core_web_sm")
+        
+        # Download required NLTK data
+        try:
+            nltk.data.find('tokenizers/punkt')
+            nltk.data.find('corpora/stopwords')
+            nltk.data.find('taggers/averaged_perceptron_tagger')
+        except LookupError:
+            logger.warning("NLTK data not found. Downloading required data...")
+            nltk.download('punkt', quiet=True)
+            nltk.download('stopwords', quiet=True)
+            nltk.download('averaged_perceptron_tagger', quiet=True)
+    
+    def analyze_writing_style(self, text_samples: List[str]) -> Dict[str, Any]:
+        """
+        Comprehensive analysis of writing style from multiple text samples.
+        
+        Args:
+            text_samples: List of text samples to analyze
+            
+        Returns:
+            Detailed linguistic analysis
+        """
+        try:
+            logger.info(f"Analyzing writing style from {len(text_samples)} text samples")
+            
+            # Combine all text samples
+            combined_text = " ".join(text_samples)
+            
+            # Basic metrics
+            basic_metrics = self._analyze_basic_metrics(combined_text)
+            
+            # Sentence analysis
+            sentence_analysis = self._analyze_sentence_patterns(combined_text)
+            
+            # Vocabulary analysis
+            vocabulary_analysis = self._analyze_vocabulary(combined_text)
+            
+            # Rhetorical analysis
+            rhetorical_analysis = self._analyze_rhetorical_devices(combined_text)
+            
+            # Style patterns
+            style_patterns = self._analyze_style_patterns(combined_text)
+            
+            # Readability analysis
+            readability_analysis = self._analyze_readability(combined_text)
+            
+            # Emotional tone analysis
+            emotional_analysis = self._analyze_emotional_tone(combined_text)
+            
+            # Consistency analysis
+            consistency_analysis = self._analyze_consistency(text_samples)
+            
+            return {
+                "basic_metrics": basic_metrics,
+                "sentence_analysis": sentence_analysis,
+                "vocabulary_analysis": vocabulary_analysis,
+                "rhetorical_analysis": rhetorical_analysis,
+                "style_patterns": style_patterns,
+                "readability_analysis": readability_analysis,
+                "emotional_analysis": emotional_analysis,
+                "consistency_analysis": consistency_analysis,
+                "analysis_metadata": {
+                    "sample_count": len(text_samples),
+                    "total_words": basic_metrics["total_words"],
+                    "total_sentences": basic_metrics["total_sentences"],
+                    "analysis_confidence": self._calculate_analysis_confidence(text_samples)
+                }
+            }
+            
+        except Exception as e:
+            logger.error(f"Error analyzing writing style: {str(e)}")
+            return {"error": f"Failed to analyze writing style: {str(e)}"}
+    
+    def _analyze_basic_metrics(self, text: str) -> Dict[str, Any]:
+        """Analyze basic text metrics."""
+        sentences = sent_tokenize(text)
+        words = word_tokenize(text.lower())
+        
+        # Filter out punctuation
+        words = [word for word in words if word.isalpha()]
+        
+        return {
+            "total_words": len(words),
+            "total_sentences": len(sentences),
+            "average_sentence_length": len(words) / len(sentences) if sentences else 0,
+            "average_word_length": sum(len(word) for word in words) / len(words) if words else 0,
+            "paragraph_count": len(text.split('\n\n')),
+            "character_count": len(text),
+            "character_count_no_spaces": len(text.replace(' ', ''))
+        }
+    
+    def _analyze_sentence_patterns(self, text: str) -> Dict[str, Any]:
+        """Analyze sentence structure patterns."""
+        sentences = sent_tokenize(text)
+        
+        sentence_lengths = [len(word_tokenize(sent)) for sent in sentences]
+        sentence_types = []
+        
+        for sentence in sentences:
+            if sentence.endswith('?'):
+                sentence_types.append('question')
+            elif sentence.endswith('!'):
+                sentence_types.append('exclamation')
+            else:
+                sentence_types.append('declarative')
+        
+        # Analyze sentence beginnings
+        sentence_beginnings = []
+        for sentence in sentences:
+            first_word = word_tokenize(sentence)[0].lower() if word_tokenize(sentence) else ""
+            sentence_beginnings.append(first_word)
+        
+        return {
+            "sentence_length_distribution": {
+                "min": min(sentence_lengths) if sentence_lengths else 0,
+                "max": max(sentence_lengths) if sentence_lengths else 0,
+                "average": sum(sentence_lengths) / len(sentence_lengths) if sentence_lengths else 0,
+                "median": sorted(sentence_lengths)[len(sentence_lengths)//2] if sentence_lengths else 0
+            },
+            "sentence_type_distribution": dict(Counter(sentence_types)),
+            "common_sentence_starters": dict(Counter(sentence_beginnings).most_common(10)),
+            "sentence_complexity": self._analyze_sentence_complexity(sentences)
+        }
+    
+    def _analyze_vocabulary(self, text: str) -> Dict[str, Any]:
+        """Analyze vocabulary patterns and preferences."""
+        words = word_tokenize(text.lower())
+        words = [word for word in words if word.isalpha()]
+        
+        # Remove stopwords for analysis
+        stop_words = set(stopwords.words('english'))
+        content_words = [word for word in words if word not in stop_words]
+        
+        # POS tagging
+        pos_tags = pos_tag(words)
+        pos_distribution = dict(Counter(tag for word, tag in pos_tags))
+        
+        # Vocabulary richness
+        unique_words = set(words)
+        unique_content_words = set(content_words)
+        
+        return {
+            "vocabulary_size": len(unique_words),
+            "content_vocabulary_size": len(unique_content_words),
+            "lexical_diversity": len(unique_words) / len(words) if words else 0,
+            "most_frequent_words": dict(Counter(words).most_common(20)),
+            "most_frequent_content_words": dict(Counter(content_words).most_common(20)),
+            "pos_distribution": pos_distribution,
+            "word_length_distribution": {
+                "short_words": len([w for w in words if len(w) <= 4]),
+                "medium_words": len([w for w in words if 5 <= len(w) <= 8]),
+                "long_words": len([w for w in words if len(w) > 8])
+            },
+            "vocabulary_sophistication": self._analyze_vocabulary_sophistication(words)
+        }
+    
+    def _analyze_rhetorical_devices(self, text: str) -> Dict[str, Any]:
+        """Analyze rhetorical devices and techniques."""
+        sentences = sent_tokenize(text)
+        
+        rhetorical_devices = {
+            "questions": len([s for s in sentences if s.strip().endswith('?')]),
+            "exclamations": len([s for s in sentences if s.strip().endswith('!')]),
+            "repetition": self._find_repetition_patterns(text),
+            "alliteration": self._find_alliteration(text),
+            "metaphors": self._find_metaphors(text),
+            "analogies": self._find_analogies(text),
+            "lists": self._find_lists(text),
+            "contrasts": self._find_contrasts(text)
+        }
+        
+        return rhetorical_devices
+    
+    def _analyze_style_patterns(self, text: str) -> Dict[str, Any]:
+        """Analyze writing style patterns."""
+        return {
+            "formality_level": self._assess_formality(text),
+            "personal_pronouns": self._count_personal_pronouns(text),
+            "passive_voice": self._count_passive_voice(text),
+            "contractions": self._count_contractions(text),
+            "transition_words": self._find_transition_words(text),
+            "hedging_language": self._find_hedging_language(text),
+            "emphasis_patterns": self._find_emphasis_patterns(text)
+        }
+    
+    def _analyze_readability(self, text: str) -> Dict[str, Any]:
+        """Analyze readability metrics."""
+        try:
+            return {
+                "flesch_reading_ease": flesch_reading_ease(text),
+                "flesch_kincaid_grade": flesch_kincaid_grade(text),
+                "reading_level": self._determine_reading_level(flesch_reading_ease(text)),
+                "complexity_score": self._calculate_complexity_score(text)
+            }
+        except Exception as e:
+            logger.warning(f"Error calculating readability: {e}")
+            return {"error": "Could not calculate readability metrics"}
+    
+    def _analyze_emotional_tone(self, text: str) -> Dict[str, Any]:
+        """Analyze emotional tone and sentiment patterns."""
+        # Simple sentiment analysis based on word patterns
+        positive_words = ['good', 'great', 'excellent', 'amazing', 'wonderful', 'fantastic', 'love', 'like', 'enjoy']
+        negative_words = ['bad', 'terrible', 'awful', 'hate', 'dislike', 'horrible', 'worst', 'problem', 'issue']
+        
+        words = word_tokenize(text.lower())
+        positive_count = sum(1 for word in words if word in positive_words)
+        negative_count = sum(1 for word in words if word in negative_words)
+        
+        return {
+            "sentiment_bias": "positive" if positive_count > negative_count else "negative" if negative_count > positive_count else "neutral",
+            "positive_word_count": positive_count,
+            "negative_word_count": negative_count,
+            "emotional_intensity": self._calculate_emotional_intensity(text),
+            "tone_consistency": self._assess_tone_consistency(text)
+        }
+    
+    def _analyze_consistency(self, text_samples: List[str]) -> Dict[str, Any]:
+        """Analyze consistency across multiple text samples."""
+        if len(text_samples) < 2:
+            return {"consistency_score": 100, "note": "Only one sample provided"}
+        
+        # Analyze consistency in various metrics
+        sentence_lengths = []
+        vocabulary_sets = []
+        
+        for sample in text_samples:
+            sentences = sent_tokenize(sample)
+            words = word_tokenize(sample.lower())
+            words = [word for word in words if word.isalpha()]
+            
+            sentence_lengths.append([len(word_tokenize(sent)) for sent in sentences])
+            vocabulary_sets.append(set(words))
+        
+        # Calculate consistency scores
+        avg_sentence_length_consistency = self._calculate_metric_consistency(
+            [sum(lengths)/len(lengths) for lengths in sentence_lengths]
+        )
+        
+        vocabulary_overlap = self._calculate_vocabulary_overlap(vocabulary_sets)
+        
+        return {
+            "consistency_score": (avg_sentence_length_consistency + vocabulary_overlap) / 2,
+            "sentence_length_consistency": avg_sentence_length_consistency,
+            "vocabulary_consistency": vocabulary_overlap,
+            "style_stability": self._assess_style_stability(text_samples)
+        }
+    
+    def _calculate_analysis_confidence(self, text_samples: List[str]) -> float:
+        """Calculate confidence in the analysis based on data quality."""
+        if not text_samples:
+            return 0.0
+        
+        total_words = sum(len(word_tokenize(sample)) for sample in text_samples)
+        sample_count = len(text_samples)
+        
+        # Confidence based on amount of data
+        word_confidence = min(100, (total_words / 1000) * 100)  # 1000 words = 100% confidence
+        sample_confidence = min(100, (sample_count / 5) * 100)  # 5 samples = 100% confidence
+        
+        return (word_confidence + sample_confidence) / 2
+    
+    # Helper methods for specific analyses
+    def _analyze_sentence_complexity(self, sentences: List[str]) -> Dict[str, Any]:
+        """Analyze sentence complexity patterns."""
+        complex_sentences = 0
+        compound_sentences = 0
+        
+        for sentence in sentences:
+            if ',' in sentence and ('and' in sentence or 'but' in sentence or 'or' in sentence):
+                compound_sentences += 1
+            if len(word_tokenize(sentence)) > 20:
+                complex_sentences += 1
+        
+        return {
+            "complex_sentence_ratio": complex_sentences / len(sentences) if sentences else 0,
+            "compound_sentence_ratio": compound_sentences / len(sentences) if sentences else 0,
+            "average_clauses_per_sentence": self._count_clauses(sentences)
+        }
+    
+    def _analyze_vocabulary_sophistication(self, words: List[str]) -> Dict[str, Any]:
+        """Analyze vocabulary sophistication level."""
+        # Simple heuristic based on word length and frequency
+        long_words = [w for w in words if len(w) > 7]
+        rare_words = [w for w in words if len(w) > 5]  # Simplified rare word detection
+        
+        return {
+            "sophistication_score": (len(long_words) + len(rare_words)) / len(words) * 100 if words else 0,
+            "long_word_ratio": len(long_words) / len(words) if words else 0,
+            "rare_word_ratio": len(rare_words) / len(words) if words else 0
+        }
+    
+    def _find_repetition_patterns(self, text: str) -> Dict[str, Any]:
+        """Find repetition patterns in text."""
+        words = word_tokenize(text.lower())
+        word_freq = Counter(words)
+        
+        # Find words that appear multiple times
+        repeated_words = {word: count for word, count in word_freq.items() if count > 2}
+        
+        return {
+            "repeated_words": repeated_words,
+            "repetition_score": len(repeated_words) / len(set(words)) * 100 if words else 0
+        }
+    
+    def _find_alliteration(self, text: str) -> List[str]:
+        """Find alliteration patterns."""
+        sentences = sent_tokenize(text)
+        alliterations = []
+        
+        for sentence in sentences:
+            words = word_tokenize(sentence.lower())
+            words = [word for word in words if word.isalpha()]
+            
+            if len(words) >= 2:
+                for i in range(len(words) - 1):
+                    if words[i][0] == words[i+1][0]:
+                        alliterations.append(f"{words[i]} {words[i+1]}")
+        
+        return alliterations
+    
+    def _find_metaphors(self, text: str) -> List[str]:
+        """Find potential metaphors in text."""
+        # Simple metaphor detection based on common patterns
+        metaphor_patterns = [
+            r'\b(is|are|was|were)\s+(like|as)\s+',
+            r'\b(like|as)\s+\w+\s+(is|are|was|were)',
+            r'\b(metaphorically|figuratively)'
+        ]
+        
+        metaphors = []
+        for pattern in metaphor_patterns:
+            matches = re.findall(pattern, text, re.IGNORECASE)
+            metaphors.extend(matches)
+        
+        return metaphors
+    
+    def _find_analogies(self, text: str) -> List[str]:
+        """Find analogies in text."""
+        analogy_patterns = [
+            r'\b(just as|similar to|comparable to|akin to)',
+            r'\b(in the same way|likewise|similarly)'
+        ]
+        
+        analogies = []
+        for pattern in analogy_patterns:
+            matches = re.findall(pattern, text, re.IGNORECASE)
+            analogies.extend(matches)
+        
+        return analogies
+    
+    def _find_lists(self, text: str) -> List[str]:
+        """Find list patterns in text."""
+        list_patterns = [
+            r'\b(first|second|third|lastly|finally)',
+            r'\b(one|two|three|four|five)',
+            r'\b(•|\*|\-|\d+\.)'
+        ]
+        
+        lists = []
+        for pattern in list_patterns:
+            matches = re.findall(pattern, text, re.IGNORECASE)
+            lists.extend(matches)
+        
+        return lists
+    
+    def _find_contrasts(self, text: str) -> List[str]:
+        """Find contrast patterns in text."""
+        contrast_words = ['but', 'however', 'although', 'whereas', 'while', 'on the other hand', 'in contrast']
+        contrasts = []
+        
+        for word in contrast_words:
+            if word in text.lower():
+                contrasts.append(word)
+        
+        return contrasts
+    
+    def _assess_formality(self, text: str) -> str:
+        """Assess formality level of text."""
+        formal_indicators = ['therefore', 'furthermore', 'moreover', 'consequently', 'nevertheless']
+        informal_indicators = ['gonna', 'wanna', 'gotta', 'yeah', 'ok', 'cool']
+        
+        formal_count = sum(1 for indicator in formal_indicators if indicator in text.lower())
+        informal_count = sum(1 for indicator in informal_indicators if indicator in text.lower())
+        
+        if formal_count > informal_count:
+            return "formal"
+        elif informal_count > formal_count:
+            return "informal"
+        else:
+            return "neutral"
+    
+    def _count_personal_pronouns(self, text: str) -> Dict[str, int]:
+        """Count personal pronouns in text."""
+        pronouns = ['i', 'me', 'my', 'mine', 'myself', 'we', 'us', 'our', 'ours', 'ourselves', 
+                   'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself',
+                   'she', 'her', 'hers', 'herself', 'they', 'them', 'their', 'theirs', 'themselves']
+        
+        words = word_tokenize(text.lower())
+        pronoun_count = {pronoun: words.count(pronoun) for pronoun in pronouns}
+        
+        return pronoun_count
+    
+    def _count_passive_voice(self, text: str) -> int:
+        """Count passive voice constructions."""
+        passive_patterns = [
+            r'\b(was|were|is|are|been|being)\s+\w+ed\b',
+            r'\b(was|were|is|are|been|being)\s+\w+en\b'
+        ]
+        
+        passive_count = 0
+        for pattern in passive_patterns:
+            passive_count += len(re.findall(pattern, text, re.IGNORECASE))
+        
+        return passive_count
+    
+    def _count_contractions(self, text: str) -> int:
+        """Count contractions in text."""
+        contraction_pattern = r"\b\w+'\w+\b"
+        return len(re.findall(contraction_pattern, text))
+    
+    def _find_transition_words(self, text: str) -> List[str]:
+        """Find transition words in text."""
+        transition_words = ['however', 'therefore', 'furthermore', 'moreover', 'nevertheless', 
+                          'consequently', 'meanwhile', 'additionally', 'similarly', 'likewise',
+                          'on the other hand', 'in contrast', 'for example', 'for instance']
+        
+        found_transitions = []
+        for word in transition_words:
+            if word in text.lower():
+                found_transitions.append(word)
+        
+        return found_transitions
+    
+    def _find_hedging_language(self, text: str) -> List[str]:
+        """Find hedging language in text."""
+        hedging_words = ['might', 'could', 'possibly', 'perhaps', 'maybe', 'likely', 'probably',
+                        'seems', 'appears', 'suggests', 'indicates', 'tends to']
+        
+        found_hedging = []
+        for word in hedging_words:
+            if word in text.lower():
+                found_hedging.append(word)
+        
+        return found_hedging
+    
+    def _find_emphasis_patterns(self, text: str) -> Dict[str, Any]:
+        """Find emphasis patterns in text."""
+        emphasis_patterns = {
+            'bold_asterisks': len(re.findall(r'\*\w+\*', text)),
+            'bold_underscores': len(re.findall(r'_\w+_', text)),
+            'caps_words': len(re.findall(r'\b[A-Z]{2,}\b', text)),
+            'exclamation_points': text.count('!'),
+            'emphasis_words': len(re.findall(r'\b(very|really|extremely|absolutely|completely)\b', text, re.IGNORECASE))
+        }
+        
+        return emphasis_patterns
+    
+    def _determine_reading_level(self, flesch_score: float) -> str:
+        """Determine reading level from Flesch score."""
+        if flesch_score >= 90:
+            return "very_easy"
+        elif flesch_score >= 80:
+            return "easy"
+        elif flesch_score >= 70:
+            return "fairly_easy"
+        elif flesch_score >= 60:
+            return "standard"
+        elif flesch_score >= 50:
+            return "fairly_difficult"
+        elif flesch_score >= 30:
+            return "difficult"
+        else:
+            return "very_difficult"
+    
+    def _calculate_complexity_score(self, text: str) -> float:
+        """Calculate overall complexity score."""
+        sentences = sent_tokenize(text)
+        words = word_tokenize(text.lower())
+        words = [word for word in words if word.isalpha()]
+        
+        if not sentences or not words:
+            return 0.0
+        
+        # Factors: sentence length, word length, vocabulary diversity
+        avg_sentence_length = len(words) / len(sentences)
+        avg_word_length = sum(len(word) for word in words) / len(words)
+        vocabulary_diversity = len(set(words)) / len(words)
+        
+        # Normalize and combine
+        complexity = (avg_sentence_length / 20) * 0.4 + (avg_word_length / 10) * 0.3 + vocabulary_diversity * 0.3
+        
+        return min(100, complexity * 100)
+    
+    def _calculate_emotional_intensity(self, text: str) -> float:
+        """Calculate emotional intensity of text."""
+        emotional_words = ['amazing', 'incredible', 'fantastic', 'terrible', 'awful', 'horrible', 
+                          'love', 'hate', 'passion', 'fury', 'joy', 'sorrow', 'excitement', 'fear']
+        
+        words = word_tokenize(text.lower())
+        emotional_word_count = sum(1 for word in words if word in emotional_words)
+        
+        return (emotional_word_count / len(words)) * 100 if words else 0
+    
+    def _assess_tone_consistency(self, text: str) -> float:
+        """Assess tone consistency throughout text."""
+        # Simple heuristic: check for tone shifts
+        sentences = sent_tokenize(text)
+        if len(sentences) < 2:
+            return 100.0
+        
+        # Analyze first half vs second half
+        mid_point = len(sentences) // 2
+        first_half = " ".join(sentences[:mid_point])
+        second_half = " ".join(sentences[mid_point:])
+        
+        first_tone = self._analyze_emotional_tone(first_half)
+        second_tone = self._analyze_emotional_tone(second_half)
+        
+        # Calculate consistency based on sentiment similarity
+        if first_tone["sentiment_bias"] == second_tone["sentiment_bias"]:
+            return 100.0
+        else:
+            return 50.0
+    
+    def _calculate_metric_consistency(self, values: List[float]) -> float:
+        """Calculate consistency of a metric across samples."""
+        if len(values) < 2:
+            return 100.0
+        
+        mean_value = sum(values) / len(values)
+        variance = sum((x - mean_value) ** 2 for x in values) / len(values)
+        std_dev = variance ** 0.5
+        
+        # Convert to consistency score (lower std dev = higher consistency)
+        consistency = max(0, 100 - (std_dev / mean_value * 100)) if mean_value > 0 else 100
+        
+        return consistency
+    
+    def _calculate_vocabulary_overlap(self, vocabulary_sets: List[set]) -> float:
+        """Calculate vocabulary overlap across samples."""
+        if len(vocabulary_sets) < 2:
+            return 100.0
+        
+        # Calculate pairwise overlaps
+        overlaps = []
+        for i in range(len(vocabulary_sets)):
+            for j in range(i + 1, len(vocabulary_sets)):
+                intersection = len(vocabulary_sets[i] & vocabulary_sets[j])
+                union = len(vocabulary_sets[i] | vocabulary_sets[j])
+                overlap = (intersection / union * 100) if union > 0 else 0
+                overlaps.append(overlap)
+        
+        return sum(overlaps) / len(overlaps) if overlaps else 0
+    
+    def _assess_style_stability(self, text_samples: List[str]) -> Dict[str, Any]:
+        """Assess style stability across samples."""
+        if len(text_samples) < 2:
+            return {"stability_score": 100, "note": "Only one sample provided"}
+        
+        # Analyze consistency in key style metrics
+        metrics = []
+        for sample in text_samples:
+            sample_metrics = {
+                "avg_sentence_length": len(word_tokenize(sample)) / len(sent_tokenize(sample)),
+                "formality": self._assess_formality(sample),
+                "emotional_intensity": self._calculate_emotional_intensity(sample)
+            }
+            metrics.append(sample_metrics)
+        
+        # Calculate stability scores
+        sentence_length_stability = self._calculate_metric_consistency(
+            [m["avg_sentence_length"] for m in metrics]
+        )
+        
+        emotional_stability = self._calculate_metric_consistency(
+            [m["emotional_intensity"] for m in metrics]
+        )
+        
+        # Formality consistency
+        formality_values = [m["formality"] for m in metrics]
+        formality_consistency = 100 if len(set(formality_values)) == 1 else 50
+        
+        overall_stability = (sentence_length_stability + emotional_stability + formality_consistency) / 3
+        
+        return {
+            "stability_score": overall_stability,
+            "sentence_length_stability": sentence_length_stability,
+            "emotional_stability": emotional_stability,
+            "formality_consistency": formality_consistency
+        }
+    
+    def _count_clauses(self, sentences: List[str]) -> float:
+        """Count average clauses per sentence."""
+        total_clauses = 0
+        for sentence in sentences:
+            # Simple clause counting based on conjunctions and punctuation
+            clauses = len(re.findall(r'[,;]', sentence)) + 1
+            total_clauses += clauses
+        
+        return total_clauses / len(sentences) if sentences else 0
+a