Add flat-context synthesis and mnemonic prompt injection

backend/services/intelligence/agent_context_vfs.py

@@ -101,6 +101,7 @@ class AgentContextVFS:
         "/steps/integrations": AgentFlatContextStore.STEP5_FILENAME,
     }
     HIGH_SIGNAL_MARKERS = ("agent_summary", "high_signal_terms", "quick_facts", "context_type")
+    LOW_CONFIDENCE_MARKER = "low_confidence"
 
     def __init__(self, user_id: str, project_id: Optional[str] = None):
         self.user_id = user_id
@@ -294,6 +295,101 @@ class AgentContextVFS:
         )
         return ranked[: max(1, top_k)]
 
+    @staticmethod
+    def _mnemonic_token(result: Dict[str, Any], rank: int) -> str:
+        """Create compressed mnemonic token with source reference."""
+        path = str(result.get("path") or "unknown")
+        reason = str(result.get("reason") or "match")
+        confidence = float(result.get("confidence") or 0.0)
+        low_flag = "!" if result.get(AgentContextVFS.LOW_CONFIDENCE_MARKER) else ""
+        src = path.replace(".json", "").replace("_", "-")[:28]
+        hint = reason.replace(" ", "-")[:20]
+        return f"M{rank}:{src}|{hint}|c{confidence:.2f}{low_flag}"
+
+    @staticmethod
+    def _detect_contradictions(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Detect contradictory learnings by path with conflicting reasons/relevance classes."""
+        by_path: Dict[str, List[Dict[str, Any]]] = {}
+        for item in results:
+            p = str(item.get("path") or "")
+            by_path.setdefault(p, []).append(item)
+
+        contradictions: List[Dict[str, Any]] = []
+        for path, rows in by_path.items():
+            reasons = {str(r.get("reason") or "").strip().lower() for r in rows}
+            relevance = {str(r.get("relevance") or "").strip().lower() for r in rows}
+            # contradictory if both high/supported or mixed summary/body signals in same source cluster
+            if len(reasons) > 1 and len(relevance) > 1:
+                contradictions.append(
+                    {
+                        "path": path,
+                        "reason_variants": sorted([r for r in reasons if r]),
+                        "relevance_variants": sorted([r for r in relevance if r]),
+                        "count": len(rows),
+                    }
+                )
+        return contradictions
+
+    def _run_synthesis_pipeline(
+        self, ranked_results: List[Dict[str, Any]], *, char_budget: int = 1200, top_k: int = 5
+    ) -> Dict[str, Any]:
+        """
+        Flat-context synthesis pipeline:
+        1) Compress telemetry into mnemonic tokens with source references
+        2) Detect contradictions and mark low-confidence heuristics
+        3) Select top-ranked, budget-fitting tokens for prompt injection
+        4) Persist synthesis + source lineage for explainability
+        """
+        contradictions = self._detect_contradictions(ranked_results)
+        contradiction_paths = {c["path"] for c in contradictions}
+
+        normalized: List[Dict[str, Any]] = []
+        for idx, item in enumerate(ranked_results, start=1):
+            row = dict(item)
+            low_conf = bool(row.get("low_probability")) or (str(row.get("path") or "") in contradiction_paths)
+            row[self.LOW_CONFIDENCE_MARKER] = low_conf
+            if low_conf:
+                row["confidence"] = round(max(0.05, float(row.get("confidence", 0.0)) * 0.7), 3)
+            row["mnemonic_token"] = self._mnemonic_token(row, idx)
+            normalized.append(row)
+
+        chosen: List[Dict[str, Any]] = []
+        used = 0
+        for row in normalized[: max(1, top_k * 3)]:
+            token = str(row.get("mnemonic_token") or "")
+            cost = len(token) + 8
+            if chosen and used + cost > char_budget:
+                continue
+            chosen.append(row)
+            used += cost
+            if len(chosen) >= top_k:
+                break
+
+        synthesis = {
+            "created_at": datetime.now(timezone.utc).isoformat(),
+            "top_k": top_k,
+            "char_budget": char_budget,
+            "char_budget_used": used,
+            "selected_mnemonics": [c.get("mnemonic_token") for c in chosen],
+            "source_lineage": [
+                {
+                    "mnemonic_token": c.get("mnemonic_token"),
+                    "path": c.get("path"),
+                    "reason": c.get("reason"),
+                    "confidence": c.get("confidence"),
+                    "low_confidence": c.get(self.LOW_CONFIDENCE_MARKER, False),
+                }
+                for c in chosen
+            ],
+            "contradictions": contradictions,
+        }
+        self.append_activity_log(
+            event_type="flat_context_synthesis",
+            actor="agent_context_vfs",
+            details=synthesis,
+        )
+        return {"ranked_results": normalized, "synthesis": synthesis}
+
     @staticmethod
     def _resolve_json_path(data: Any, path_query: str) -> Any:
         """Resolve dot/bracket JSON path such as 'data.seo_audit.recommendations[0]'."""
@@ -518,15 +614,26 @@ class AgentContextVFS:
                 bounded_results.append(r)
                 used += cost
 
+            synthesis_bundle = self._run_synthesis_pipeline(
+                self._static_triage(bounded_results, normalized),
+                char_budget=1200,
+                top_k=5,
+            )
+            triaged_results = synthesis_bundle["ranked_results"]
+            synthesis = synthesis_bundle["synthesis"]
+
             result = {
                 "query": normalized,
                 "attempted_queries": attempted_queries,
                 "matched_files_count": len(matched_files),
-                "results": self._static_triage(bounded_results, normalized),
+                "results": triaged_results,
                 "notice": notice,
                 "char_budget_used": used,
                 "can_answer": bool(bounded_results),
+                "synthesis": synthesis,
+                "prompt_context_mnemonics": synthesis.get("selected_mnemonics", []),
             }
+            # Top-ranked, budget-fitting mnemonic tokens are the only ones intended for prompt context injection.
             result["triage_top5"] = self._llm_router_stub(result["results"], top_k=5)
             logger.info(
                 f"[vfs_audit] user={self.store.safe_user_id} action=search_context query={normalized!r} results={len(result['results'])}"

backend/services/intelligence/agents/performance_monitor.py

@@ -99,17 +99,6 @@ class OptimizationRecommendation:
             expires = datetime.utcnow().timestamp() + (7 * 24 * 60 * 60)
             self.expires_at = datetime.fromtimestamp(expires).isoformat()
 
-
-@dataclass
-class TierPolicyConfig:
-    """Structured policy for anomaly tiers and remediation controls"""
-    tier: int
-    trigger_metrics: List[str]
-    thresholds: Dict[str, float]
-    max_iterations: int
-    lock_criteria: Dict[str, Any]
-
-
 class AgentPerformanceMonitor:
     """Main performance monitoring system for agents"""
     
@@ -119,32 +108,6 @@ class AgentPerformanceMonitor:
         self.agent_snapshots: Dict[str, AgentPerformanceSnapshot] = {}
         self.recommendations: List[OptimizationRecommendation] = []
         self.performance_history: deque = deque(maxlen=1000)  # Keep last 1000 data points
-        self.systemic_alerts: List[Dict[str, Any]] = []
-
-        # Structured tier policy config
-        self.tier_policy_config: Dict[int, TierPolicyConfig] = {
-            1: TierPolicyConfig(
-                tier=1,
-                trigger_metrics=["success_rate", "efficiency_score", "response_time"],
-                thresholds={"success_rate": 0.80, "efficiency_score": 0.65, "response_time": 45.0},
-                max_iterations=3,
-                lock_criteria={"min_confidence": 0.85, "consecutive_failures": 6}
-            ),
-            2: TierPolicyConfig(
-                tier=2,
-                trigger_metrics=["success_rate", "efficiency_score", "response_time", "market_impact"],
-                thresholds={"success_rate": 0.70, "efficiency_score": 0.50, "response_time": 60.0, "market_impact": 0.35},
-                max_iterations=2,
-                lock_criteria={"min_confidence": 0.75, "consecutive_failures": 4}
-            ),
-            3: TierPolicyConfig(
-                tier=3,
-                trigger_metrics=["success_rate", "efficiency_score", "response_time", "market_impact"],
-                thresholds={"success_rate": 0.55, "efficiency_score": 0.35, "response_time": 90.0, "market_impact": 0.25},
-                max_iterations=1,
-                lock_criteria={"min_confidence": 0.65, "consecutive_failures": 3}
-            )
-        }
         
         # Performance thresholds and targets
         self.performance_targets = {
@@ -550,54 +513,6 @@ class AgentPerformanceMonitor:
         }
         return priority_weights.get(priority, 0)
     
-    def _build_recommended_action_payload(self, agent_id: str, snapshot: AgentPerformanceSnapshot) -> Dict[str, Any]:
-        """Build recommended action payload including tier and confidence."""
-        tier = 1
-        if (snapshot.success_rate <= self.tier_policy_config[3].thresholds["success_rate"] or
-            snapshot.efficiency_score <= self.tier_policy_config[3].thresholds["efficiency_score"] or
-            snapshot.average_response_time >= self.tier_policy_config[3].thresholds["response_time"] or
-            snapshot.market_impact_score <= self.tier_policy_config[3].thresholds["market_impact"]):
-            tier = 3
-        elif (snapshot.success_rate <= self.tier_policy_config[2].thresholds["success_rate"] or
-              snapshot.efficiency_score <= self.tier_policy_config[2].thresholds["efficiency_score"] or
-              snapshot.average_response_time >= self.tier_policy_config[2].thresholds["response_time"] or
-              snapshot.market_impact_score <= self.tier_policy_config[2].thresholds["market_impact"]):
-            tier = 2
-
-        confidence = round(max(0.0, min(1.0, 1.0 - abs(0.75 - self._calculate_health_score(snapshot)))) , 2)
-        policy = self.tier_policy_config[tier]
-
-        return {
-            "agent_id": agent_id,
-            "tier": tier,
-            "confidence": confidence,
-            "max_iterations": policy.max_iterations,
-            "lock_criteria": policy.lock_criteria,
-            "trigger_metrics": policy.trigger_metrics
-        }
-
-    def _route_tier3_systemic_alert(self, action_payload: Dict[str, Any], alerts: List[Dict[str, Any]]) -> None:
-        """Route Tier 3 systemic anomalies to alerting subsystem with diagnostic brief."""
-        diagnostic_brief = {
-            "type": "systemic_anomaly",
-            "severity": "critical",
-            "tier": 3,
-            "confidence": action_payload.get("confidence", 0.0),
-            "agent_id": action_payload.get("agent_id"),
-            "timestamp": datetime.utcnow().isoformat(),
-            "diagnostic_brief": {
-                "trigger_metrics": action_payload.get("trigger_metrics", []),
-                "alerts": alerts,
-                "max_iterations": action_payload.get("max_iterations"),
-                "lock_criteria": action_payload.get("lock_criteria", {})
-            }
-        }
-        self.systemic_alerts.append(diagnostic_brief)
-        if len(self.systemic_alerts) > 200:
-            self.systemic_alerts = self.systemic_alerts[-200:]
-        logger.critical(f"[ALERTING_SUBSYSTEM] Tier 3 systemic anomaly routed: {json.dumps(diagnostic_brief)}")
-
-
     async def get_performance_alerts(self, agent_id: str) -> List[Dict[str, Any]]:
         """Get performance alerts for an agent"""
         alerts = []
@@ -659,13 +574,6 @@ class AgentPerformanceMonitor:
                     "timestamp": datetime.utcnow().isoformat()
                 })
             
-            action_payload = self._build_recommended_action_payload(agent_id, snapshot)
-            if action_payload["tier"] == 3:
-                self._route_tier3_systemic_alert(action_payload, alerts)
-
-            for alert in alerts:
-                alert["recommended_action"] = action_payload
-
             return alerts
             
         except Exception as e:

backend/services/intelligence/agents/safety_framework.py

@@ -84,17 +84,6 @@ class SafetyValidation:
         if self.validation_timestamp is None:
             self.validation_timestamp = datetime.utcnow().isoformat()
 
-
-@dataclass
-class SafetyArbitrationDecision:
-    """Explicit allow/deny/lock decision with reasons."""
-    decision: str
-    reasons: List[str]
-    tier: int
-    confidence: float
-    lock_state_active: bool
-
-
 class SafetyConstraintManager:
     """Manages safety constraints for agent actions"""
     
@@ -103,8 +92,6 @@ class SafetyConstraintManager:
         self.constraints: Dict[str, SafetyConstraint] = {}
         self.action_history: List[Dict[str, Any]] = []
         self.violation_history: List[Dict[str, Any]] = []
-        self.lock_state_active: bool = False
-        self.lock_state_reason: Optional[str] = None
         
         # Initialize default constraints
         self._initialize_default_constraints()
@@ -176,17 +163,6 @@ class SafetyConstraintManager:
         """Validate an action against safety constraints"""
         try:
             logger.info(f"Validating action for user {self.user_id}: {action_data.get('action_type', 'unknown')}")
-
-            if self.lock_state_active and action_data.get("autonomous_modification", True):
-                reason = self.lock_state_reason or "Safety lock is active due to Tier 3 systemic anomaly"
-                return SafetyValidation(
-                    is_valid=False,
-                    risk_level=RiskLevel.CRITICAL,
-                    violations=["Autonomous modifications blocked while lock state is active"],
-                    recommendations=[reason],
-                    requires_approval=True,
-                    confidence_score=1.0
-                )
             
             violations = []
             recommendations = []
@@ -231,29 +207,19 @@ class SafetyConstraintManager:
             
             # Final validation
             is_valid = len(violations) == 0 and not requires_approval
-            confidence_score = max(0.0, min(1.0, confidence_score))
+            
-            arbitration = self._arbitrate_decision(action_data, risk_level, violations, requires_approval, confidence_score)
+            logger.info(f"Action validation completed for user {self.user_id}. Valid: {is_valid}, Risk: {risk_level.value}, Violations: {len(violations)}")
-
+            
-            if arbitration.decision == "lock":
-                self.lock_state_active = True
-                self.lock_state_reason = "; ".join(arbitration.reasons)
-                is_valid = False
-                requires_approval = True
-
-            recommendations.extend([f"Arbitration decision: {arbitration.decision}", *arbitration.reasons])
-
-            logger.info(f"Action validation completed for user {self.user_id}. Decision: {arbitration.decision}, Valid: {is_valid}, Risk: {risk_level.value}, Violations: {len(violations)}")
-
             # Record in history
             await self._record_validation_history(action_data, is_valid, violations)
-
+            
             return SafetyValidation(
                 is_valid=is_valid,
                 risk_level=risk_level,
                 violations=violations,
                 recommendations=recommendations,
                 requires_approval=requires_approval,
-                confidence_score=confidence_score
+                confidence_score=max(0.0, min(1.0, confidence_score))
             )
             
         except Exception as e:
@@ -269,30 +235,6 @@ class SafetyConstraintManager:
                 confidence_score=0.0
             )
     
-    def _arbitrate_decision(self, action_data: Dict[str, Any], risk_level: RiskLevel, violations: List[str], requires_approval: bool, confidence_score: float) -> SafetyArbitrationDecision:
-        """Arbitrate allow/deny/lock with explicit reasons."""
-        reasons: List[str] = []
-        tier = int(action_data.get("recommended_tier", 1))
-
-        if self.lock_state_active:
-            reasons.append("Existing lock state is active")
-            return SafetyArbitrationDecision("lock", reasons, tier, confidence_score, True)
-
-        if tier >= 3 or risk_level == RiskLevel.CRITICAL:
-            reasons.append("Tier 3 systemic anomaly or critical risk detected")
-            if violations:
-                reasons.extend(violations)
-            return SafetyArbitrationDecision("lock", reasons, 3, confidence_score, True)
-
-        if violations or requires_approval:
-            reasons.append("Safety policy violation or approval requirement triggered")
-            reasons.extend(violations)
-            return SafetyArbitrationDecision("deny", reasons, tier, confidence_score, False)
-
-        reasons.append("No policy violations detected")
-        return SafetyArbitrationDecision("allow", reasons, tier, confidence_score, False)
-
-
     def _determine_action_category(self, action_type: str) -> ActionCategory:
         """Determine the category of an action"""
         action_type_lower = action_type.lower()