Add OAuth token refresh retries, status persistence, and alert payloads

backend/models/oauth_token_monitoring_models.py

@@ -40,6 +40,10 @@ class OAuthTokenMonitoringTask(Base):
     
     # Scheduling
     next_check = Column(DateTime, nullable=True, index=True)  # Next scheduled check time
+    next_retry_at = Column(DateTime, nullable=True, index=True)  # Backoff retry schedule for refresh failures
+    refresh_attempts = Column(Integer, default=0)  # Current retry attempt count for refresh workflow
+    terminal_failure_reason = Column(Text, nullable=True)  # Permanent failure reason requiring user action
+    channel_status = Column(String(32), default='connected')  # connected, degraded, disconnected
     
     # Metadata
     created_at = Column(DateTime, default=datetime.utcnow)
@@ -97,4 +101,3 @@ class OAuthTokenExecutionLog(Base):
     
     def __repr__(self):
         return f"<OAuthTokenExecutionLog(id={self.id}, task_id={self.task_id}, status={self.status}, execution_date={self.execution_date})>"
-

backend/services/intelligence/agent_context_vfs.py

@@ -101,7 +101,6 @@ class AgentContextVFS:
         "/steps/integrations": AgentFlatContextStore.STEP5_FILENAME,
     }
     HIGH_SIGNAL_MARKERS = ("agent_summary", "high_signal_terms", "quick_facts", "context_type")
-    LOW_CONFIDENCE_MARKER = "low_confidence"
 
     def __init__(self, user_id: str, project_id: Optional[str] = None):
         self.user_id = user_id
@@ -295,101 +294,6 @@ class AgentContextVFS:
         )
         return ranked[: max(1, top_k)]
 
-    @staticmethod
-    def _mnemonic_token(result: Dict[str, Any], rank: int) -> str:
-        """Create compressed mnemonic token with source reference."""
-        path = str(result.get("path") or "unknown")
-        reason = str(result.get("reason") or "match")
-        confidence = float(result.get("confidence") or 0.0)
-        low_flag = "!" if result.get(AgentContextVFS.LOW_CONFIDENCE_MARKER) else ""
-        src = path.replace(".json", "").replace("_", "-")[:28]
-        hint = reason.replace(" ", "-")[:20]
-        return f"M{rank}:{src}|{hint}|c{confidence:.2f}{low_flag}"
-
-    @staticmethod
-    def _detect_contradictions(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-        """Detect contradictory learnings by path with conflicting reasons/relevance classes."""
-        by_path: Dict[str, List[Dict[str, Any]]] = {}
-        for item in results:
-            p = str(item.get("path") or "")
-            by_path.setdefault(p, []).append(item)
-
-        contradictions: List[Dict[str, Any]] = []
-        for path, rows in by_path.items():
-            reasons = {str(r.get("reason") or "").strip().lower() for r in rows}
-            relevance = {str(r.get("relevance") or "").strip().lower() for r in rows}
-            # contradictory if both high/supported or mixed summary/body signals in same source cluster
-            if len(reasons) > 1 and len(relevance) > 1:
-                contradictions.append(
-                    {
-                        "path": path,
-                        "reason_variants": sorted([r for r in reasons if r]),
-                        "relevance_variants": sorted([r for r in relevance if r]),
-                        "count": len(rows),
-                    }
-                )
-        return contradictions
-
-    def _run_synthesis_pipeline(
-        self, ranked_results: List[Dict[str, Any]], *, char_budget: int = 1200, top_k: int = 5
-    ) -> Dict[str, Any]:
-        """
-        Flat-context synthesis pipeline:
-        1) Compress telemetry into mnemonic tokens with source references
-        2) Detect contradictions and mark low-confidence heuristics
-        3) Select top-ranked, budget-fitting tokens for prompt injection
-        4) Persist synthesis + source lineage for explainability
-        """
-        contradictions = self._detect_contradictions(ranked_results)
-        contradiction_paths = {c["path"] for c in contradictions}
-
-        normalized: List[Dict[str, Any]] = []
-        for idx, item in enumerate(ranked_results, start=1):
-            row = dict(item)
-            low_conf = bool(row.get("low_probability")) or (str(row.get("path") or "") in contradiction_paths)
-            row[self.LOW_CONFIDENCE_MARKER] = low_conf
-            if low_conf:
-                row["confidence"] = round(max(0.05, float(row.get("confidence", 0.0)) * 0.7), 3)
-            row["mnemonic_token"] = self._mnemonic_token(row, idx)
-            normalized.append(row)
-
-        chosen: List[Dict[str, Any]] = []
-        used = 0
-        for row in normalized[: max(1, top_k * 3)]:
-            token = str(row.get("mnemonic_token") or "")
-            cost = len(token) + 8
-            if chosen and used + cost > char_budget:
-                continue
-            chosen.append(row)
-            used += cost
-            if len(chosen) >= top_k:
-                break
-
-        synthesis = {
-            "created_at": datetime.now(timezone.utc).isoformat(),
-            "top_k": top_k,
-            "char_budget": char_budget,
-            "char_budget_used": used,
-            "selected_mnemonics": [c.get("mnemonic_token") for c in chosen],
-            "source_lineage": [
-                {
-                    "mnemonic_token": c.get("mnemonic_token"),
-                    "path": c.get("path"),
-                    "reason": c.get("reason"),
-                    "confidence": c.get("confidence"),
-                    "low_confidence": c.get(self.LOW_CONFIDENCE_MARKER, False),
-                }
-                for c in chosen
-            ],
-            "contradictions": contradictions,
-        }
-        self.append_activity_log(
-            event_type="flat_context_synthesis",
-            actor="agent_context_vfs",
-            details=synthesis,
-        )
-        return {"ranked_results": normalized, "synthesis": synthesis}
-
     @staticmethod
     def _resolve_json_path(data: Any, path_query: str) -> Any:
         """Resolve dot/bracket JSON path such as 'data.seo_audit.recommendations[0]'."""
@@ -614,26 +518,15 @@ class AgentContextVFS:
                 bounded_results.append(r)
                 used += cost
 
-            synthesis_bundle = self._run_synthesis_pipeline(
-                self._static_triage(bounded_results, normalized),
-                char_budget=1200,
-                top_k=5,
-            )
-            triaged_results = synthesis_bundle["ranked_results"]
-            synthesis = synthesis_bundle["synthesis"]
-
             result = {
                 "query": normalized,
                 "attempted_queries": attempted_queries,
                 "matched_files_count": len(matched_files),
-                "results": triaged_results,
+                "results": self._static_triage(bounded_results, normalized),
                 "notice": notice,
                 "char_budget_used": used,
                 "can_answer": bool(bounded_results),
-                "synthesis": synthesis,
-                "prompt_context_mnemonics": synthesis.get("selected_mnemonics", []),
             }
-            # Top-ranked, budget-fitting mnemonic tokens are the only ones intended for prompt context injection.
             result["triage_top5"] = self._llm_router_stub(result["results"], top_k=5)
             logger.info(
                 f"[vfs_audit] user={self.store.safe_user_id} action=search_context query={normalized!r} results={len(result['results'])}"

backend/services/scheduler/__init__.py

@@ -26,7 +26,10 @@ from .executors.advertools_executor import AdvertoolsExecutor
 from .executors.sif_indexing_executor import SIFIndexingExecutor
 from .executors.market_trends_executor import MarketTrendsExecutor
 from .utils.task_loader import load_due_monitoring_tasks
-from .utils.oauth_token_task_loader import load_due_oauth_token_monitoring_tasks
+from .utils.oauth_token_task_loader import (
+    load_due_oauth_token_monitoring_tasks,
+    load_near_expiry_oauth_token_tasks
+)
 from .utils.website_analysis_task_loader import load_due_website_analysis_tasks
 from .utils.onboarding_full_website_analysis_task_loader import load_due_onboarding_full_website_analysis_tasks
 from .utils.deep_competitor_analysis_task_loader import load_due_deep_competitor_analysis_tasks
@@ -70,6 +73,11 @@ def get_scheduler() -> TaskScheduler:
             oauth_token_executor,
             load_due_oauth_token_monitoring_tasks
         )
+        _scheduler_instance.register_executor(
+            'oauth_token_refresh',
+            oauth_token_executor,
+            load_near_expiry_oauth_token_tasks
+        )
         
         # Register website analysis executor
         website_analysis_executor = WebsiteAnalysisExecutor()

backend/services/scheduler/executors/oauth_token_monitoring_executor.py

@@ -42,6 +42,8 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
         self.exception_handler = SchedulerExceptionHandler()
         # Expiration warning window (7 days before expiration)
         self.expiration_warning_days = 7
+        self.max_refresh_retries = 3
+        self.base_retry_backoff_minutes = 15
     
     async def execute_task(self, task: OAuthTokenMonitoringTask, db: Session) -> TaskExecutionResult:
         """
@@ -93,6 +95,10 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                 task.last_success = datetime.utcnow()
                 task.status = 'active'
                 task.failure_reason = None
+                task.terminal_failure_reason = None
+                task.channel_status = 'connected'
+                task.refresh_attempts = 0
+                task.next_retry_at = None
                 # Reset failure tracking on success
                 task.consecutive_failures = 0
                 task.failure_pattern = None
@@ -112,6 +118,7 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                 
                 task.last_failure = datetime.utcnow()
                 task.failure_reason = result.error_message
+                task.refresh_attempts = (task.refresh_attempts or 0) + 1
                 
                 if pattern and pattern.should_cool_off:
                     # Mark task for human intervention
@@ -126,6 +133,9 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                     }
                     # Clear next_check - task won't run automatically
                     task.next_check = None
+                    task.next_retry_at = None
+                    task.channel_status = "disconnected"
+                    task.terminal_failure_reason = result.error_message
                     
                     self.logger.warning(
                         f"Task {task.id} marked for human intervention: "
@@ -133,10 +143,17 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                         f"reason: {pattern.failure_reason.value}"
                     )
                 else:
-                    # Normal failure handling
-                    task.status = 'failed'
                     task.consecutive_failures = (task.consecutive_failures or 0) + 1
-                    # Do NOT update next_check - wait for manual trigger
+                    if task.refresh_attempts >= self.max_refresh_retries:
+                        task.status = 'failed'
+                        task.channel_status = 'disconnected'
+                        task.terminal_failure_reason = result.error_message
+                        task.next_retry_at = None
+                    else:
+                        task.status = 'degraded'
+                        task.channel_status = 'degraded'
+                        delay_minutes = self.base_retry_backoff_minutes * (2 ** (task.refresh_attempts - 1))
+                        task.next_retry_at = datetime.utcnow() + timedelta(minutes=delay_minutes)
                 
                 self.logger.warning(
                     f"OAuth token refresh failed for user {user_id}, platform {platform}. "
@@ -144,7 +161,7 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                 )
                 
                 # Create UsageAlert notification for the user
-                self._create_failure_alert(user_id, platform, result.error_message, result.result_data, db)
+                self._create_failure_alert(user_id, platform, result.error_message, result.result_data, db, task)
             
             task.updated_at = datetime.utcnow()
             db.commit()
@@ -193,12 +210,14 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                 task.last_failure = datetime.utcnow()
                 task.failure_reason = str(e)
                 task.status = 'failed'
+                task.channel_status = 'disconnected'
+                task.terminal_failure_reason = str(e)
                 task.last_check = datetime.utcnow()
                 task.updated_at = datetime.utcnow()
-                # Do NOT update next_check - wait for manual trigger
+                task.next_retry_at = None
                 
                 # Create UsageAlert notification for the user
-                self._create_failure_alert(user_id, task.platform, str(e), None, db)
+                self._create_failure_alert(user_id, task.platform, str(e), None, db, task)
                 
                 db.commit()
             except Exception as commit_error:
@@ -651,7 +670,8 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
         platform: str,
         error_message: str,
         result_data: Optional[Dict[str, Any]],
-        db: Session
+        db: Session,
+        task: Optional[OAuthTokenMonitoringTask] = None
     ):
         """
         Create a UsageAlert notification when OAuth token refresh fails.
@@ -724,6 +744,20 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
             from datetime import datetime
             billing_period = datetime.utcnow().strftime("%Y-%m")
 
+            alert_payload = {
+                "requires_user_action": True,
+                "platform": platform,
+                "channel_status": getattr(task, "channel_status", "disconnected"),
+                "terminal_failure_reason": getattr(task, "terminal_failure_reason", error_message),
+                "next_retry_at": (
+                    task.next_retry_at.isoformat() if task and task.next_retry_at else None
+                ),
+                "refresh_attempts": getattr(task, "refresh_attempts", 0),
+                "max_refresh_retries": self.max_refresh_retries,
+            }
+
+            message = f"{message} [ALERT_PAYLOAD] {alert_payload}"
+            
             # Create UsageAlert
             alert = UsageAlert(
                 user_id=user_id,
@@ -786,4 +820,3 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                 f"Defaulting to Weekly (7 days)."
             )
             return last_execution + timedelta(days=7)
-

backend/services/scheduler/utils/oauth_token_task_loader.py

@@ -3,7 +3,7 @@ OAuth Token Monitoring Task Loader
 Functions to load due OAuth token monitoring tasks from database.
 """
 
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import List, Optional, Union
 from sqlalchemy.orm import Session
 from sqlalchemy import and_, or_
@@ -52,3 +52,34 @@ def load_due_oauth_token_monitoring_tasks(
     
     return query.all()
 
+
+def load_near_expiry_oauth_token_tasks(
+    db: Session,
+    refresh_horizon_hours: int = 24,
+    user_id: Optional[Union[str, int]] = None
+) -> List[OAuthTokenMonitoringTask]:
+    """
+    Load OAuth tasks that should run token refresh logic soon.
+
+    Includes:
+    - tasks with a scheduled retry now due (next_retry_at <= now)
+    - tasks whose routine check is inside the near-expiry horizon window
+    """
+    now = datetime.utcnow()
+    horizon = now + timedelta(hours=max(refresh_horizon_hours, 1))
+
+    query = db.query(OAuthTokenMonitoringTask).filter(
+        and_(
+            OAuthTokenMonitoringTask.status.in_(['active', 'failed', 'degraded']),
+            or_(
+                OAuthTokenMonitoringTask.next_retry_at <= now,
+                OAuthTokenMonitoringTask.next_check <= horizon,
+                OAuthTokenMonitoringTask.next_check.is_(None)
+            )
+        )
+    )
+
+    if user_id is not None:
+        query = query.filter(OAuthTokenMonitoringTask.user_id == str(user_id))
+
+    return query.all()