Add OAuth token refresh retries, status persistence, and alert payloads

backend/models/oauth_token_monitoring_models.py

@@ -40,6 +40,10 @@ class OAuthTokenMonitoringTask(Base):
     
     # Scheduling
     next_check = Column(DateTime, nullable=True, index=True)  # Next scheduled check time
+    next_retry_at = Column(DateTime, nullable=True, index=True)  # Backoff retry schedule for refresh failures
+    refresh_attempts = Column(Integer, default=0)  # Current retry attempt count for refresh workflow
+    terminal_failure_reason = Column(Text, nullable=True)  # Permanent failure reason requiring user action
+    channel_status = Column(String(32), default='connected')  # connected, degraded, disconnected
     
     # Metadata
     created_at = Column(DateTime, default=datetime.utcnow)
@@ -97,4 +101,3 @@ class OAuthTokenExecutionLog(Base):
     
     def __repr__(self):
         return f"<OAuthTokenExecutionLog(id={self.id}, task_id={self.task_id}, status={self.status}, execution_date={self.execution_date})>"
-

backend/services/intelligence/agents/performance_monitor.py

@@ -99,58 +99,6 @@ class OptimizationRecommendation:
             expires = datetime.utcnow().timestamp() + (7 * 24 * 60 * 60)
             self.expires_at = datetime.fromtimestamp(expires).isoformat()
 
-
-
-@dataclass
-class EscalationVelocitySignal:
-    """Measured action velocity signal used for escalation tiering."""
-    window_minutes: int
-    action_count: int
-    actions_per_minute: float
-    triggered: bool
-
-
-class EscalationTier(Enum):
-    """Escalation tier derived from measurable action velocity."""
-    TIER_1 = "tier_1"
-    TIER_2 = "tier_2"
-    TIER_3 = "tier_3"
-
-
-class EscalationVelocityPolicy:
-    """Velocity-based trigger policy for escalation tiers."""
-
-    def __init__(self):
-        self.tier_thresholds = {
-            EscalationTier.TIER_1: {"window_minutes": 15, "actions_per_minute": 0.8},
-            EscalationTier.TIER_2: {"window_minutes": 10, "actions_per_minute": 1.5},
-            EscalationTier.TIER_3: {"window_minutes": 5, "actions_per_minute": 3.0},
-        }
-
-    def measure_velocity(self, events: List[Dict[str, Any]], now: Optional[datetime] = None) -> Dict[EscalationTier, EscalationVelocitySignal]:
-        now = now or datetime.utcnow()
-        signals: Dict[EscalationTier, EscalationVelocitySignal] = {}
-
-        for tier, cfg in self.tier_thresholds.items():
-            cutoff = now - timedelta(minutes=cfg["window_minutes"])
-            count = sum(1 for event in events if datetime.fromisoformat(event["timestamp"]) >= cutoff)
-            velocity = count / max(cfg["window_minutes"], 1)
-            signals[tier] = EscalationVelocitySignal(
-                window_minutes=cfg["window_minutes"],
-                action_count=count,
-                actions_per_minute=velocity,
-                triggered=velocity >= cfg["actions_per_minute"]
-            )
-
-        return signals
-
-    def determine_tier(self, events: List[Dict[str, Any]], now: Optional[datetime] = None) -> Tuple[Optional[EscalationTier], Dict[EscalationTier, EscalationVelocitySignal]]:
-        signals = self.measure_velocity(events, now=now)
-        for tier in [EscalationTier.TIER_3, EscalationTier.TIER_2, EscalationTier.TIER_1]:
-            if signals[tier].triggered:
-                return tier, signals
-        return None, signals
-
 class AgentPerformanceMonitor:
     """Main performance monitoring system for agents"""
     

backend/services/intelligence/agents/safety_framework.py

@@ -13,7 +13,6 @@ from enum import Enum
 
 from utils.logger_utils import get_service_logger
 from services.database import get_session_for_user
-from services.intelligence.agents.performance_monitor import EscalationVelocityPolicy, EscalationTier
 
 logger = get_service_logger(__name__)
 
@@ -85,25 +84,6 @@ class SafetyValidation:
         if self.validation_timestamp is None:
             self.validation_timestamp = datetime.utcnow().isoformat()
 
-
-
-@dataclass
-class EscalationDecision:
-    """Structured escalation payload for autonomous safety routing."""
-    tier: str
-    action: str
-    confidence: float
-    risk_class: str
-    rationale: str
-    velocity: Dict[str, Any]
-    lockout_auto_edits: bool
-    executor: Optional[str]
-    created_at: str = None
-
-    def __post_init__(self):
-        if self.created_at is None:
-            self.created_at = datetime.utcnow().isoformat()
-
 class SafetyConstraintManager:
     """Manages safety constraints for agent actions"""
     
@@ -112,11 +92,6 @@ class SafetyConstraintManager:
         self.constraints: Dict[str, SafetyConstraint] = {}
         self.action_history: List[Dict[str, Any]] = []
         self.violation_history: List[Dict[str, Any]] = []
-        self.escalation_policy = EscalationVelocityPolicy()
-        self.escalation_history: List[Dict[str, Any]] = []
-        self.auto_edit_lockout = False
-        self.executor_routes = {"tier_1": "autonomous_guardian_executor", "tier_2": "autonomous_recovery_executor"}
-        self.alert_history: List[Dict[str, Any]] = []
         
         # Initialize default constraints
         self._initialize_default_constraints()
@@ -238,7 +213,7 @@ class SafetyConstraintManager:
             # Record in history
             await self._record_validation_history(action_data, is_valid, violations)
             
-            validation = SafetyValidation(
+            return SafetyValidation(
                 is_valid=is_valid,
                 risk_level=risk_level,
                 violations=violations,
@@ -246,10 +221,6 @@ class SafetyConstraintManager:
                 requires_approval=requires_approval,
                 confidence_score=max(0.0, min(1.0, confidence_score))
             )
-            escalation = await self.evaluate_escalation(action_data, validation)
-            if escalation:
-                recommendations.append(f"Escalation action: {escalation.action} ({escalation.tier})")
-            return validation
             
         except Exception as e:
             logger.error(f"Error validating action for user {self.user_id}: {e}")
@@ -495,97 +466,6 @@ class SafetyConstraintManager:
             if len(self.violation_history) > 500:
                 self.violation_history = self.violation_history[-500:]
     
-    async def evaluate_escalation(self, action_data: Dict[str, Any], validation: SafetyValidation) -> Optional[EscalationDecision]:
-        """Evaluate velocity-triggered escalation and produce structured decision payload."""
-        if self.auto_edit_lockout:
-            decision = EscalationDecision(
-                tier=EscalationTier.TIER_3.value,
-                action="lockout_enforced",
-                confidence=1.0,
-                risk_class=RiskLevel.CRITICAL.value,
-                rationale="Tier 3 lockout already active; autonomous edits blocked until manual reset",
-                velocity={},
-                lockout_auto_edits=True,
-                executor=None
-            )
-            await self._persist_escalation_decision(decision, action_data, outcome={"status": "blocked_by_lockout"})
-            return decision
-
-        tier, signals = self.escalation_policy.determine_tier(self.action_history)
-        if not tier:
-            return None
-
-        risk_class_map = {EscalationTier.TIER_1: RiskLevel.MEDIUM.value, EscalationTier.TIER_2: RiskLevel.HIGH.value, EscalationTier.TIER_3: RiskLevel.CRITICAL.value}
-        confidence = min(1.0, max(0.1, 0.55 + (len(validation.violations) * 0.05) + ((1 - validation.confidence_score) * 0.4)))
-
-        velocity_signal = signals[tier]
-        velocity_payload = {
-            "window_minutes": velocity_signal.window_minutes,
-            "action_count": velocity_signal.action_count,
-            "actions_per_minute": round(velocity_signal.actions_per_minute, 4),
-            "threshold_actions_per_minute": self.escalation_policy.tier_thresholds[tier]["actions_per_minute"],
-        }
-
-        executor = self.executor_routes.get(tier.value)
-        action = "route_to_autonomous_executor" if tier in (EscalationTier.TIER_1, EscalationTier.TIER_2) else "lockout_autonomous_edits"
-        rationale = f"{tier.value} triggered by velocity {velocity_payload['actions_per_minute']}/min over {velocity_signal.window_minutes}m window"
-
-        decision = EscalationDecision(
-            tier=tier.value,
-            action=action,
-            confidence=round(confidence, 3),
-            risk_class=risk_class_map[tier],
-            rationale=rationale,
-            velocity=velocity_payload,
-            lockout_auto_edits=(tier == EscalationTier.TIER_3),
-            executor=executor if tier != EscalationTier.TIER_3 else None
-        )
-
-        outcome = await self._apply_escalation_decision(decision, action_data, validation)
-        await self._persist_escalation_decision(decision, action_data, outcome=outcome)
-        return decision
-
-    async def _apply_escalation_decision(self, decision: EscalationDecision, action_data: Dict[str, Any], validation: SafetyValidation) -> Dict[str, Any]:
-        if decision.tier in (EscalationTier.TIER_1.value, EscalationTier.TIER_2.value):
-            return {
-                "status": "routed",
-                "executor": decision.executor,
-                "reason": decision.rationale
-            }
-
-        self.auto_edit_lockout = True
-        brief = {
-            "type": "diagnostic_brief",
-            "severity": "critical",
-            "tier": decision.tier,
-            "user_rationale": "Autonomous edits have been paused to protect account safety after sustained high-velocity actions.",
-            "validation_violations": validation.violations,
-            "action_type": action_data.get("action_type", "unknown"),
-            "timestamp": datetime.utcnow().isoformat()
-        }
-        self.alert_history.append(brief)
-        if len(self.alert_history) > 500:
-            self.alert_history = self.alert_history[-500:]
-
-        return {"status": "lockout_enabled", "diagnostic_brief": brief}
-
-    async def _persist_escalation_decision(self, decision: EscalationDecision, action_data: Dict[str, Any], outcome: Dict[str, Any]):
-        record = {
-            "timestamp": datetime.utcnow().isoformat(),
-            "decision": asdict(decision),
-            "action_data": action_data,
-            "outcome": outcome
-        }
-        self.escalation_history.append(record)
-        if len(self.escalation_history) > 2000:
-            self.escalation_history = self.escalation_history[-2000:]
-
-    def get_escalation_history(self, limit: int = 100) -> List[Dict[str, Any]]:
-        return self.escalation_history[-limit:] if self.escalation_history else []
-
-    def reset_auto_edit_lockout(self):
-        self.auto_edit_lockout = False
-
     def add_custom_constraint(self, constraint: SafetyConstraint):
         """Add a custom safety constraint"""
         self.constraints[constraint.constraint_id] = constraint

backend/services/scheduler/__init__.py

@@ -26,7 +26,10 @@ from .executors.advertools_executor import AdvertoolsExecutor
 from .executors.sif_indexing_executor import SIFIndexingExecutor
 from .executors.market_trends_executor import MarketTrendsExecutor
 from .utils.task_loader import load_due_monitoring_tasks
-from .utils.oauth_token_task_loader import load_due_oauth_token_monitoring_tasks
+from .utils.oauth_token_task_loader import (
+    load_due_oauth_token_monitoring_tasks,
+    load_near_expiry_oauth_token_tasks
+)
 from .utils.website_analysis_task_loader import load_due_website_analysis_tasks
 from .utils.onboarding_full_website_analysis_task_loader import load_due_onboarding_full_website_analysis_tasks
 from .utils.deep_competitor_analysis_task_loader import load_due_deep_competitor_analysis_tasks
@@ -70,6 +73,11 @@ def get_scheduler() -> TaskScheduler:
             oauth_token_executor,
             load_due_oauth_token_monitoring_tasks
         )
+        _scheduler_instance.register_executor(
+            'oauth_token_refresh',
+            oauth_token_executor,
+            load_near_expiry_oauth_token_tasks
+        )
         
         # Register website analysis executor
         website_analysis_executor = WebsiteAnalysisExecutor()

backend/services/scheduler/executors/oauth_token_monitoring_executor.py

@@ -42,6 +42,8 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
         self.exception_handler = SchedulerExceptionHandler()
         # Expiration warning window (7 days before expiration)
         self.expiration_warning_days = 7
+        self.max_refresh_retries = 3
+        self.base_retry_backoff_minutes = 15
     
     async def execute_task(self, task: OAuthTokenMonitoringTask, db: Session) -> TaskExecutionResult:
         """
@@ -93,6 +95,10 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                 task.last_success = datetime.utcnow()
                 task.status = 'active'
                 task.failure_reason = None
+                task.terminal_failure_reason = None
+                task.channel_status = 'connected'
+                task.refresh_attempts = 0
+                task.next_retry_at = None
                 # Reset failure tracking on success
                 task.consecutive_failures = 0
                 task.failure_pattern = None
@@ -112,6 +118,7 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                 
                 task.last_failure = datetime.utcnow()
                 task.failure_reason = result.error_message
+                task.refresh_attempts = (task.refresh_attempts or 0) + 1
                 
                 if pattern and pattern.should_cool_off:
                     # Mark task for human intervention
@@ -126,6 +133,9 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                     }
                     # Clear next_check - task won't run automatically
                     task.next_check = None
+                    task.next_retry_at = None
+                    task.channel_status = "disconnected"
+                    task.terminal_failure_reason = result.error_message
                     
                     self.logger.warning(
                         f"Task {task.id} marked for human intervention: "
@@ -133,10 +143,17 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                         f"reason: {pattern.failure_reason.value}"
                     )
                 else:
-                    # Normal failure handling
-                    task.status = 'failed'
                     task.consecutive_failures = (task.consecutive_failures or 0) + 1
-                    # Do NOT update next_check - wait for manual trigger
+                    if task.refresh_attempts >= self.max_refresh_retries:
+                        task.status = 'failed'
+                        task.channel_status = 'disconnected'
+                        task.terminal_failure_reason = result.error_message
+                        task.next_retry_at = None
+                    else:
+                        task.status = 'degraded'
+                        task.channel_status = 'degraded'
+                        delay_minutes = self.base_retry_backoff_minutes * (2 ** (task.refresh_attempts - 1))
+                        task.next_retry_at = datetime.utcnow() + timedelta(minutes=delay_minutes)
                 
                 self.logger.warning(
                     f"OAuth token refresh failed for user {user_id}, platform {platform}. "
@@ -144,7 +161,7 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                 )
                 
                 # Create UsageAlert notification for the user
-                self._create_failure_alert(user_id, platform, result.error_message, result.result_data, db)
+                self._create_failure_alert(user_id, platform, result.error_message, result.result_data, db, task)
             
             task.updated_at = datetime.utcnow()
             db.commit()
@@ -193,12 +210,14 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                 task.last_failure = datetime.utcnow()
                 task.failure_reason = str(e)
                 task.status = 'failed'
+                task.channel_status = 'disconnected'
+                task.terminal_failure_reason = str(e)
                 task.last_check = datetime.utcnow()
                 task.updated_at = datetime.utcnow()
-                # Do NOT update next_check - wait for manual trigger
+                task.next_retry_at = None
                 
                 # Create UsageAlert notification for the user
-                self._create_failure_alert(user_id, task.platform, str(e), None, db)
+                self._create_failure_alert(user_id, task.platform, str(e), None, db, task)
                 
                 db.commit()
             except Exception as commit_error:
@@ -651,7 +670,8 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
         platform: str,
         error_message: str,
         result_data: Optional[Dict[str, Any]],
-        db: Session
+        db: Session,
+        task: Optional[OAuthTokenMonitoringTask] = None
     ):
         """
         Create a UsageAlert notification when OAuth token refresh fails.
@@ -723,6 +743,20 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
             # Get current billing period (YYYY-MM format)
             from datetime import datetime
             billing_period = datetime.utcnow().strftime("%Y-%m")
+
+            alert_payload = {
+                "requires_user_action": True,
+                "platform": platform,
+                "channel_status": getattr(task, "channel_status", "disconnected"),
+                "terminal_failure_reason": getattr(task, "terminal_failure_reason", error_message),
+                "next_retry_at": (
+                    task.next_retry_at.isoformat() if task and task.next_retry_at else None
+                ),
+                "refresh_attempts": getattr(task, "refresh_attempts", 0),
+                "max_refresh_retries": self.max_refresh_retries,
+            }
+
+            message = f"{message} [ALERT_PAYLOAD] {alert_payload}"
             
             # Create UsageAlert
             alert = UsageAlert(
@@ -786,4 +820,3 @@ class OAuthTokenMonitoringExecutor(TaskExecutor):
                 f"Defaulting to Weekly (7 days)."
             )
             return last_execution + timedelta(days=7)
-

backend/services/scheduler/utils/oauth_token_task_loader.py

@@ -3,7 +3,7 @@ OAuth Token Monitoring Task Loader
 Functions to load due OAuth token monitoring tasks from database.
 """
 
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import List, Optional, Union
 from sqlalchemy.orm import Session
 from sqlalchemy import and_, or_
@@ -52,3 +52,34 @@ def load_due_oauth_token_monitoring_tasks(
     
     return query.all()
 
+
+def load_near_expiry_oauth_token_tasks(
+    db: Session,
+    refresh_horizon_hours: int = 24,
+    user_id: Optional[Union[str, int]] = None
+) -> List[OAuthTokenMonitoringTask]:
+    """
+    Load OAuth tasks that should run token refresh logic soon.
+
+    Includes:
+    - tasks with a scheduled retry now due (next_retry_at <= now)
+    - tasks whose routine check is inside the near-expiry horizon window
+    """
+    now = datetime.utcnow()
+    horizon = now + timedelta(hours=max(refresh_horizon_hours, 1))
+
+    query = db.query(OAuthTokenMonitoringTask).filter(
+        and_(
+            OAuthTokenMonitoringTask.status.in_(['active', 'failed', 'degraded']),
+            or_(
+                OAuthTokenMonitoringTask.next_retry_at <= now,
+                OAuthTokenMonitoringTask.next_check <= horizon,
+                OAuthTokenMonitoringTask.next_check.is_(None)
+            )
+        )
+    )
+
+    if user_id is not None:
+        query = query.filter(OAuthTokenMonitoringTask.user_id == str(user_id))
+
+    return query.all()