Add tiered agent activity responses with redaction and UI toggle

backend/api/agents_api.py

@@ -6,6 +6,7 @@ Provides REST API access to agent orchestration functionality
 from fastapi import APIRouter, HTTPException, Depends, BackgroundTasks
 from typing import Dict, List, Any, Optional
 import asyncio
+import os
 from datetime import datetime
 
 from middleware.auth_middleware import get_current_user
@@ -18,6 +19,15 @@ from services.intelligence.agents.market_signal_detector import MarketSignal
 from services.intelligence.agents.performance_monitor import PerformanceMetric, AgentStatus
 from services.database import get_db
 from services.agent_activity_service import AgentActivityService
+from services.agent_activity_serializers import (
+    DETAIL_TIER_DEBUG,
+    DETAIL_TIER_SUMMARY,
+    normalize_detail_tier,
+    serialize_alert,
+    serialize_approval,
+    serialize_event,
+    serialize_run,
+)
 from sqlalchemy.orm import Session
 from models.agent_activity_models import AgentProfile
 from services.intelligence.agents.team_catalog import AGENT_TEAM_CATALOG, get_agent_catalog_entry
@@ -26,6 +36,35 @@ logger = get_service_logger(__name__)
 
 router = APIRouter(prefix="/api/agents", tags=["Autonomous Agents"])
 
+
+def _can_access_advanced_activity(current_user: Dict[str, Any]) -> bool:
+    role = str(current_user.get("role") or "").lower().strip()
+    metadata = current_user.get("public_metadata")
+    if isinstance(metadata, dict):
+        role = str(metadata.get("role") or role).lower().strip()
+
+    feature_flags = current_user.get("feature_flags")
+    if not feature_flags and isinstance(metadata, dict):
+        feature_flags = metadata.get("feature_flags") or metadata.get("features")
+
+    has_flag = False
+    if isinstance(feature_flags, list):
+        has_flag = any(str(flag).strip().lower() in {"agent_activity_detailed", "agents_activity_detailed"} for flag in feature_flags)
+    elif isinstance(feature_flags, dict):
+        has_flag = bool(feature_flags.get("agent_activity_detailed") or feature_flags.get("agents_activity_detailed"))
+
+    if os.getenv("DISABLE_AUTH", "false").lower() == "true":
+        return True
+
+    return role in {"admin", "internal"} or has_flag
+
+
+def _resolve_detail_tier(requested_tier: str, current_user: Dict[str, Any]) -> str:
+    tier = normalize_detail_tier(requested_tier)
+    if tier == DETAIL_TIER_DEBUG and not _can_access_advanced_activity(current_user):
+        return DETAIL_TIER_SUMMARY
+    return tier
+
 @router.get("/team")
 async def get_agent_team_endpoint(
     current_user: dict = Depends(get_current_user),
@@ -427,32 +466,21 @@ Return ONLY a JSON object that matches the schema.
 async def get_agent_alerts_endpoint(
     unread_only: bool = True,
     limit: int = 50,
+    detail_tier: str = DETAIL_TIER_SUMMARY,
     current_user: dict = Depends(get_current_user),
     db: Session = Depends(get_db),
 ) -> Dict[str, Any]:
     try:
         user_id = str(current_user.get("id"))
+        resolved_tier = _resolve_detail_tier(detail_tier, current_user)
         service = AgentActivityService(db, user_id)
         alerts = service.list_alerts(unread_only=unread_only, limit=limit)
         return {
             "success": True,
             "data": {
-                "alerts": [
+                "alerts": [serialize_alert(a, resolved_tier) for a in alerts],
-                    {
-                        "id": a.id,
-                        "source": a.source,
-                        "type": a.alert_type,
-                        "severity": a.severity,
-                        "title": a.title,
-                        "message": a.message,
-                        "cta_path": a.cta_path,
-                        "payload": a.payload,
-                        "created_at": a.created_at.isoformat() if a.created_at else None,
-                        "read_at": a.read_at.isoformat() if a.read_at else None,
-                    }
-                    for a in alerts
-                ],
                 "total": len(alerts),
+                "detail_tier": resolved_tier,
             },
             "timestamp": datetime.utcnow().isoformat(),
             "user_id": user_id,
@@ -485,31 +513,20 @@ async def mark_agent_alert_read_endpoint(
 @router.get("/runs")
 async def get_agent_runs_endpoint(
     limit: int = 30,
+    detail_tier: str = DETAIL_TIER_SUMMARY,
     current_user: dict = Depends(get_current_user),
     db: Session = Depends(get_db),
 ) -> Dict[str, Any]:
     try:
         user_id = str(current_user.get("id"))
+        resolved_tier = _resolve_detail_tier(detail_tier, current_user)
         service = AgentActivityService(db, user_id)
         runs = service.list_runs(limit=limit)
         return {
             "success": True,
             "data": {
-                "runs": [
+                "runs": [serialize_run(r, resolved_tier) for r in runs],
-                    {
+                "detail_tier": resolved_tier,
-                        "id": r.id,
-                        "user_id": r.user_id,
-                        "agent_type": r.agent_type,
-                        "status": r.status,
-                        "success": r.success,
-                        "error_message": r.error_message,
-                        "result_summary": r.result_summary,
-                        "mlflow_run_id": r.mlflow_run_id,
-                        "started_at": r.started_at.isoformat() if r.started_at else None,
-                        "finished_at": r.finished_at.isoformat() if r.finished_at else None,
-                    }
-                    for r in runs
-                ]
             },
             "timestamp": datetime.utcnow().isoformat(),
             "user_id": user_id,
@@ -523,29 +540,20 @@ async def get_agent_runs_endpoint(
 async def get_agent_run_events_endpoint(
     run_id: int,
     limit: int = 200,
+    detail_tier: str = DETAIL_TIER_SUMMARY,
     current_user: dict = Depends(get_current_user),
     db: Session = Depends(get_db),
 ) -> Dict[str, Any]:
     try:
         user_id = str(current_user.get("id"))
+        resolved_tier = _resolve_detail_tier(detail_tier, current_user)
         service = AgentActivityService(db, user_id)
         events = service.list_events(run_id=run_id, limit=limit)
         return {
             "success": True,
             "data": {
-                "events": [
+                "events": [serialize_event(e, resolved_tier) for e in events],
-                    {
+                "detail_tier": resolved_tier,
-                        "id": e.id,
-                        "run_id": e.run_id,
-                        "agent_type": e.agent_type,
-                        "event_type": e.event_type,
-                        "severity": e.severity,
-                        "message": e.message,
-                        "payload": e.payload,
-                        "created_at": e.created_at.isoformat() if e.created_at else None,
-                    }
-                    for e in events
-                ]
             },
             "timestamp": datetime.utcnow().isoformat(),
             "user_id": user_id,
@@ -559,32 +567,20 @@ async def get_agent_run_events_endpoint(
 async def get_agent_approvals_endpoint(
     status: str = "pending",
     limit: int = 50,
+    detail_tier: str = DETAIL_TIER_SUMMARY,
     current_user: dict = Depends(get_current_user),
     db: Session = Depends(get_db),
 ) -> Dict[str, Any]:
     try:
         user_id = str(current_user.get("id"))
+        resolved_tier = _resolve_detail_tier(detail_tier, current_user)
         service = AgentActivityService(db, user_id)
         approvals = service.list_approval_requests(status=status, limit=limit)
         return {
             "success": True,
             "data": {
-                "approvals": [
+                "approvals": [serialize_approval(a, resolved_tier) for a in approvals],
-                    {
+                "detail_tier": resolved_tier,
-                        "id": a.id,
-                        "status": a.status,
-                        "decision": a.decision,
-                        "action_id": a.action_id,
-                        "action_type": a.action_type,
-                        "agent_type": a.agent_type,
-                        "target_resource": a.target_resource,
-                        "risk_level": a.risk_level,
-                        "payload": a.payload,
-                        "created_at": a.created_at.isoformat() if a.created_at else None,
-                        "decided_at": a.decided_at.isoformat() if a.decided_at else None,
-                    }
-                    for a in approvals
-                ]
             },
             "timestamp": datetime.utcnow().isoformat(),
             "user_id": user_id,

backend/services/agent_activity_serializers.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from typing import Any, Dict
+from urllib.parse import urlparse, parse_qsl, urlencode, urlunparse
+
+from models.agent_activity_models import AgentAlert, AgentApprovalRequest, AgentEvent, AgentRun
+
+DETAIL_TIER_SUMMARY = "summary"
+DETAIL_TIER_DETAILED = "detailed"
+DETAIL_TIER_DEBUG = "debug"
+ALLOWED_DETAIL_TIERS = {DETAIL_TIER_SUMMARY, DETAIL_TIER_DETAILED, DETAIL_TIER_DEBUG}
+
+SENSITIVE_KEYWORDS = {
+    "token", "secret", "password", "pass", "api_key", "apikey", "auth", "authorization",
+    "cookie", "session", "credential", "ssn", "email", "phone", "address", "prompt", "raw_prompt",
+}
+SENSITIVE_QUERY_PARAMS = {
+    "token", "access_token", "auth", "authorization", "apikey", "api_key", "signature", "sig", "secret",
+}
+
+
+def normalize_detail_tier(value: Any) -> str:
+    tier = str(value or DETAIL_TIER_SUMMARY).strip().lower()
+    if tier not in ALLOWED_DETAIL_TIERS:
+        return DETAIL_TIER_SUMMARY
+    return tier
+
+
+def redact_sensitive_data(value: Any, key_hint: str | None = None) -> Any:
+    if isinstance(value, dict):
+        return {k: redact_sensitive_data(v, key_hint=str(k)) for k, v in value.items()}
+    if isinstance(value, list):
+        return [redact_sensitive_data(item, key_hint=key_hint) for item in value]
+
+    lowered_hint = (key_hint or "").lower()
+    if any(word in lowered_hint for word in SENSITIVE_KEYWORDS):
+        return "[REDACTED]"
+
+    if isinstance(value, str):
+        if "@" in value and any(word in lowered_hint for word in {"user", "contact", "owner", "email"}):
+            return "[REDACTED_EMAIL]"
+        if _looks_like_secret(value):
+            return "[REDACTED]"
+        return _sanitize_url(value)
+    return value
+
+
+def serialize_alert(alert: AgentAlert, detail_tier: str) -> Dict[str, Any]:
+    payload = redact_sensitive_data(alert.payload)
+    base = {
+        "id": alert.id,
+        "source": alert.source,
+        "type": alert.alert_type,
+        "severity": alert.severity,
+        "title": alert.title,
+        "message": alert.message,
+        "created_at": alert.created_at.isoformat() if alert.created_at else None,
+        "read_at": alert.read_at.isoformat() if alert.read_at else None,
+    }
+    if detail_tier in {DETAIL_TIER_DETAILED, DETAIL_TIER_DEBUG}:
+        base["cta_path"] = alert.cta_path
+        base["payload"] = payload
+    return base
+
+
+def serialize_run(run: AgentRun, detail_tier: str) -> Dict[str, Any]:
+    serialized = {
+        "id": run.id,
+        "agent_type": run.agent_type,
+        "status": run.status,
+        "success": run.success,
+        "result_summary": run.result_summary,
+        "started_at": run.started_at.isoformat() if run.started_at else None,
+        "finished_at": run.finished_at.isoformat() if run.finished_at else None,
+    }
+    if detail_tier in {DETAIL_TIER_DETAILED, DETAIL_TIER_DEBUG}:
+        serialized["error_message"] = redact_sensitive_data(run.error_message, key_hint="error_message")
+        serialized["mlflow_run_id"] = run.mlflow_run_id
+    if detail_tier == DETAIL_TIER_DEBUG:
+        serialized["user_id"] = run.user_id
+        serialized["prompt"] = redact_sensitive_data(run.prompt, key_hint="prompt")
+    return serialized
+
+
+def serialize_event(event: AgentEvent, detail_tier: str) -> Dict[str, Any]:
+    serialized = {
+        "id": event.id,
+        "run_id": event.run_id,
+        "agent_type": event.agent_type,
+        "event_type": event.event_type,
+        "severity": event.severity,
+        "message": event.message,
+        "created_at": event.created_at.isoformat() if event.created_at else None,
+    }
+    if detail_tier in {DETAIL_TIER_DETAILED, DETAIL_TIER_DEBUG}:
+        serialized["payload"] = redact_sensitive_data(event.payload)
+    return serialized
+
+
+def serialize_approval(approval: AgentApprovalRequest, detail_tier: str) -> Dict[str, Any]:
+    serialized = {
+        "id": approval.id,
+        "status": approval.status,
+        "decision": approval.decision,
+        "action_id": approval.action_id,
+        "action_type": approval.action_type,
+        "agent_type": approval.agent_type,
+        "target_resource": redact_sensitive_data(approval.target_resource, key_hint="target_resource"),
+        "risk_level": approval.risk_level,
+        "created_at": approval.created_at.isoformat() if approval.created_at else None,
+        "decided_at": approval.decided_at.isoformat() if approval.decided_at else None,
+    }
+    if detail_tier in {DETAIL_TIER_DETAILED, DETAIL_TIER_DEBUG}:
+        serialized["payload"] = redact_sensitive_data(approval.payload)
+    if detail_tier == DETAIL_TIER_DEBUG:
+        serialized["user_comments"] = redact_sensitive_data(approval.user_comments, key_hint="user_comments")
+    return serialized
+
+
+def _looks_like_secret(value: str) -> bool:
+    compact = value.strip()
+    if len(compact) > 32 and any(ch.isdigit() for ch in compact) and any(ch.isalpha() for ch in compact):
+        return True
+    return False
+
+
+def _sanitize_url(value: str) -> str:
+    if not value.startswith(("http://", "https://")):
+        return value
+    try:
+        parsed = urlparse(value)
+        query_items = []
+        for k, v in parse_qsl(parsed.query, keep_blank_values=True):
+            if k.lower() in SENSITIVE_QUERY_PARAMS:
+                query_items.append((k, "[REDACTED]"))
+            else:
+                query_items.append((k, v))
+        netloc = parsed.netloc
+        if parsed.username or parsed.password:
+            host = parsed.hostname or ""
+            if parsed.port:
+                host = f"{host}:{parsed.port}"
+            netloc = f"[REDACTED]@{host}"
+        return urlunparse(parsed._replace(netloc=netloc, query=urlencode(query_items)))
+    except Exception:
+        return "[REDACTED_URL]"

frontend/src/pages/ApprovalsPage.tsx

@@ -1,6 +1,7 @@
 import React from 'react';
-import { Box, Typography, Paper, Stack, Button, Chip, CircularProgress } from '@mui/material';
+import { Box, Typography, Paper, Stack, Button, Chip, CircularProgress, ToggleButtonGroup, ToggleButton } from '@mui/material';
 import { apiClient } from '../api/client';
+import { useUser } from '@clerk/clerk-react';
 
 type Approval = {
   id: number;
@@ -16,15 +17,27 @@ type Approval = {
 };
 
 export default function ApprovalsPage() {
+  const { user } = useUser();
   const [loading, setLoading] = React.useState(false);
   const [approvals, setApprovals] = React.useState<Approval[]>([]);
   const [error, setError] = React.useState<string | null>(null);
+  const [detailTier, setDetailTier] = React.useState<'summary' | 'detailed'>('summary');
+
+  const canUseDetailed = React.useMemo(() => {
+    const role = String(user?.publicMetadata?.role || '').toLowerCase().trim();
+    const featureFlags = user?.publicMetadata?.feature_flags as Record<string, any> | string[] | undefined;
+    const hasFeatureFlag = Array.isArray(featureFlags)
+      ? featureFlags.map((flag) => String(flag).toLowerCase()).includes('agent_activity_detailed')
+      : Boolean(featureFlags && (featureFlags['agent_activity_detailed'] || featureFlags['agents_activity_detailed']));
+    return role === 'admin' || role === 'internal' || hasFeatureFlag;
+  }, [user]);
 
   const loadApprovals = React.useCallback(async () => {
     setLoading(true);
     setError(null);
     try {
-      const resp = await apiClient.get('/api/agents/approvals', { params: { status: 'pending', limit: 50 } });
+      const tier = canUseDetailed ? detailTier : 'summary';
+      const resp = await apiClient.get('/api/agents/approvals', { params: { status: 'pending', limit: 50, detail_tier: tier } });
       const items = resp?.data?.data?.approvals || [];
       setApprovals(items);
     } catch (e: any) {
@@ -32,7 +45,7 @@ export default function ApprovalsPage() {
     } finally {
       setLoading(false);
     }
-  }, []);
+  }, [canUseDetailed, detailTier]);
 
   React.useEffect(() => {
     loadApprovals();
@@ -55,7 +68,22 @@ export default function ApprovalsPage() {
     <Box sx={{ p: 3 }}>
       <Stack direction="row" alignItems="center" justifyContent="space-between" sx={{ mb: 2 }}>
         <Typography variant="h5" sx={{ fontWeight: 700 }}>Agent Approvals</Typography>
-        <Button variant="outlined" onClick={loadApprovals} disabled={loading}>Refresh</Button>
+        <Stack direction="row" spacing={1} alignItems="center">
+          <ToggleButtonGroup
+            size="small"
+            value={detailTier}
+            exclusive
+            onChange={(_, value) => {
+              if (value) {
+                setDetailTier(value);
+              }
+            }}
+          >
+            <ToggleButton value="summary">Basic</ToggleButton>
+            <ToggleButton value="detailed" disabled={!canUseDetailed}>Detailed</ToggleButton>
+          </ToggleButtonGroup>
+          <Button variant="outlined" onClick={loadApprovals} disabled={loading}>Refresh</Button>
+        </Stack>
       </Stack>
 
       {error && (
@@ -89,6 +117,16 @@ export default function ApprovalsPage() {
             {a.target_resource && (
               <Typography sx={{ color: '#6b7280', mb: 1 }}>{a.target_resource}</Typography>
             )}
+            {detailTier === 'detailed' && a.payload && (
+              <Paper variant="outlined" sx={{ p: 1.5, mb: 1.5, bgcolor: '#fafafa' }}>
+                <Typography variant="caption" sx={{ display: 'block', color: '#6b7280', mb: 0.5 }}>
+                  Detailed payload
+                </Typography>
+                <Typography component="pre" sx={{ m: 0, whiteSpace: 'pre-wrap', fontSize: 12 }}>
+                  {JSON.stringify(a.payload, null, 2)}
+                </Typography>
+              </Paper>
+            )}
 
             <Stack direction="row" spacing={1}>
               <Button variant="contained" onClick={() => decide(a.id, 'approved')} disabled={loading}>Approve</Button>
@@ -100,4 +138,3 @@ export default function ApprovalsPage() {
     </Box>
   );
 }
-