AI Incident Response Playbook

"""AI incident detection framework.

Monitors model outputs for quality degradation, drift,
and safety violations. Designed to run as a background
service alongside your inference pipeline.
"""

from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Callable
import time


class IncidentSeverity(Enum):
    CRITICAL = 1
    HIGH = 2
    MEDIUM = 3
    LOW = 4


@dataclass
class DetectionResult:
    """Result of an incident detection check."""
    detected: bool
    severity: Optional[IncidentSeverity]
    detector_name: str
    message: str
    metric_value: float
    threshold: float
    timestamp: float


class AIIncidentDetector:
    """Composite detector that runs multiple checks
    against model outputs and triggers alerts.
    """

    def __init__(self):
        self.detectors: List[Callable] = []
        self.alert_handlers: List[Callable] = []

    def register_detector(self, fn: Callable) -> None:
        """Register a detection function."""
        self.detectors.append(fn)

    def register_alert_handler(self, fn: Callable) -> None:
        """Register a handler for detected incidents."""
        self.alert_handlers.append(fn)

    def check(self, outputs: list, labels: list = None) -> List[DetectionResult]:
        """Run all registered detectors against recent outputs."""
        results = []
        for detector in self.detectors:
            result = detector(outputs, labels)
            if result.detected:
                for handler in self.alert_handlers:
                    handler(result)
            results.append(result)
        return results


def confidence_drop_detector(
    outputs: list,
    labels: list = None,
    window_size: int = 100,
    threshold: float = 0.7,
) -> DetectionResult:
    """Detect when average model confidence drops below threshold."""
    if not outputs:
        return DetectionResult(
            detected=False,
            severity=None,
            detector_name="confidence_drop",
            message="No outputs to evaluate",
            metric_value=0.0,
            threshold=threshold,
            timestamp=time.time(),
        )

    recent = outputs[-window_size:]
    avg_confidence = sum(
        o.get("confidence", 0) for o in recent
    ) / len(recent)

    return DetectionResult(
        detected=avg_confidence < threshold,
        severity=IncidentSeverity.HIGH if avg_confidence < threshold else None,
        detector_name="confidence_drop",
        message=f"Average confidence {avg_confidence:.3f} vs threshold {threshold}",
        metric_value=avg_confidence,
        threshold=threshold,
        timestamp=time.time(),
    )

## AI Incident Report: [INCIDENT-ID]

**Severity**: SEV-[1/2/3/4]
**Status**: [Investigating / Contained / Resolved]
**Duration**: [Start time] - [End time or ongoing]
**Affected System**: [Model name and version]

### Impact Summary
- **Users affected**: [Number or percentage]
- **Decisions impacted**: [Description of affected outputs]
- **Business impact**: [Revenue, user experience, compliance]

### Timeline
- [HH:MM] Alert triggered by [detection mechanism]
- [HH:MM] On-call engineer acknowledged
- [HH:MM] [Containment action taken]
- [HH:MM] Root cause identified
- [HH:MM] Fix deployed / Model rolled back

### Root Cause
[One-paragraph description of what went wrong and why]

### Remediation
- **Immediate**: [Actions already taken]
- **Short-term**: [Actions planned this week]
- **Long-term**: [Systemic improvements planned]

### Prevention
- [ ] [Monitoring improvement]
- [ ] [Testing gap closure]
- [ ] [Process change]