"""Replay reports and promotion gate evaluation. Produces field-level, calibration, resource, and difficulty-bucket reports. Enforces safety-critical gates for promotion decisions. """ from __future__ import annotations import enum from dataclasses import dataclass, field from typing import Any from uuid import UUID class GateStatus(str, enum.Enum): """Promotion gate evaluation status.""" PASSED = "passed" FAILED = "failed" WARNING = "warning" NOT_EVALUATED = "not_evaluated" @dataclass(frozen=True) class PromotionGate: """A single promotion gate with a threshold and evaluation logic.""" name: str metric_name: str threshold: float direction: str # "above" (value must be >= threshold) or "below" (value must be <= threshold) safety_critical: bool = False description: str = "" def evaluate(self, value: float) -> GateStatus: """Evaluate the gate against a metric value.""" if self.direction == "above": return GateStatus.PASSED if value >= self.threshold else GateStatus.FAILED elif self.direction == "below": return GateStatus.PASSED if value <= self.threshold else GateStatus.FAILED return GateStatus.NOT_EVALUATED # Default promotion gates per Requirement 16.5 DEFAULT_PROMOTION_GATES: list[PromotionGate] = [ PromotionGate( name="entity_f1", metric_name="entity_f1", threshold=0.0, # No regression allowed (relative) direction="above", safety_critical=True, description="Entity/ticker F1 must not regress", ), PromotionGate( name="evidence_support_rate", metric_name="evidence_support_rate", threshold=0.85, direction="above", safety_critical=True, description="Evidence support rate must exceed 85%", ), PromotionGate( name="schema_validity", metric_name="schema_validity_rate", threshold=0.99, direction="above", safety_critical=True, description="Schema validity must exceed 99%", ), PromotionGate( name="calibration_ece", metric_name="calibration_ece", threshold=0.08, direction="below", safety_critical=False, description="Calibration ECE should be below 8%", ), PromotionGate( name="fast_path_coverage", metric_name="fast_path_rate", threshold=0.60, direction="above", safety_critical=False, description="Fast-path coverage should reach 60%", ), PromotionGate( name="gpu_reduction", metric_name="gpu_seconds_ratio", threshold=0.50, direction="below", safety_critical=False, description="GPU-seconds per doc should be ≤50% of baseline (2x improvement)", ), ] @dataclass class FieldReport: """Field-level metrics for a specific field across all documents.""" field_name: str precision: float = 0.0 recall: float = 0.0 f1: float = 0.0 exact_match: float = 0.0 support_count: int = 0 error_count: int = 0 @property def accuracy(self) -> float: if self.support_count == 0: return 0.0 return (self.support_count - self.error_count) / self.support_count @dataclass class ReplayReport: """Complete replay comparison report. Compares configurations, evaluates promotion gates, and produces field-level, resource, and difficulty-bucket breakdowns. """ report_id: UUID config_id: UUID baseline_config_id: UUID | None total_documents: int = 0 success_rate: float = 0.0 avg_latency_ms: float = 0.0 total_gpu_seconds: float = 0.0 schema_validity_rate: float = 0.0 fast_path_rate: float = 0.0 field_reports: list[FieldReport] = field(default_factory=list) gate_results: dict[str, GateStatus] = field(default_factory=dict) difficulty_buckets: dict[str, dict[str, float]] = field(default_factory=dict) document_type_breakdown: dict[str, dict[str, float]] = field( default_factory=dict ) def evaluate_gates( self, metrics: dict[str, float], gates: list[PromotionGate] | None = None, ) -> dict[str, GateStatus]: """Evaluate all promotion gates against collected metrics.""" gates = gates or DEFAULT_PROMOTION_GATES results: dict[str, GateStatus] = {} for gate in gates: value = metrics.get(gate.metric_name) if value is None: results[gate.name] = GateStatus.NOT_EVALUATED else: results[gate.name] = gate.evaluate(value) self.gate_results = results return results @property def all_safety_gates_passed(self) -> bool: """Whether all safety-critical gates passed.""" for gate in DEFAULT_PROMOTION_GATES: if gate.safety_critical: status = self.gate_results.get(gate.name, GateStatus.NOT_EVALUATED) if status != GateStatus.PASSED: return False return True @property def all_gates_passed(self) -> bool: """Whether all gates passed.""" return all( status == GateStatus.PASSED for status in self.gate_results.values() ) def to_dict(self) -> dict[str, Any]: """Serialize for storage/API response.""" return { "report_id": str(self.report_id), "config_id": str(self.config_id), "total_documents": self.total_documents, "success_rate": self.success_rate, "avg_latency_ms": self.avg_latency_ms, "total_gpu_seconds": self.total_gpu_seconds, "schema_validity_rate": self.schema_validity_rate, "fast_path_rate": self.fast_path_rate, "gate_results": {k: v.value for k, v in self.gate_results.items()}, "all_safety_gates_passed": self.all_safety_gates_passed, "all_gates_passed": self.all_gates_passed, }