"""Replay runner — executes pipeline configurations against the Gold Corpus. Compares every required system configuration on identical inputs and produces structured output for report generation and gate evaluation. """ from __future__ import annotations import enum from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any from uuid import UUID, uuid4 class ReplayMode(str, enum.Enum): """Pipeline configurations to compare.""" CURRENT_V2 = "current_v2" CURRENT_V2_STRICT = "current_v2_strict" # Temperature 0 + strict schema V3_FAST_PATH = "v3_fast_path" V3_FULL = "v3_full" # Fast path + adjudication V3_SPECIALIST_ONLY = "v3_specialist_only" @dataclass(frozen=True) class ReplayConfig: """Configuration for a replay run.""" config_id: UUID mode: ReplayMode corpus_version: str pipeline_version: str model_version: str | None = None temperature: float = 0.0 strict_schema: bool = True description: str = "" @classmethod def create( cls, mode: ReplayMode, corpus_version: str = "1.0", pipeline_version: str = "v3", **kwargs: Any, ) -> ReplayConfig: return cls( config_id=uuid4(), mode=mode, corpus_version=corpus_version, pipeline_version=pipeline_version, **kwargs, ) @dataclass class ReplayResult: """Result of processing a single document in replay mode.""" document_id: str config_id: UUID success: bool latency_ms: float tokens_used: int = 0 gpu_seconds: float = 0.0 cpu_seconds: float = 0.0 extracted_entities: int = 0 extracted_facts: int = 0 evidence_spans: int = 0 schema_valid: bool = True errors: list[str] = field(default_factory=list) field_scores: dict[str, float] = field(default_factory=dict) metadata: dict[str, Any] = field(default_factory=dict) @dataclass class ReplayRunner: """Executes replay runs against a corpus. Processes documents through the specified pipeline configuration and collects results for comparison and reporting. """ config: ReplayConfig _results: list[ReplayResult] = field(default_factory=list) started_at: datetime | None = None completed_at: datetime | None = None def start(self) -> None: """Mark the replay as started.""" self.started_at = datetime.now(timezone.utc) def complete(self) -> None: """Mark the replay as completed.""" self.completed_at = datetime.now(timezone.utc) def record_result(self, result: ReplayResult) -> None: """Add a document processing result.""" self._results.append(result) @property def results(self) -> list[ReplayResult]: return list(self._results) @property def total_documents(self) -> int: return len(self._results) @property def success_count(self) -> int: return sum(1 for r in self._results if r.success) @property def failure_count(self) -> int: return sum(1 for r in self._results if not r.success) @property def success_rate(self) -> float: if not self._results: return 0.0 return self.success_count / len(self._results) @property def avg_latency_ms(self) -> float: if not self._results: return 0.0 return sum(r.latency_ms for r in self._results) / len(self._results) @property def total_gpu_seconds(self) -> float: return sum(r.gpu_seconds for r in self._results) @property def schema_validity_rate(self) -> float: if not self._results: return 0.0 return sum(1 for r in self._results if r.schema_valid) / len(self._results) def is_complete(self) -> bool: return self.completed_at is not None