"""Tests for offline replay module — Task 45.""" from __future__ import annotations from uuid import uuid4 from services.intelligence_pipeline_v3.replay.reports import ( DEFAULT_PROMOTION_GATES, FieldReport, GateStatus, PromotionGate, ReplayReport, ) from services.intelligence_pipeline_v3.replay.runner import ( ReplayConfig, ReplayMode, ReplayResult, ReplayRunner, ) class TestReplayRunner: """Task 45.1: Run configurations on Gold Corpus.""" def test_create_config(self): config = ReplayConfig.create( mode=ReplayMode.V3_FULL, corpus_version="1.0", pipeline_version="v3", ) assert config.mode == ReplayMode.V3_FULL assert config.temperature == 0.0 assert config.strict_schema is True def test_record_results(self): config = ReplayConfig.create(mode=ReplayMode.V3_FULL) runner = ReplayRunner(config=config) runner.start() runner.record_result( ReplayResult( document_id="doc-001", config_id=config.config_id, success=True, latency_ms=150.0, gpu_seconds=0.5, ) ) runner.record_result( ReplayResult( document_id="doc-002", config_id=config.config_id, success=True, latency_ms=200.0, gpu_seconds=0.3, ) ) runner.complete() assert runner.total_documents == 2 assert runner.success_rate == 1.0 assert runner.avg_latency_ms == 175.0 assert runner.total_gpu_seconds == 0.8 def test_failure_rate(self): config = ReplayConfig.create(mode=ReplayMode.CURRENT_V2) runner = ReplayRunner(config=config) runner.record_result( ReplayResult( document_id="doc-001", config_id=config.config_id, success=True, latency_ms=100, ) ) runner.record_result( ReplayResult( document_id="doc-002", config_id=config.config_id, success=False, latency_ms=50, errors=["schema_invalid"], ) ) assert runner.success_rate == 0.5 assert runner.failure_count == 1 def test_schema_validity_rate(self): config = ReplayConfig.create(mode=ReplayMode.V3_FAST_PATH) runner = ReplayRunner(config=config) for i in range(10): runner.record_result( ReplayResult( document_id=f"doc-{i}", config_id=config.config_id, success=True, latency_ms=100, schema_valid=(i < 9), # 1 invalid ) ) assert runner.schema_validity_rate == 0.9 class TestPromotionGates: """Task 45.3-45.4: Gate evaluation and safety-critical enforcement.""" def test_gate_passes_above_threshold(self): gate = PromotionGate( name="entity_f1", metric_name="entity_f1", threshold=0.85, direction="above", ) assert gate.evaluate(0.90) == GateStatus.PASSED assert gate.evaluate(0.80) == GateStatus.FAILED def test_gate_passes_below_threshold(self): gate = PromotionGate( name="calibration", metric_name="ece", threshold=0.08, direction="below", ) assert gate.evaluate(0.05) == GateStatus.PASSED assert gate.evaluate(0.10) == GateStatus.FAILED def test_replay_report_evaluate_all_gates(self): report = ReplayReport( report_id=uuid4(), config_id=uuid4(), baseline_config_id=uuid4(), ) metrics = { "entity_f1": 0.92, "evidence_support_rate": 0.90, "schema_validity_rate": 0.995, "calibration_ece": 0.05, "fast_path_rate": 0.70, "gpu_seconds_ratio": 0.40, } results = report.evaluate_gates(metrics) assert results["entity_f1"] == GateStatus.PASSED assert results["evidence_support_rate"] == GateStatus.PASSED assert results["schema_validity"] == GateStatus.PASSED assert report.all_safety_gates_passed def test_safety_critical_gate_failure(self): report = ReplayReport( report_id=uuid4(), config_id=uuid4(), baseline_config_id=uuid4(), ) metrics = { "entity_f1": 0.0, # Regression — fails gate "evidence_support_rate": 0.90, "schema_validity_rate": 0.995, "calibration_ece": 0.05, "fast_path_rate": 0.70, "gpu_seconds_ratio": 0.40, } report.evaluate_gates(metrics) # entity_f1 gate threshold is 0.0 (no regression), but the gate # checks value >= threshold. 0.0 >= 0.0 passes. # Let's check a real failure case metrics["evidence_support_rate"] = 0.50 # Below 85% threshold report.evaluate_gates(metrics) assert not report.all_safety_gates_passed def test_default_gates_exist(self): assert len(DEFAULT_PROMOTION_GATES) >= 5 safety_gates = [g for g in DEFAULT_PROMOTION_GATES if g.safety_critical] assert len(safety_gates) >= 2 class TestFieldReport: """Task 45.2: Field-level reports.""" def test_field_report_accuracy(self): report = FieldReport( field_name="entity", precision=0.90, recall=0.85, f1=0.87, support_count=100, error_count=10, ) assert report.accuracy == 0.9 def test_zero_support(self): report = FieldReport(field_name="relation", support_count=0) assert report.accuracy == 0.0