"""Tests for production shadow mode — Task 46.""" from __future__ import annotations from datetime import datetime, timedelta, timezone from services.intelligence_pipeline_v3.shadow.runner import ( DisagreementLevel, ShadowComparison, ShadowConfig, ShadowRunner, ) class TestShadowRunner: """Task 46.1-46.4: Shadow mode operation and stability.""" def test_start_shadow(self): runner = ShadowRunner(config=ShadowConfig()) assert not runner.is_active runner.start() assert runner.is_active def test_record_comparison(self): runner = ShadowRunner(config=ShadowConfig(enabled=True)) runner.started_at = datetime.now(timezone.utc) comp = ShadowComparison.create( document_id="doc-001", v2_output={"sentiment": "positive"}, v3_output={"sentiment": "positive"}, disagreement_level=DisagreementLevel.NONE, ) runner.record_comparison(comp) assert runner.documents_processed == 1 def test_critical_disagreements_tracked(self): runner = ShadowRunner(config=ShadowConfig(enabled=True)) runner.started_at = datetime.now(timezone.utc) for i in range(3): runner.record_comparison( ShadowComparison.create( document_id=f"doc-{i}", v2_output={}, v3_output={}, disagreement_level=DisagreementLevel.CRITICAL, ) ) assert runner.critical_disagreements == 3 def test_major_disagreement_rate(self): runner = ShadowRunner(config=ShadowConfig(enabled=True)) runner.started_at = datetime.now(timezone.utc) # 2 major out of 10 = 20% for i in range(8): runner.record_comparison( ShadowComparison.create( f"doc-{i}", {}, {}, disagreement_level=DisagreementLevel.MINOR, ) ) for i in range(2): runner.record_comparison( ShadowComparison.create( f"doc-major-{i}", {}, {}, disagreement_level=DisagreementLevel.MAJOR, ) ) assert runner.major_disagreement_rate == 0.2 def test_promotion_requires_min_duration(self): config = ShadowConfig( enabled=True, min_duration=timedelta(days=7), min_documents=10, ) runner = ShadowRunner(config=config) runner.started_at = datetime.now(timezone.utc) # Just started for i in range(20): runner.record_comparison( ShadowComparison.create(f"doc-{i}", {}, {}) ) # Not enough time elapsed assert not runner.meets_promotion_criteria() def test_promotion_requires_min_documents(self): config = ShadowConfig( enabled=True, min_duration=timedelta(seconds=0), min_documents=100, ) runner = ShadowRunner(config=config) runner.started_at = datetime.now(timezone.utc) - timedelta(days=10) for i in range(50): # Below minimum runner.record_comparison( ShadowComparison.create(f"doc-{i}", {}, {}) ) assert not runner.meets_promotion_criteria() def test_promotion_criteria_met(self): config = ShadowConfig( enabled=True, min_duration=timedelta(seconds=0), min_documents=5, max_critical_disagreements=10, max_major_disagreement_rate=0.5, ) runner = ShadowRunner(config=config) runner.started_at = datetime.now(timezone.utc) - timedelta(days=10) for i in range(10): runner.record_comparison( ShadowComparison.create(f"doc-{i}", {}, {}) ) assert runner.meets_promotion_criteria() def test_fast_path_rate_tracking(self): runner = ShadowRunner(config=ShadowConfig(enabled=True)) runner.started_at = datetime.now(timezone.utc) runner.record_processing(fast_path=True) runner.record_processing(fast_path=True) runner.record_processing(fast_path=False) assert runner.fast_path_rate == pytest.approx(2 / 3) def test_auto_disable_on_errors(self): config = ShadowConfig( enabled=True, auto_disable_on_errors=True, error_threshold=3 ) runner = ShadowRunner(config=config) runner.started_at = datetime.now(timezone.utc) for _ in range(3): runner.record_error() assert not runner.is_active def test_get_review_sample(self): runner = ShadowRunner( config=ShadowConfig(enabled=True, sample_review_rate=0.5) ) runner.started_at = datetime.now(timezone.utc) for i in range(4): runner.record_comparison( ShadowComparison.create( f"doc-{i}", {}, {}, disagreement_level=DisagreementLevel.MODERATE, risk_score=0.5 + i * 0.1, ) ) sample = runner.get_review_sample() assert len(sample) == 2 # 50% of 4 # Should be sorted by priority/risk assert sample[0].risk_score >= sample[1].risk_score def test_summary(self): runner = ShadowRunner(config=ShadowConfig(enabled=True)) runner.started_at = datetime.now(timezone.utc) summary = runner.summary() assert summary["active"] is True assert "documents_processed" in summary # Need this import for pytest.approx import pytest # noqa: E402