feat: Intelligence Pipeline v3 — full implementation

Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
@@ -0,0 +1,160 @@
+"""Tests for production shadow mode — Task 46."""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+from services.intelligence_pipeline_v3.shadow.runner import (
+    DisagreementLevel,
+    ShadowComparison,
+    ShadowConfig,
+    ShadowRunner,
+)
+
+
+class TestShadowRunner:
+    """Task 46.1-46.4: Shadow mode operation and stability."""
+
+    def test_start_shadow(self):
+        runner = ShadowRunner(config=ShadowConfig())
+        assert not runner.is_active
+        runner.start()
+        assert runner.is_active
+
+    def test_record_comparison(self):
+        runner = ShadowRunner(config=ShadowConfig(enabled=True))
+        runner.started_at = datetime.now(timezone.utc)
+        comp = ShadowComparison.create(
+            document_id="doc-001",
+            v2_output={"sentiment": "positive"},
+            v3_output={"sentiment": "positive"},
+            disagreement_level=DisagreementLevel.NONE,
+        )
+        runner.record_comparison(comp)
+        assert runner.documents_processed == 1
+
+    def test_critical_disagreements_tracked(self):
+        runner = ShadowRunner(config=ShadowConfig(enabled=True))
+        runner.started_at = datetime.now(timezone.utc)
+        for i in range(3):
+            runner.record_comparison(
+                ShadowComparison.create(
+                    document_id=f"doc-{i}",
+                    v2_output={},
+                    v3_output={},
+                    disagreement_level=DisagreementLevel.CRITICAL,
+                )
+            )
+        assert runner.critical_disagreements == 3
+
+    def test_major_disagreement_rate(self):
+        runner = ShadowRunner(config=ShadowConfig(enabled=True))
+        runner.started_at = datetime.now(timezone.utc)
+        # 2 major out of 10 = 20%
+        for i in range(8):
+            runner.record_comparison(
+                ShadowComparison.create(
+                    f"doc-{i}", {}, {},
+                    disagreement_level=DisagreementLevel.MINOR,
+                )
+            )
+        for i in range(2):
+            runner.record_comparison(
+                ShadowComparison.create(
+                    f"doc-major-{i}", {}, {},
+                    disagreement_level=DisagreementLevel.MAJOR,
+                )
+            )
+        assert runner.major_disagreement_rate == 0.2
+
+    def test_promotion_requires_min_duration(self):
+        config = ShadowConfig(
+            enabled=True,
+            min_duration=timedelta(days=7),
+            min_documents=10,
+        )
+        runner = ShadowRunner(config=config)
+        runner.started_at = datetime.now(timezone.utc)  # Just started
+        for i in range(20):
+            runner.record_comparison(
+                ShadowComparison.create(f"doc-{i}", {}, {})
+            )
+        # Not enough time elapsed
+        assert not runner.meets_promotion_criteria()
+
+    def test_promotion_requires_min_documents(self):
+        config = ShadowConfig(
+            enabled=True,
+            min_duration=timedelta(seconds=0),
+            min_documents=100,
+        )
+        runner = ShadowRunner(config=config)
+        runner.started_at = datetime.now(timezone.utc) - timedelta(days=10)
+        for i in range(50):  # Below minimum
+            runner.record_comparison(
+                ShadowComparison.create(f"doc-{i}", {}, {})
+            )
+        assert not runner.meets_promotion_criteria()
+
+    def test_promotion_criteria_met(self):
+        config = ShadowConfig(
+            enabled=True,
+            min_duration=timedelta(seconds=0),
+            min_documents=5,
+            max_critical_disagreements=10,
+            max_major_disagreement_rate=0.5,
+        )
+        runner = ShadowRunner(config=config)
+        runner.started_at = datetime.now(timezone.utc) - timedelta(days=10)
+        for i in range(10):
+            runner.record_comparison(
+                ShadowComparison.create(f"doc-{i}", {}, {})
+            )
+        assert runner.meets_promotion_criteria()
+
+    def test_fast_path_rate_tracking(self):
+        runner = ShadowRunner(config=ShadowConfig(enabled=True))
+        runner.started_at = datetime.now(timezone.utc)
+        runner.record_processing(fast_path=True)
+        runner.record_processing(fast_path=True)
+        runner.record_processing(fast_path=False)
+        assert runner.fast_path_rate == pytest.approx(2 / 3)
+
+    def test_auto_disable_on_errors(self):
+        config = ShadowConfig(
+            enabled=True, auto_disable_on_errors=True, error_threshold=3
+        )
+        runner = ShadowRunner(config=config)
+        runner.started_at = datetime.now(timezone.utc)
+        for _ in range(3):
+            runner.record_error()
+        assert not runner.is_active
+
+    def test_get_review_sample(self):
+        runner = ShadowRunner(
+            config=ShadowConfig(enabled=True, sample_review_rate=0.5)
+        )
+        runner.started_at = datetime.now(timezone.utc)
+        for i in range(4):
+            runner.record_comparison(
+                ShadowComparison.create(
+                    f"doc-{i}", {}, {},
+                    disagreement_level=DisagreementLevel.MODERATE,
+                    risk_score=0.5 + i * 0.1,
+                )
+            )
+        sample = runner.get_review_sample()
+        assert len(sample) == 2  # 50% of 4
+        # Should be sorted by priority/risk
+        assert sample[0].risk_score >= sample[1].risk_score
+
+    def test_summary(self):
+        runner = ShadowRunner(config=ShadowConfig(enabled=True))
+        runner.started_at = datetime.now(timezone.utc)
+        summary = runner.summary()
+        assert summary["active"] is True
+        assert "documents_processed" in summary
+
+
+# Need this import for pytest.approx
+import pytest  # noqa: E402