feat: Intelligence Pipeline v3 — full implementation

Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
@@ -0,0 +1,249 @@
+"""Tests for benchmark comparison and attribution logic.
+
+Validates: Requirements 16.2, 16.3, 16.5
+"""
+from __future__ import annotations
+
+from services.intelligence_pipeline_v3.benchmark.comparison import (
+    ComparisonReport,
+    ConfigDelta,
+    FieldDelta,
+    ResourceDelta,
+    compare_configurations,
+)
+from services.intelligence_pipeline_v3.benchmark.runner import (
+    BenchmarkDocumentResult,
+    BenchmarkRun,
+)
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+def _make_result(
+    doc_id: str,
+    *,
+    schema_valid: bool = True,
+    duration_ms: int = 100,
+    input_tokens: int = 500,
+    output_tokens: int = 200,
+    retries: int = 0,
+    error: str | None = None,
+) -> BenchmarkDocumentResult:
+    """Helper to create a BenchmarkDocumentResult."""
+    return BenchmarkDocumentResult(
+        document_id=doc_id,
+        raw_output='{"test": true}' if schema_valid else "invalid",
+        parsed_output={"test": True} if schema_valid else None,
+        schema_valid=schema_valid,
+        retries=retries,
+        duration_ms=duration_ms,
+        input_tokens=input_tokens,
+        output_tokens=output_tokens,
+        error=error,
+    )
+
+
+def _make_run(
+    config_name: str,
+    results: list[BenchmarkDocumentResult],
+) -> BenchmarkRun:
+    """Helper to create a BenchmarkRun."""
+    return BenchmarkRun(
+        config_name=config_name,
+        document_ids=[r.document_id for r in results],
+        results=results,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+class TestCompareConfigurations:
+    """Tests for compare_configurations function."""
+
+    def test_empty_comparison_runs(self) -> None:
+        baseline = _make_run("baseline_current", [
+            _make_result("doc1", schema_valid=True),
+        ])
+        report = compare_configurations(baseline, [])
+        assert report.configs_compared == ["baseline_current"]
+        assert report.deltas == []
+
+    def test_basic_comparison_produces_deltas(self) -> None:
+        # Baseline: 50% schema validity
+        baseline = _make_run("baseline_current", [
+            _make_result("doc1", schema_valid=True),
+            _make_result("doc2", schema_valid=False, error="parse error"),
+        ])
+        # Temp zero: 100% schema validity
+        temp_zero = _make_run("baseline_temp_zero", [
+            _make_result("doc1", schema_valid=True),
+            _make_result("doc2", schema_valid=True),
+        ])
+
+        report = compare_configurations(baseline, [temp_zero])
+
+        assert len(report.configs_compared) == 2
+        assert len(report.deltas) == 1
+        delta = report.deltas[0]
+        assert delta.baseline_config == "baseline_current"
+        assert delta.comparison_config == "baseline_temp_zero"
+        assert len(delta.field_deltas) > 0
+        assert len(delta.resource_deltas) > 0
+
+    def test_schema_validity_improvement_detected(self) -> None:
+        baseline = _make_run("baseline_current", [
+            _make_result("doc1", schema_valid=True),
+            _make_result("doc2", schema_valid=False, error="err"),
+            _make_result("doc3", schema_valid=False, error="err"),
+            _make_result("doc4", schema_valid=True),
+        ])
+        strict = _make_run("baseline_strict_schema", [
+            _make_result("doc1", schema_valid=True),
+            _make_result("doc2", schema_valid=True),
+            _make_result("doc3", schema_valid=True),
+            _make_result("doc4", schema_valid=True),
+        ])
+
+        report = compare_configurations(baseline, [strict])
+        delta = report.deltas[0]
+
+        # Find the schema_validity_rate field delta
+        validity_delta = next(
+            (d for d in delta.field_deltas if d.field_name == "schema_validity_rate"),
+            None,
+        )
+        assert validity_delta is not None
+        assert validity_delta.improved is True
+        assert validity_delta.comparison_value == 1.0
+        assert validity_delta.baseline_value == 0.5
+
+    def test_attribution_with_incremental_improvement(self) -> None:
+        # Baseline: 50% valid
+        baseline = _make_run("baseline_current", [
+            _make_result("d1", schema_valid=True),
+            _make_result("d2", schema_valid=False, error="e"),
+        ])
+        # Temp zero: 75% (fixes half the remaining)
+        # We simulate by 3/4 valid
+        temp_zero = _make_run("baseline_temp_zero", [
+            _make_result("d1", schema_valid=True),
+            _make_result("d2", schema_valid=True),
+            _make_result("d3", schema_valid=True),
+            _make_result("d4", schema_valid=False, error="e"),
+        ])
+        # Strict schema: 100% valid
+        strict = _make_run("baseline_strict_schema", [
+            _make_result("d1", schema_valid=True),
+            _make_result("d2", schema_valid=True),
+        ])
+
+        report = compare_configurations(baseline, [temp_zero, strict])
+
+        # Attribution should exist
+        assert "temperature_fix" in report.attribution_summary
+        assert "schema_constraint" in report.attribution_summary
+
+        # All attribution values should be between 0 and 1
+        for val in report.attribution_summary.values():
+            assert 0.0 <= val <= 1.0
+
+    def test_attribution_no_improvement(self) -> None:
+        # Both configurations have same validity
+        baseline = _make_run("baseline_current", [
+            _make_result("d1", schema_valid=True),
+        ])
+        temp_zero = _make_run("baseline_temp_zero", [
+            _make_result("d1", schema_valid=True),
+        ])
+
+        report = compare_configurations(baseline, [temp_zero])
+
+        # No improvement means zero attribution
+        assert report.attribution_summary.get("temperature_fix", 0.0) == 0.0
+        assert report.attribution_summary.get("schema_constraint", 0.0) == 0.0
+
+    def test_resource_improvement_lower_is_better(self) -> None:
+        baseline = _make_run("baseline_current", [
+            _make_result("d1", duration_ms=500, retries=3),
+        ])
+        improved = _make_run("baseline_temp_zero", [
+            _make_result("d1", duration_ms=200, retries=0),
+        ])
+
+        report = compare_configurations(baseline, [improved])
+        delta = report.deltas[0]
+
+        # Duration should show improvement (lower)
+        duration_delta = next(
+            (d for d in delta.resource_deltas if d.metric_name == "mean_duration_ms"),
+            None,
+        )
+        assert duration_delta is not None
+        assert duration_delta.improved is True
+        assert duration_delta.comparison_value < duration_delta.baseline_value
+
+    def test_multiple_comparisons(self) -> None:
+        baseline = _make_run("baseline_current", [
+            _make_result("d1", schema_valid=True),
+        ])
+        comp1 = _make_run("baseline_temp_zero", [
+            _make_result("d1", schema_valid=True),
+        ])
+        comp2 = _make_run("baseline_strict_schema", [
+            _make_result("d1", schema_valid=True),
+        ])
+
+        report = compare_configurations(baseline, [comp1, comp2])
+        assert len(report.deltas) == 2
+        assert report.configs_compared == [
+            "baseline_current",
+            "baseline_temp_zero",
+            "baseline_strict_schema",
+        ]
+
+
+class TestComparisonReportModel:
+    """Tests for the ComparisonReport Pydantic model."""
+
+    def test_serialization_roundtrip(self) -> None:
+        report = ComparisonReport(
+            configs_compared=["a", "b"],
+            deltas=[
+                ConfigDelta(
+                    baseline_config="a",
+                    comparison_config="b",
+                    field_deltas=[
+                        FieldDelta(
+                            field_name="accuracy",
+                            baseline_value=0.5,
+                            comparison_value=0.8,
+                            absolute_delta=0.3,
+                            relative_delta_percent=60.0,
+                            improved=True,
+                        )
+                    ],
+                    resource_deltas=[
+                        ResourceDelta(
+                            metric_name="latency_ms",
+                            baseline_value=500.0,
+                            comparison_value=300.0,
+                            absolute_delta=-200.0,
+                            relative_delta_percent=-40.0,
+                            improved=True,
+                        )
+                    ],
+                )
+            ],
+            attribution_summary={"temperature_fix": 0.6, "schema_constraint": 0.4},
+        )
+
+        json_str = report.model_dump_json()
+        restored = ComparisonReport.model_validate_json(json_str)
+        assert restored.configs_compared == report.configs_compared
+        assert len(restored.deltas) == 1
+        assert restored.attribution_summary["temperature_fix"] == 0.6