feat: Intelligence Pipeline v3 — full implementation

Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
@@ -0,0 +1,543 @@
+"""Unit tests for evidence offset validity, support rate, and related metrics.
+
+Validates: Requirements 16.3, 16.4
+"""
+from __future__ import annotations
+
+from services.intelligence_pipeline_v3.evaluation.evidence_metrics import (
+    EvidenceMetricsResult,
+    EvidenceSpan,
+    ExtractionResult,
+    FieldType,
+    compute_coverage_score,
+    compute_offset_validity,
+    compute_orphan_rate,
+    compute_per_field_support,
+    compute_support_rate,
+    compute_unsupported_claim_rate,
+    evaluate_evidence,
+)
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+SOURCE_TEXT = "Apple reported revenue of $94.8 billion for Q3 2024. Tim Cook said growth was strong."
+
+
+def _span(span_id: str, text: str, start: int, end: int) -> EvidenceSpan:
+    return EvidenceSpan(span_id=span_id, text=text, start_char=start, end_char=end)
+
+
+def _item(
+    item_id: str,
+    field_type: FieldType,
+    evidence_ids: list[str] | None = None,
+    required_fields: list[str] | None = None,
+    supported_fields: list[str] | None = None,
+) -> ExtractionResult:
+    return ExtractionResult(
+        item_id=item_id,
+        field_type=field_type,
+        evidence_ids=evidence_ids or [],
+        required_fields=required_fields or [],
+        supported_fields=supported_fields or [],
+    )
+
+
+# ---------------------------------------------------------------------------
+# Offset Validity
+# ---------------------------------------------------------------------------
+
+
+class TestOffsetValidity:
+    def test_all_valid(self) -> None:
+        spans = [
+            _span("s1", "Apple", 0, 5),
+            _span("s2", "revenue", 15, 22),
+        ]
+        rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
+        assert rate == 1.0
+        assert valid == 2
+        assert total == 2
+
+    def test_one_invalid_text_mismatch(self) -> None:
+        spans = [
+            _span("s1", "Apple", 0, 5),
+            _span("s2", "WRONG", 15, 22),  # text doesn't match source
+        ]
+        rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
+        assert rate == 0.5
+        assert valid == 1
+        assert total == 2
+
+    def test_offset_out_of_bounds(self) -> None:
+        spans = [
+            _span("s1", "Apple", 0, 5),
+            _span("s2", "text", 1000, 1004),  # beyond source length
+        ]
+        rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
+        assert rate == 0.5
+        assert valid == 1
+        assert total == 2
+
+    def test_negative_offsets(self) -> None:
+        spans = [
+            _span("s1", "Apple", -1, 5),
+        ]
+        rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
+        assert rate == 0.0
+        assert valid == 0
+        assert total == 1
+
+    def test_start_greater_than_end(self) -> None:
+        spans = [
+            _span("s1", "Apple", 5, 0),
+        ]
+        rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
+        assert rate == 0.0
+        assert valid == 0
+        assert total == 1
+
+    def test_empty_spans(self) -> None:
+        rate, valid, total = compute_offset_validity([], SOURCE_TEXT)
+        assert rate == 1.0
+        assert valid == 0
+        assert total == 0
+
+    def test_empty_text_span_at_boundary(self) -> None:
+        # An empty span (start == end) should match empty string
+        spans = [_span("s1", "", 5, 5)]
+        rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
+        assert rate == 1.0
+        assert valid == 1
+
+    def test_all_invalid(self) -> None:
+        spans = [
+            _span("s1", "WRONG", 0, 5),
+            _span("s2", "ALSO_WRONG", 10, 20),
+        ]
+        rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
+        assert rate == 0.0
+        assert valid == 0
+        assert total == 2
+
+
+# ---------------------------------------------------------------------------
+# Support Rate
+# ---------------------------------------------------------------------------
+
+
+class TestSupportRate:
+    def test_all_supported(self) -> None:
+        valid_ids = {"s1", "s2"}
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1"]),
+            _item("i2", FieldType.fact, evidence_ids=["s2"]),
+        ]
+        rate, supported, total = compute_support_rate(items, valid_ids)
+        assert rate == 1.0
+        assert supported == 2
+        assert total == 2
+
+    def test_none_supported(self) -> None:
+        valid_ids = {"s1"}
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s99"]),
+            _item("i2", FieldType.fact, evidence_ids=["s100"]),
+        ]
+        rate, supported, total = compute_support_rate(items, valid_ids)
+        assert rate == 0.0
+        assert supported == 0
+        assert total == 2
+
+    def test_partial_support(self) -> None:
+        valid_ids = {"s1"}
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1"]),
+            _item("i2", FieldType.fact, evidence_ids=["s99"]),
+        ]
+        rate, supported, total = compute_support_rate(items, valid_ids)
+        assert rate == 0.5
+        assert supported == 1
+        assert total == 2
+
+    def test_item_with_multiple_evidence_one_valid(self) -> None:
+        valid_ids = {"s2"}
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1", "s2"]),
+        ]
+        rate, supported, total = compute_support_rate(items, valid_ids)
+        assert rate == 1.0
+        assert supported == 1
+
+    def test_empty_items(self) -> None:
+        rate, supported, total = compute_support_rate([], {"s1"})
+        assert rate == 1.0
+        assert supported == 0
+        assert total == 0
+
+    def test_item_with_no_evidence_ids(self) -> None:
+        valid_ids = {"s1"}
+        items = [_item("i1", FieldType.entity, evidence_ids=[])]
+        rate, supported, total = compute_support_rate(items, valid_ids)
+        assert rate == 0.0
+        assert supported == 0
+
+
+# ---------------------------------------------------------------------------
+# Coverage Score
+# ---------------------------------------------------------------------------
+
+
+class TestCoverageScore:
+    def test_full_coverage(self) -> None:
+        items = [
+            _item(
+                "i1", FieldType.entity,
+                required_fields=["name", "type"],
+                supported_fields=["name", "type"],
+            ),
+        ]
+        score = compute_coverage_score(items)
+        assert score == 1.0
+
+    def test_partial_coverage(self) -> None:
+        items = [
+            _item(
+                "i1", FieldType.entity,
+                required_fields=["name", "type", "value"],
+                supported_fields=["name"],
+            ),
+        ]
+        score = compute_coverage_score(items)
+        assert abs(score - 1 / 3) < 1e-9
+
+    def test_no_coverage(self) -> None:
+        items = [
+            _item(
+                "i1", FieldType.entity,
+                required_fields=["name", "type"],
+                supported_fields=[],
+            ),
+        ]
+        score = compute_coverage_score(items)
+        assert score == 0.0
+
+    def test_no_required_fields_full_coverage(self) -> None:
+        items = [
+            _item("i1", FieldType.entity, required_fields=[], supported_fields=[]),
+        ]
+        score = compute_coverage_score(items)
+        assert score == 1.0
+
+    def test_average_across_items(self) -> None:
+        items = [
+            _item(
+                "i1", FieldType.entity,
+                required_fields=["name", "type"],
+                supported_fields=["name", "type"],
+            ),  # 1.0
+            _item(
+                "i2", FieldType.fact,
+                required_fields=["value", "unit"],
+                supported_fields=["value"],
+            ),  # 0.5
+        ]
+        score = compute_coverage_score(items)
+        assert abs(score - 0.75) < 1e-9
+
+    def test_empty_items(self) -> None:
+        score = compute_coverage_score([])
+        assert score == 1.0
+
+    def test_supported_field_not_in_required(self) -> None:
+        # Extra supported fields beyond required don't inflate the score
+        items = [
+            _item(
+                "i1", FieldType.entity,
+                required_fields=["name"],
+                supported_fields=["name", "extra_field"],
+            ),
+        ]
+        score = compute_coverage_score(items)
+        assert score == 1.0
+
+
+# ---------------------------------------------------------------------------
+# Orphan Rate
+# ---------------------------------------------------------------------------
+
+
+class TestOrphanRate:
+    def test_no_orphans(self) -> None:
+        spans = [_span("s1", "Apple", 0, 5), _span("s2", "revenue", 15, 22)]
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1"]),
+            _item("i2", FieldType.fact, evidence_ids=["s2"]),
+        ]
+        rate, count = compute_orphan_rate(spans, items)
+        assert rate == 0.0
+        assert count == 0
+
+    def test_all_orphans(self) -> None:
+        spans = [_span("s1", "Apple", 0, 5), _span("s2", "revenue", 15, 22)]
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s99"]),
+        ]
+        rate, count = compute_orphan_rate(spans, items)
+        assert rate == 1.0
+        assert count == 2
+
+    def test_partial_orphans(self) -> None:
+        spans = [
+            _span("s1", "Apple", 0, 5),
+            _span("s2", "revenue", 15, 22),
+            _span("s3", "Q3 2024", 43, 50),
+        ]
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1"]),
+        ]
+        rate, count = compute_orphan_rate(spans, items)
+        assert abs(rate - 2 / 3) < 1e-9
+        assert count == 2
+
+    def test_empty_spans(self) -> None:
+        items = [_item("i1", FieldType.entity, evidence_ids=["s1"])]
+        rate, count = compute_orphan_rate([], items)
+        assert rate == 0.0
+        assert count == 0
+
+    def test_empty_items_all_orphans(self) -> None:
+        spans = [_span("s1", "Apple", 0, 5)]
+        rate, count = compute_orphan_rate(spans, [])
+        assert rate == 1.0
+        assert count == 1
+
+    def test_shared_evidence(self) -> None:
+        # Multiple items referencing the same span - span is not orphan
+        spans = [_span("s1", "Apple", 0, 5)]
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1"]),
+            _item("i2", FieldType.sentiment, evidence_ids=["s1"]),
+        ]
+        rate, count = compute_orphan_rate(spans, items)
+        assert rate == 0.0
+        assert count == 0
+
+
+# ---------------------------------------------------------------------------
+# Per-Field Support
+# ---------------------------------------------------------------------------
+
+
+class TestPerFieldSupport:
+    def test_all_types_supported(self) -> None:
+        valid_ids = {"s1", "s2", "s3", "s4"}
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1"]),
+            _item("i2", FieldType.event, evidence_ids=["s2"]),
+            _item("i3", FieldType.fact, evidence_ids=["s3"]),
+            _item("i4", FieldType.sentiment, evidence_ids=["s4"]),
+        ]
+        result = compute_per_field_support(items, valid_ids)
+        assert result["entity"] == 1.0
+        assert result["event"] == 1.0
+        assert result["fact"] == 1.0
+        assert result["sentiment"] == 1.0
+
+    def test_mixed_support(self) -> None:
+        valid_ids = {"s1"}
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1"]),
+            _item("i2", FieldType.entity, evidence_ids=["s99"]),
+            _item("i3", FieldType.fact, evidence_ids=["s1"]),
+        ]
+        result = compute_per_field_support(items, valid_ids)
+        assert result["entity"] == 0.5
+        assert result["fact"] == 1.0
+
+    def test_empty_items(self) -> None:
+        result = compute_per_field_support([], {"s1"})
+        assert result == {}
+
+    def test_single_type(self) -> None:
+        valid_ids = {"s1"}
+        items = [
+            _item("i1", FieldType.sentiment, evidence_ids=["s1"]),
+            _item("i2", FieldType.sentiment, evidence_ids=["s1"]),
+        ]
+        result = compute_per_field_support(items, valid_ids)
+        assert len(result) == 1
+        assert result["sentiment"] == 1.0
+
+
+# ---------------------------------------------------------------------------
+# Unsupported Claim Rate
+# ---------------------------------------------------------------------------
+
+
+class TestUnsupportedClaimRate:
+    def test_all_supported(self) -> None:
+        valid_ids = {"s1", "s2"}
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1"]),
+            _item("i2", FieldType.fact, evidence_ids=["s2"]),
+        ]
+        rate = compute_unsupported_claim_rate(items, valid_ids)
+        assert rate == 0.0
+
+    def test_all_unsupported_no_evidence(self) -> None:
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=[]),
+            _item("i2", FieldType.fact, evidence_ids=[]),
+        ]
+        rate = compute_unsupported_claim_rate(items, {"s1"})
+        assert rate == 1.0
+
+    def test_all_unsupported_invalid_evidence(self) -> None:
+        valid_ids = {"s1"}
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s99"]),
+            _item("i2", FieldType.fact, evidence_ids=["s100"]),
+        ]
+        rate = compute_unsupported_claim_rate(items, valid_ids)
+        assert rate == 1.0
+
+    def test_partial_unsupported(self) -> None:
+        valid_ids = {"s1"}
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1"]),
+            _item("i2", FieldType.fact, evidence_ids=[]),
+            _item("i3", FieldType.event, evidence_ids=["s99"]),
+        ]
+        rate = compute_unsupported_claim_rate(items, valid_ids)
+        assert abs(rate - 2 / 3) < 1e-9
+
+    def test_empty_items(self) -> None:
+        rate = compute_unsupported_claim_rate([], {"s1"})
+        assert rate == 0.0
+
+    def test_mixed_evidence_one_valid(self) -> None:
+        valid_ids = {"s2"}
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1", "s2"]),
+        ]
+        rate = compute_unsupported_claim_rate(items, valid_ids)
+        assert rate == 0.0
+
+
+# ---------------------------------------------------------------------------
+# Full Evaluation
+# ---------------------------------------------------------------------------
+
+
+class TestEvaluateEvidence:
+    def test_perfect_evaluation(self) -> None:
+        source = "Apple reported revenue of $94.8 billion"
+        spans = [
+            _span("s1", "Apple", 0, 5),
+            _span("s2", "$94.8 billion", 26, 39),
+        ]
+        items = [
+            _item(
+                "i1", FieldType.entity, evidence_ids=["s1"],
+                required_fields=["name"], supported_fields=["name"],
+            ),
+            _item(
+                "i2", FieldType.fact, evidence_ids=["s2"],
+                required_fields=["value", "unit"], supported_fields=["value", "unit"],
+            ),
+        ]
+        result = evaluate_evidence(spans, source, items)
+
+        assert isinstance(result, EvidenceMetricsResult)
+        assert result.validity_rate == 1.0
+        assert result.support_rate == 1.0
+        assert result.coverage_score == 1.0
+        assert result.orphan_rate == 0.0
+        assert result.unsupported_claim_rate == 0.0
+        assert result.total_spans == 2
+        assert result.valid_spans == 2
+        assert result.total_items == 2
+        assert result.supported_items == 2
+        assert result.orphan_spans == 0
+
+    def test_evaluation_with_invalid_spans(self) -> None:
+        source = "Apple reported revenue"
+        spans = [
+            _span("s1", "Apple", 0, 5),  # valid
+            _span("s2", "WRONG", 6, 14),  # invalid text
+        ]
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1"]),
+            _item("i2", FieldType.fact, evidence_ids=["s2"]),
+        ]
+        result = evaluate_evidence(spans, source, items)
+
+        assert result.validity_rate == 0.5
+        assert result.support_rate == 0.5  # only i1 has valid evidence
+        assert result.unsupported_claim_rate == 0.5
+
+    def test_evaluation_with_orphans(self) -> None:
+        source = "Apple reported revenue of $94.8 billion"
+        spans = [
+            _span("s1", "Apple", 0, 5),
+            _span("s2", "revenue", 15, 22),
+            _span("s3", "$94.8 billion", 26, 39),
+        ]
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1"]),
+        ]
+        result = evaluate_evidence(spans, source, items)
+
+        assert result.validity_rate == 1.0
+        assert result.support_rate == 1.0
+        assert abs(result.orphan_rate - 2 / 3) < 1e-9
+        assert result.orphan_spans == 2
+
+    def test_evaluation_empty_inputs(self) -> None:
+        result = evaluate_evidence([], "", [])
+
+        assert result.validity_rate == 1.0
+        assert result.support_rate == 1.0
+        assert result.coverage_score == 1.0
+        assert result.orphan_rate == 0.0
+        assert result.unsupported_claim_rate == 0.0
+        assert result.total_spans == 0
+        assert result.total_items == 0
+
+    def test_per_field_support_in_report(self) -> None:
+        source = "Apple reported strong growth in Q3"
+        spans = [
+            _span("s1", "Apple", 0, 5),
+            _span("s2", "strong growth", 15, 28),
+        ]
+        items = [
+            _item("i1", FieldType.entity, evidence_ids=["s1"]),
+            _item("i2", FieldType.sentiment, evidence_ids=["s2"]),
+            _item("i3", FieldType.fact, evidence_ids=["s99"]),  # unsupported
+        ]
+        result = evaluate_evidence(spans, source, items)
+
+        assert result.per_field_support["entity"] == 1.0
+        assert result.per_field_support["sentiment"] == 1.0
+        assert result.per_field_support["fact"] == 0.0
+
+    def test_result_model_fields(self) -> None:
+        result = EvidenceMetricsResult(
+            validity_rate=0.9,
+            support_rate=0.8,
+            coverage_score=0.85,
+            orphan_rate=0.1,
+            per_field_support={"entity": 0.9, "fact": 0.7},
+            unsupported_claim_rate=0.2,
+            total_spans=10,
+            valid_spans=9,
+            total_items=5,
+            supported_items=4,
+            orphan_spans=1,
+        )
+        assert result.validity_rate == 0.9
+        assert result.per_field_support["entity"] == 0.9
+        assert result.orphan_spans == 1