feat: Intelligence Pipeline v3 — full implementation

Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
@@ -0,0 +1,559 @@
+"""Tests for the adjudication layer of Intelligence Pipeline v3.
+
+Covers:
+- Schema models validate correctly
+- Packet building includes only relevant chunks
+- Evidence ID verification catches missing references
+- VRAM gate enforcement
+- Repeated failure routing to review
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from services.intelligence_pipeline_v3.adjudication.deployment import (
+    APPROVED_MODEL,
+    APPROVED_VLLM_VERSION,
+    AlertConfig,
+    ConcurrencySemaphore,
+    check_vram_gate,
+    verify_structured_output,
+)
+from services.intelligence_pipeline_v3.adjudication.prompts import (
+    MAX_OUTPUT_TOKENS,
+    PromptMetadata,
+    build_adjudication_packet,
+    build_request_payload,
+    get_decision_json_schema,
+)
+from services.intelligence_pipeline_v3.adjudication.schemas import (
+    AdjudicationCandidate,
+    AdjudicationDecision,
+    AdjudicationQuestion,
+    CandidateType,
+    ConflictDescription,
+    ConflictType,
+    DecisionVerdict,
+    EvidencePacket,
+    QuestionCode,
+)
+from services.intelligence_pipeline_v3.adjudication.verification import (
+    AdjudicationRecord,
+    preserve_pre_and_post,
+    reject_unsupported_decisions,
+    route_repeated_failures,
+    verify_evidence_references,
+)
+from services.intelligence_pipeline_v3.segmenter.models import DocumentChunk
+
+# --- Fixtures ---
+
+
+def _make_chunk(chunk_id: str, doc_id: str = "doc-1", text: str = "Sample text") -> DocumentChunk:
+    return DocumentChunk(
+        chunk_id=chunk_id,
+        document_id=doc_id,
+        document_type="article",
+        start_char=0,
+        end_char=len(text),
+        text=text,
+    )
+
+
+def _make_evidence(evidence_id: str, chunk_id: str = "chunk-1") -> EvidencePacket:
+    return EvidencePacket(
+        evidence_id=evidence_id,
+        chunk_id=chunk_id,
+        start_char=0,
+        end_char=10,
+        text="Evidence text",
+        source_document_id="doc-1",
+    )
+
+
+def _make_candidate(
+    candidate_id: str,
+    source_chunk_ids: list[str] | None = None,
+    evidence_ids: list[str] | None = None,
+) -> AdjudicationCandidate:
+    return AdjudicationCandidate(
+        candidate_id=candidate_id,
+        candidate_type=CandidateType.ENTITY,
+        label="Test Candidate",
+        source_chunk_ids=source_chunk_ids or [],
+        evidence_ids=evidence_ids or [],
+    )
+
+
+def _make_decision(
+    decision_id: str = "dec-1",
+    evidence_ids: list[str] | None = None,
+    candidate_ids: list[str] | None = None,
+) -> AdjudicationDecision:
+    return AdjudicationDecision(
+        decision_id=decision_id,
+        question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
+        verdict=DecisionVerdict.ACCEPT,
+        candidate_ids=candidate_ids or ["cand-1"],
+        evidence_ids=evidence_ids or ["ev-1"],
+        reasoning="Test reasoning",
+    )
+
+
+# --- Task 32: Schema model validation tests ---
+
+
+class TestAdjudicationSchemas:
+    """Test that schema models validate correctly."""
+
+    def test_adjudication_candidate_valid(self):
+        candidate = AdjudicationCandidate(
+            candidate_id="cand-1",
+            candidate_type=CandidateType.ENTITY,
+            label="Apple Inc.",
+            source_chunk_ids=["chunk-1", "chunk-2"],
+            evidence_ids=["ev-1"],
+            score=0.85,
+        )
+        assert candidate.candidate_id == "cand-1"
+        assert candidate.candidate_type == CandidateType.ENTITY
+        assert candidate.score == 0.85
+
+    def test_adjudication_candidate_score_bounds(self):
+        with pytest.raises(Exception):
+            AdjudicationCandidate(
+                candidate_id="cand-1",
+                candidate_type=CandidateType.ENTITY,
+                label="Test",
+                score=1.5,  # Over 1.0
+            )
+
+    def test_conflict_description_requires_two_candidates(self):
+        with pytest.raises(Exception):
+            ConflictDescription(
+                conflict_id="conf-1",
+                conflict_type=ConflictType.CONTRADICTORY_VALUES,
+                candidate_ids=["only-one"],  # Needs at least 2
+                description="Test conflict",
+            )
+
+    def test_conflict_description_valid(self):
+        conflict = ConflictDescription(
+            conflict_id="conf-1",
+            conflict_type=ConflictType.AMBIGUOUS_IDENTITY,
+            candidate_ids=["cand-1", "cand-2"],
+            description="Two candidates for same entity",
+            evidence_ids=["ev-1"],
+        )
+        assert len(conflict.candidate_ids) == 2
+
+    def test_adjudication_question_valid(self):
+        question = AdjudicationQuestion(
+            question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
+            description="Which company does AAPL refer to here?",
+            candidate_ids=["cand-1", "cand-2"],
+        )
+        assert question.question_code == QuestionCode.RESOLVE_ENTITY_IDENTITY
+
+    def test_evidence_packet_valid(self):
+        evidence = EvidencePacket(
+            evidence_id="ev-1",
+            chunk_id="chunk-1",
+            start_char=10,
+            end_char=50,
+            text="Apple reported revenue of $94.8B",
+            source_document_id="doc-1",
+        )
+        assert evidence.start_char == 10
+        assert evidence.end_char == 50
+
+    def test_decision_excludes_confidence_novelty_impact_horizon(self):
+        """Task 32.2: Decision model excludes confidence, novelty, impact, horizon."""
+        fields = set(AdjudicationDecision.model_fields.keys())
+        # These fields MUST NOT be in the decision model
+        assert "confidence" not in fields
+        assert "novelty" not in fields
+        assert "impact" not in fields
+        assert "impact_score" not in fields
+        assert "horizon" not in fields
+        assert "impact_horizon" not in fields
+
+    def test_decision_requires_evidence_ids(self):
+        """Task 32.3: Every decision requires evidence_ids."""
+        with pytest.raises(Exception):
+            AdjudicationDecision(
+                decision_id="dec-1",
+                question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
+                verdict=DecisionVerdict.ACCEPT,
+                candidate_ids=["cand-1"],
+                evidence_ids=[],  # Empty — min_length=1 should reject
+                reasoning="No evidence",
+            )
+
+    def test_decision_valid_with_evidence(self):
+        """Task 32.3: Decision with evidence_ids is accepted."""
+        decision = AdjudicationDecision(
+            decision_id="dec-1",
+            question_code=QuestionCode.RESOLVE_CAUSAL_DIRECTION,
+            verdict=DecisionVerdict.MERGE,
+            candidate_ids=["cand-1", "cand-2"],
+            evidence_ids=["ev-1", "ev-2"],
+            reasoning="Both refer to same event",
+            resolved_value={"merged_event": "earnings_beat"},
+        )
+        assert len(decision.evidence_ids) == 2
+        assert decision.verdict == DecisionVerdict.MERGE
+
+
+# --- Task 33: Focused adjudication prompts tests ---
+
+
+class TestAdjudicationPrompts:
+    """Test packet building and prompt configuration."""
+
+    def test_packet_includes_only_relevant_chunks(self):
+        """Task 33.1: Packet includes only relevant chunks."""
+        chunks = [
+            _make_chunk("chunk-1", text="Relevant chunk about Apple"),
+            _make_chunk("chunk-2", text="Irrelevant chunk about weather"),
+            _make_chunk("chunk-3", text="Another relevant chunk"),
+        ]
+        candidates = [
+            _make_candidate("cand-1", source_chunk_ids=["chunk-1", "chunk-3"]),
+        ]
+        questions = [
+            AdjudicationQuestion(
+                question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
+                description="Resolve Apple identity",
+                candidate_ids=["cand-1"],
+            ),
+        ]
+        evidence = [_make_evidence("ev-1", "chunk-1")]
+
+        packet = build_adjudication_packet(
+            document_id="doc-1",
+            document_type="article",
+            document_chunks=chunks,
+            candidates=candidates,
+            conflicts=[],
+            questions=questions,
+            evidence=evidence,
+        )
+
+        # Only chunk-1 and chunk-3 should be included
+        chunk_ids = [c.chunk_id for c in packet.relevant_chunks]
+        assert "chunk-1" in chunk_ids
+        assert "chunk-3" in chunk_ids
+        assert "chunk-2" not in chunk_ids
+
+    def test_packet_uses_strict_json_schema(self):
+        """Task 33.2: Uses strict JSON Schema and temperature zero."""
+        schema = get_decision_json_schema()
+        assert schema["type"] == "object"
+        assert "decisions" in schema["properties"]
+        assert schema["additionalProperties"] is False
+
+        # Verify required evidence_ids in decisions
+        decision_schema = schema["properties"]["decisions"]["items"]
+        assert "evidence_ids" in decision_schema["required"]
+
+    def test_request_payload_temperature_zero(self):
+        """Task 33.2: Temperature is zero for deterministic output."""
+        chunks = [_make_chunk("chunk-1")]
+        candidates = [_make_candidate("cand-1", source_chunk_ids=["chunk-1"])]
+        questions = [
+            AdjudicationQuestion(
+                question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
+                description="Resolve identity",
+            ),
+        ]
+        evidence = [_make_evidence("ev-1")]
+
+        packet = build_adjudication_packet(
+            document_id="doc-1",
+            document_type="article",
+            document_chunks=chunks,
+            candidates=candidates,
+            conflicts=[],
+            questions=questions,
+            evidence=evidence,
+        )
+
+        payload = build_request_payload(packet)
+        assert payload["temperature"] == 0.0
+        assert payload["response_format"]["type"] == "json_schema"
+        assert payload["response_format"]["json_schema"]["strict"] is True
+
+    def test_bounded_output_budget(self):
+        """Task 33.3: Bounded output budget max 1536 tokens."""
+        assert MAX_OUTPUT_TOKENS == 1536
+
+        chunks = [_make_chunk("chunk-1")]
+        candidates = [_make_candidate("cand-1", source_chunk_ids=["chunk-1"])]
+        questions = [
+            AdjudicationQuestion(
+                question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
+                description="Test",
+            ),
+        ]
+        evidence = [_make_evidence("ev-1")]
+
+        packet = build_adjudication_packet(
+            document_id="doc-1",
+            document_type="article",
+            document_chunks=chunks,
+            candidates=candidates,
+            conflicts=[],
+            questions=questions,
+            evidence=evidence,
+        )
+
+        payload = build_request_payload(packet)
+        assert payload["max_tokens"] == 1536
+
+    def test_prompt_metadata_fields(self):
+        """Task 33.4: PromptMetadata has version, schema_version, provider lineage."""
+        meta = PromptMetadata()
+        assert meta.prompt_version == "1.0.0"
+        assert meta.schema_version == "1.0.0"
+        assert meta.provider_lineage == "adjudication_v3"
+        assert meta.max_output_tokens == 1536
+        assert meta.temperature == 0.0
+
+
+# --- Task 34: 9B deployment config tests ---
+
+
+class TestDeploymentConfig:
+    """Test deployment constants and VRAM gating."""
+
+    def test_approved_model_constant(self):
+        """Task 34.1: Pinned approved model."""
+        assert APPROVED_MODEL == "AxionML/Qwen3.5-9B-NVFP4"
+
+    def test_approved_vllm_version(self):
+        """Task 34.1: Pinned vLLM version."""
+        assert APPROVED_VLLM_VERSION == "0.8.5"
+
+    def test_verify_structured_output_passes(self):
+        """Task 34.2: Verify structured output with valid target."""
+        target = {
+            "capabilities": {"json_schema": True},
+            "served_model_name": "stonks-adjudicator-9b",
+            "vllm_version": "0.8.5",
+            "model": "AxionML/Qwen3.5-9B-NVFP4",
+        }
+        assert verify_structured_output(target) is True
+
+    def test_verify_structured_output_fails_no_schema(self):
+        """Task 34.2: Fails without json_schema capability."""
+        target = {
+            "capabilities": {"json_schema": False},
+            "served_model_name": "stonks-adjudicator-9b",
+            "vllm_version": "0.8.5",
+        }
+        assert verify_structured_output(target) is False
+
+    def test_verify_structured_output_fails_wrong_model(self):
+        """Task 34.2: Fails with wrong model name."""
+        target = {
+            "capabilities": {"json_schema": True},
+            "served_model_name": "stonks-adjudicator-9b",
+            "vllm_version": "0.8.5",
+            "model": "wrong-model/7B",
+        }
+        assert verify_structured_output(target) is False
+
+    def test_vram_gate_within_limit(self):
+        """Task 34.3: VRAM within +5% passes."""
+        baseline = 10000.0  # 10 GB
+        peak = 10400.0  # 4% over -> passes
+        assert check_vram_gate(peak, baseline) is True
+
+    def test_vram_gate_at_limit(self):
+        """Task 34.3: VRAM at exactly +5% passes."""
+        baseline = 10000.0
+        peak = 10500.0  # Exactly 5%
+        assert check_vram_gate(peak, baseline) is True
+
+    def test_vram_gate_over_limit(self):
+        """Task 34.3: VRAM over +5% fails."""
+        baseline = 10000.0
+        peak = 10501.0  # Just over 5%
+        assert check_vram_gate(peak, baseline) is False
+
+    def test_vram_gate_zero_baseline(self):
+        """Task 34.3: Zero baseline returns False."""
+        assert check_vram_gate(100.0, 0.0) is False
+
+    def test_concurrency_semaphore_defaults(self):
+        """Task 34.4: Semaphore defaults match vLLM max-num-seqs."""
+        sem_config = ConcurrencySemaphore()
+        assert sem_config.max_concurrent == 8
+        assert sem_config.queue_timeout_seconds == 120.0
+
+    def test_concurrency_semaphore_creates_asyncio_semaphore(self):
+        """Task 34.4: Can create an asyncio semaphore."""
+        sem_config = ConcurrencySemaphore(max_concurrent=4)
+        sem = sem_config.create_semaphore()
+        # asyncio.Semaphore has _value attribute
+        assert sem._value == 4
+
+    def test_alert_config_defaults(self):
+        """Task 34.5: Alert config has queue-depth and availability thresholds."""
+        config = AlertConfig()
+        assert config.queue_depth_warning == 16
+        assert config.queue_depth_critical == 32
+        assert config.availability_threshold_percent == 95.0
+        assert config.consecutive_failures_alert == 3
+
+    def test_alert_config_custom(self):
+        """Task 34.5: Alert config accepts custom values."""
+        config = AlertConfig(
+            queue_depth_warning=8,
+            queue_depth_critical=16,
+            availability_threshold_percent=99.0,
+            latency_p95_warning_ms=3000,
+        )
+        assert config.queue_depth_warning == 8
+        assert config.latency_p95_warning_ms == 3000
+
+
+# --- Task 35: Post-adjudication verification tests ---
+
+
+class TestPostAdjudicationVerification:
+    """Test evidence verification and failure routing."""
+
+    def test_verify_evidence_references_all_present(self):
+        """Task 35.1: No missing refs when all evidence IDs are in packet."""
+        chunks = [_make_chunk("chunk-1")]
+        evidence = [_make_evidence("ev-1"), _make_evidence("ev-2")]
+        candidates = [_make_candidate("cand-1", source_chunk_ids=["chunk-1"])]
+        questions = [
+            AdjudicationQuestion(
+                question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
+                description="Test",
+            ),
+        ]
+
+        packet = build_adjudication_packet(
+            document_id="doc-1",
+            document_type="article",
+            document_chunks=chunks,
+            candidates=candidates,
+            conflicts=[],
+            questions=questions,
+            evidence=evidence,
+        )
+
+        decision = _make_decision(evidence_ids=["ev-1", "ev-2"])
+        missing = verify_evidence_references(decision, packet)
+        assert missing == []
+
+    def test_verify_evidence_references_catches_missing(self):
+        """Task 35.1: Catches evidence IDs not in the packet."""
+        chunks = [_make_chunk("chunk-1")]
+        evidence = [_make_evidence("ev-1")]
+        candidates = [_make_candidate("cand-1", source_chunk_ids=["chunk-1"])]
+        questions = [
+            AdjudicationQuestion(
+                question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
+                description="Test",
+            ),
+        ]
+
+        packet = build_adjudication_packet(
+            document_id="doc-1",
+            document_type="article",
+            document_chunks=chunks,
+            candidates=candidates,
+            conflicts=[],
+            questions=questions,
+            evidence=evidence,
+        )
+
+        # Decision references ev-3 which is NOT in the packet
+        decision = _make_decision(evidence_ids=["ev-1", "ev-3"])
+        missing = verify_evidence_references(decision, packet)
+        assert "ev-3" in missing
+        assert "ev-1" not in missing
+
+    def test_reject_unsupported_empty_evidence(self):
+        """Task 35.2: Rejects decisions with empty evidence_ids."""
+        # Create decision manually bypassing validator
+        decision = AdjudicationDecision.model_construct(
+            decision_id="dec-1",
+            question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
+            verdict=DecisionVerdict.ACCEPT,
+            candidate_ids=["cand-1"],
+            evidence_ids=[],
+            reasoning="No evidence",
+            resolved_value={},
+        )
+        result = reject_unsupported_decisions(decision)
+        assert result.rejected is True
+        assert any("empty_evidence" in r.value for r in result.reasons)
+
+    def test_reject_unsupported_invalid_candidate_ref(self):
+        """Task 35.2: Rejects decisions referencing invalid candidates."""
+        decision = _make_decision(candidate_ids=["cand-99"])
+        result = reject_unsupported_decisions(
+            decision,
+            valid_candidate_ids={"cand-1", "cand-2"},
+        )
+        assert result.rejected is True
+
+    def test_accept_valid_decision(self):
+        """Task 35.2: Accepts schema-compatible decisions."""
+        decision = _make_decision(
+            evidence_ids=["ev-1"],
+            candidate_ids=["cand-1"],
+        )
+        result = reject_unsupported_decisions(
+            decision,
+            valid_candidate_ids={"cand-1"},
+            valid_evidence_ids={"ev-1"},
+        )
+        assert result.rejected is False
+        assert result.reasons == []
+
+    def test_preserve_pre_and_post(self):
+        """Task 35.3: Stores both pre-candidates and post-decisions."""
+        candidates = [
+            _make_candidate("cand-1"),
+            _make_candidate("cand-2"),
+        ]
+        decisions = [_make_decision("dec-1")]
+
+        record = preserve_pre_and_post(
+            document_id="doc-1",
+            pre_candidates=candidates,
+            post_decisions=decisions,
+            packet_evidence_ids=["ev-1", "ev-2"],
+        )
+
+        assert isinstance(record, AdjudicationRecord)
+        assert record.document_id == "doc-1"
+        assert len(record.pre_candidates) == 2
+        assert len(record.post_decisions) == 1
+        assert record.packet_evidence_ids == ["ev-1", "ev-2"]
+        assert record.timestamp is not None
+
+    def test_route_repeated_failures_to_review(self):
+        """Task 35.4: Routes repeated failures to 'review'."""
+        assert route_repeated_failures(3, 3) == "review"
+        assert route_repeated_failures(5, 3) == "review"
+        assert route_repeated_failures(10, 5) == "review"
+
+    def test_route_never_returns_accept_repaired(self):
+        """Task 35.4: Never returns 'accept_repaired'."""
+        # Even below threshold, should route to review
+        result = route_repeated_failures(1, 3)
+        assert result == "review"
+        assert result != "accept_repaired"
+
+        result = route_repeated_failures(0, 3)
+        assert result == "review"
+        assert result != "accept_repaired"