Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
560 lines
20 KiB
Python
560 lines
20 KiB
Python
"""Tests for the adjudication layer of Intelligence Pipeline v3.
|
|
|
|
Covers:
|
|
- Schema models validate correctly
|
|
- Packet building includes only relevant chunks
|
|
- Evidence ID verification catches missing references
|
|
- VRAM gate enforcement
|
|
- Repeated failure routing to review
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from services.intelligence_pipeline_v3.adjudication.deployment import (
|
|
APPROVED_MODEL,
|
|
APPROVED_VLLM_VERSION,
|
|
AlertConfig,
|
|
ConcurrencySemaphore,
|
|
check_vram_gate,
|
|
verify_structured_output,
|
|
)
|
|
from services.intelligence_pipeline_v3.adjudication.prompts import (
|
|
MAX_OUTPUT_TOKENS,
|
|
PromptMetadata,
|
|
build_adjudication_packet,
|
|
build_request_payload,
|
|
get_decision_json_schema,
|
|
)
|
|
from services.intelligence_pipeline_v3.adjudication.schemas import (
|
|
AdjudicationCandidate,
|
|
AdjudicationDecision,
|
|
AdjudicationQuestion,
|
|
CandidateType,
|
|
ConflictDescription,
|
|
ConflictType,
|
|
DecisionVerdict,
|
|
EvidencePacket,
|
|
QuestionCode,
|
|
)
|
|
from services.intelligence_pipeline_v3.adjudication.verification import (
|
|
AdjudicationRecord,
|
|
preserve_pre_and_post,
|
|
reject_unsupported_decisions,
|
|
route_repeated_failures,
|
|
verify_evidence_references,
|
|
)
|
|
from services.intelligence_pipeline_v3.segmenter.models import DocumentChunk
|
|
|
|
# --- Fixtures ---
|
|
|
|
|
|
def _make_chunk(chunk_id: str, doc_id: str = "doc-1", text: str = "Sample text") -> DocumentChunk:
|
|
return DocumentChunk(
|
|
chunk_id=chunk_id,
|
|
document_id=doc_id,
|
|
document_type="article",
|
|
start_char=0,
|
|
end_char=len(text),
|
|
text=text,
|
|
)
|
|
|
|
|
|
def _make_evidence(evidence_id: str, chunk_id: str = "chunk-1") -> EvidencePacket:
|
|
return EvidencePacket(
|
|
evidence_id=evidence_id,
|
|
chunk_id=chunk_id,
|
|
start_char=0,
|
|
end_char=10,
|
|
text="Evidence text",
|
|
source_document_id="doc-1",
|
|
)
|
|
|
|
|
|
def _make_candidate(
|
|
candidate_id: str,
|
|
source_chunk_ids: list[str] | None = None,
|
|
evidence_ids: list[str] | None = None,
|
|
) -> AdjudicationCandidate:
|
|
return AdjudicationCandidate(
|
|
candidate_id=candidate_id,
|
|
candidate_type=CandidateType.ENTITY,
|
|
label="Test Candidate",
|
|
source_chunk_ids=source_chunk_ids or [],
|
|
evidence_ids=evidence_ids or [],
|
|
)
|
|
|
|
|
|
def _make_decision(
|
|
decision_id: str = "dec-1",
|
|
evidence_ids: list[str] | None = None,
|
|
candidate_ids: list[str] | None = None,
|
|
) -> AdjudicationDecision:
|
|
return AdjudicationDecision(
|
|
decision_id=decision_id,
|
|
question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
|
|
verdict=DecisionVerdict.ACCEPT,
|
|
candidate_ids=candidate_ids or ["cand-1"],
|
|
evidence_ids=evidence_ids or ["ev-1"],
|
|
reasoning="Test reasoning",
|
|
)
|
|
|
|
|
|
# --- Task 32: Schema model validation tests ---
|
|
|
|
|
|
class TestAdjudicationSchemas:
|
|
"""Test that schema models validate correctly."""
|
|
|
|
def test_adjudication_candidate_valid(self):
|
|
candidate = AdjudicationCandidate(
|
|
candidate_id="cand-1",
|
|
candidate_type=CandidateType.ENTITY,
|
|
label="Apple Inc.",
|
|
source_chunk_ids=["chunk-1", "chunk-2"],
|
|
evidence_ids=["ev-1"],
|
|
score=0.85,
|
|
)
|
|
assert candidate.candidate_id == "cand-1"
|
|
assert candidate.candidate_type == CandidateType.ENTITY
|
|
assert candidate.score == 0.85
|
|
|
|
def test_adjudication_candidate_score_bounds(self):
|
|
with pytest.raises(Exception):
|
|
AdjudicationCandidate(
|
|
candidate_id="cand-1",
|
|
candidate_type=CandidateType.ENTITY,
|
|
label="Test",
|
|
score=1.5, # Over 1.0
|
|
)
|
|
|
|
def test_conflict_description_requires_two_candidates(self):
|
|
with pytest.raises(Exception):
|
|
ConflictDescription(
|
|
conflict_id="conf-1",
|
|
conflict_type=ConflictType.CONTRADICTORY_VALUES,
|
|
candidate_ids=["only-one"], # Needs at least 2
|
|
description="Test conflict",
|
|
)
|
|
|
|
def test_conflict_description_valid(self):
|
|
conflict = ConflictDescription(
|
|
conflict_id="conf-1",
|
|
conflict_type=ConflictType.AMBIGUOUS_IDENTITY,
|
|
candidate_ids=["cand-1", "cand-2"],
|
|
description="Two candidates for same entity",
|
|
evidence_ids=["ev-1"],
|
|
)
|
|
assert len(conflict.candidate_ids) == 2
|
|
|
|
def test_adjudication_question_valid(self):
|
|
question = AdjudicationQuestion(
|
|
question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
|
|
description="Which company does AAPL refer to here?",
|
|
candidate_ids=["cand-1", "cand-2"],
|
|
)
|
|
assert question.question_code == QuestionCode.RESOLVE_ENTITY_IDENTITY
|
|
|
|
def test_evidence_packet_valid(self):
|
|
evidence = EvidencePacket(
|
|
evidence_id="ev-1",
|
|
chunk_id="chunk-1",
|
|
start_char=10,
|
|
end_char=50,
|
|
text="Apple reported revenue of $94.8B",
|
|
source_document_id="doc-1",
|
|
)
|
|
assert evidence.start_char == 10
|
|
assert evidence.end_char == 50
|
|
|
|
def test_decision_excludes_confidence_novelty_impact_horizon(self):
|
|
"""Task 32.2: Decision model excludes confidence, novelty, impact, horizon."""
|
|
fields = set(AdjudicationDecision.model_fields.keys())
|
|
# These fields MUST NOT be in the decision model
|
|
assert "confidence" not in fields
|
|
assert "novelty" not in fields
|
|
assert "impact" not in fields
|
|
assert "impact_score" not in fields
|
|
assert "horizon" not in fields
|
|
assert "impact_horizon" not in fields
|
|
|
|
def test_decision_requires_evidence_ids(self):
|
|
"""Task 32.3: Every decision requires evidence_ids."""
|
|
with pytest.raises(Exception):
|
|
AdjudicationDecision(
|
|
decision_id="dec-1",
|
|
question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
|
|
verdict=DecisionVerdict.ACCEPT,
|
|
candidate_ids=["cand-1"],
|
|
evidence_ids=[], # Empty — min_length=1 should reject
|
|
reasoning="No evidence",
|
|
)
|
|
|
|
def test_decision_valid_with_evidence(self):
|
|
"""Task 32.3: Decision with evidence_ids is accepted."""
|
|
decision = AdjudicationDecision(
|
|
decision_id="dec-1",
|
|
question_code=QuestionCode.RESOLVE_CAUSAL_DIRECTION,
|
|
verdict=DecisionVerdict.MERGE,
|
|
candidate_ids=["cand-1", "cand-2"],
|
|
evidence_ids=["ev-1", "ev-2"],
|
|
reasoning="Both refer to same event",
|
|
resolved_value={"merged_event": "earnings_beat"},
|
|
)
|
|
assert len(decision.evidence_ids) == 2
|
|
assert decision.verdict == DecisionVerdict.MERGE
|
|
|
|
|
|
# --- Task 33: Focused adjudication prompts tests ---
|
|
|
|
|
|
class TestAdjudicationPrompts:
|
|
"""Test packet building and prompt configuration."""
|
|
|
|
def test_packet_includes_only_relevant_chunks(self):
|
|
"""Task 33.1: Packet includes only relevant chunks."""
|
|
chunks = [
|
|
_make_chunk("chunk-1", text="Relevant chunk about Apple"),
|
|
_make_chunk("chunk-2", text="Irrelevant chunk about weather"),
|
|
_make_chunk("chunk-3", text="Another relevant chunk"),
|
|
]
|
|
candidates = [
|
|
_make_candidate("cand-1", source_chunk_ids=["chunk-1", "chunk-3"]),
|
|
]
|
|
questions = [
|
|
AdjudicationQuestion(
|
|
question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
|
|
description="Resolve Apple identity",
|
|
candidate_ids=["cand-1"],
|
|
),
|
|
]
|
|
evidence = [_make_evidence("ev-1", "chunk-1")]
|
|
|
|
packet = build_adjudication_packet(
|
|
document_id="doc-1",
|
|
document_type="article",
|
|
document_chunks=chunks,
|
|
candidates=candidates,
|
|
conflicts=[],
|
|
questions=questions,
|
|
evidence=evidence,
|
|
)
|
|
|
|
# Only chunk-1 and chunk-3 should be included
|
|
chunk_ids = [c.chunk_id for c in packet.relevant_chunks]
|
|
assert "chunk-1" in chunk_ids
|
|
assert "chunk-3" in chunk_ids
|
|
assert "chunk-2" not in chunk_ids
|
|
|
|
def test_packet_uses_strict_json_schema(self):
|
|
"""Task 33.2: Uses strict JSON Schema and temperature zero."""
|
|
schema = get_decision_json_schema()
|
|
assert schema["type"] == "object"
|
|
assert "decisions" in schema["properties"]
|
|
assert schema["additionalProperties"] is False
|
|
|
|
# Verify required evidence_ids in decisions
|
|
decision_schema = schema["properties"]["decisions"]["items"]
|
|
assert "evidence_ids" in decision_schema["required"]
|
|
|
|
def test_request_payload_temperature_zero(self):
|
|
"""Task 33.2: Temperature is zero for deterministic output."""
|
|
chunks = [_make_chunk("chunk-1")]
|
|
candidates = [_make_candidate("cand-1", source_chunk_ids=["chunk-1"])]
|
|
questions = [
|
|
AdjudicationQuestion(
|
|
question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
|
|
description="Resolve identity",
|
|
),
|
|
]
|
|
evidence = [_make_evidence("ev-1")]
|
|
|
|
packet = build_adjudication_packet(
|
|
document_id="doc-1",
|
|
document_type="article",
|
|
document_chunks=chunks,
|
|
candidates=candidates,
|
|
conflicts=[],
|
|
questions=questions,
|
|
evidence=evidence,
|
|
)
|
|
|
|
payload = build_request_payload(packet)
|
|
assert payload["temperature"] == 0.0
|
|
assert payload["response_format"]["type"] == "json_schema"
|
|
assert payload["response_format"]["json_schema"]["strict"] is True
|
|
|
|
def test_bounded_output_budget(self):
|
|
"""Task 33.3: Bounded output budget max 1536 tokens."""
|
|
assert MAX_OUTPUT_TOKENS == 1536
|
|
|
|
chunks = [_make_chunk("chunk-1")]
|
|
candidates = [_make_candidate("cand-1", source_chunk_ids=["chunk-1"])]
|
|
questions = [
|
|
AdjudicationQuestion(
|
|
question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
|
|
description="Test",
|
|
),
|
|
]
|
|
evidence = [_make_evidence("ev-1")]
|
|
|
|
packet = build_adjudication_packet(
|
|
document_id="doc-1",
|
|
document_type="article",
|
|
document_chunks=chunks,
|
|
candidates=candidates,
|
|
conflicts=[],
|
|
questions=questions,
|
|
evidence=evidence,
|
|
)
|
|
|
|
payload = build_request_payload(packet)
|
|
assert payload["max_tokens"] == 1536
|
|
|
|
def test_prompt_metadata_fields(self):
|
|
"""Task 33.4: PromptMetadata has version, schema_version, provider lineage."""
|
|
meta = PromptMetadata()
|
|
assert meta.prompt_version == "1.0.0"
|
|
assert meta.schema_version == "1.0.0"
|
|
assert meta.provider_lineage == "adjudication_v3"
|
|
assert meta.max_output_tokens == 1536
|
|
assert meta.temperature == 0.0
|
|
|
|
|
|
# --- Task 34: 9B deployment config tests ---
|
|
|
|
|
|
class TestDeploymentConfig:
|
|
"""Test deployment constants and VRAM gating."""
|
|
|
|
def test_approved_model_constant(self):
|
|
"""Task 34.1: Pinned approved model."""
|
|
assert APPROVED_MODEL == "AxionML/Qwen3.5-9B-NVFP4"
|
|
|
|
def test_approved_vllm_version(self):
|
|
"""Task 34.1: Pinned vLLM version."""
|
|
assert APPROVED_VLLM_VERSION == "0.8.5"
|
|
|
|
def test_verify_structured_output_passes(self):
|
|
"""Task 34.2: Verify structured output with valid target."""
|
|
target = {
|
|
"capabilities": {"json_schema": True},
|
|
"served_model_name": "stonks-adjudicator-9b",
|
|
"vllm_version": "0.8.5",
|
|
"model": "AxionML/Qwen3.5-9B-NVFP4",
|
|
}
|
|
assert verify_structured_output(target) is True
|
|
|
|
def test_verify_structured_output_fails_no_schema(self):
|
|
"""Task 34.2: Fails without json_schema capability."""
|
|
target = {
|
|
"capabilities": {"json_schema": False},
|
|
"served_model_name": "stonks-adjudicator-9b",
|
|
"vllm_version": "0.8.5",
|
|
}
|
|
assert verify_structured_output(target) is False
|
|
|
|
def test_verify_structured_output_fails_wrong_model(self):
|
|
"""Task 34.2: Fails with wrong model name."""
|
|
target = {
|
|
"capabilities": {"json_schema": True},
|
|
"served_model_name": "stonks-adjudicator-9b",
|
|
"vllm_version": "0.8.5",
|
|
"model": "wrong-model/7B",
|
|
}
|
|
assert verify_structured_output(target) is False
|
|
|
|
def test_vram_gate_within_limit(self):
|
|
"""Task 34.3: VRAM within +5% passes."""
|
|
baseline = 10000.0 # 10 GB
|
|
peak = 10400.0 # 4% over -> passes
|
|
assert check_vram_gate(peak, baseline) is True
|
|
|
|
def test_vram_gate_at_limit(self):
|
|
"""Task 34.3: VRAM at exactly +5% passes."""
|
|
baseline = 10000.0
|
|
peak = 10500.0 # Exactly 5%
|
|
assert check_vram_gate(peak, baseline) is True
|
|
|
|
def test_vram_gate_over_limit(self):
|
|
"""Task 34.3: VRAM over +5% fails."""
|
|
baseline = 10000.0
|
|
peak = 10501.0 # Just over 5%
|
|
assert check_vram_gate(peak, baseline) is False
|
|
|
|
def test_vram_gate_zero_baseline(self):
|
|
"""Task 34.3: Zero baseline returns False."""
|
|
assert check_vram_gate(100.0, 0.0) is False
|
|
|
|
def test_concurrency_semaphore_defaults(self):
|
|
"""Task 34.4: Semaphore defaults match vLLM max-num-seqs."""
|
|
sem_config = ConcurrencySemaphore()
|
|
assert sem_config.max_concurrent == 8
|
|
assert sem_config.queue_timeout_seconds == 120.0
|
|
|
|
def test_concurrency_semaphore_creates_asyncio_semaphore(self):
|
|
"""Task 34.4: Can create an asyncio semaphore."""
|
|
sem_config = ConcurrencySemaphore(max_concurrent=4)
|
|
sem = sem_config.create_semaphore()
|
|
# asyncio.Semaphore has _value attribute
|
|
assert sem._value == 4
|
|
|
|
def test_alert_config_defaults(self):
|
|
"""Task 34.5: Alert config has queue-depth and availability thresholds."""
|
|
config = AlertConfig()
|
|
assert config.queue_depth_warning == 16
|
|
assert config.queue_depth_critical == 32
|
|
assert config.availability_threshold_percent == 95.0
|
|
assert config.consecutive_failures_alert == 3
|
|
|
|
def test_alert_config_custom(self):
|
|
"""Task 34.5: Alert config accepts custom values."""
|
|
config = AlertConfig(
|
|
queue_depth_warning=8,
|
|
queue_depth_critical=16,
|
|
availability_threshold_percent=99.0,
|
|
latency_p95_warning_ms=3000,
|
|
)
|
|
assert config.queue_depth_warning == 8
|
|
assert config.latency_p95_warning_ms == 3000
|
|
|
|
|
|
# --- Task 35: Post-adjudication verification tests ---
|
|
|
|
|
|
class TestPostAdjudicationVerification:
|
|
"""Test evidence verification and failure routing."""
|
|
|
|
def test_verify_evidence_references_all_present(self):
|
|
"""Task 35.1: No missing refs when all evidence IDs are in packet."""
|
|
chunks = [_make_chunk("chunk-1")]
|
|
evidence = [_make_evidence("ev-1"), _make_evidence("ev-2")]
|
|
candidates = [_make_candidate("cand-1", source_chunk_ids=["chunk-1"])]
|
|
questions = [
|
|
AdjudicationQuestion(
|
|
question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
|
|
description="Test",
|
|
),
|
|
]
|
|
|
|
packet = build_adjudication_packet(
|
|
document_id="doc-1",
|
|
document_type="article",
|
|
document_chunks=chunks,
|
|
candidates=candidates,
|
|
conflicts=[],
|
|
questions=questions,
|
|
evidence=evidence,
|
|
)
|
|
|
|
decision = _make_decision(evidence_ids=["ev-1", "ev-2"])
|
|
missing = verify_evidence_references(decision, packet)
|
|
assert missing == []
|
|
|
|
def test_verify_evidence_references_catches_missing(self):
|
|
"""Task 35.1: Catches evidence IDs not in the packet."""
|
|
chunks = [_make_chunk("chunk-1")]
|
|
evidence = [_make_evidence("ev-1")]
|
|
candidates = [_make_candidate("cand-1", source_chunk_ids=["chunk-1"])]
|
|
questions = [
|
|
AdjudicationQuestion(
|
|
question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
|
|
description="Test",
|
|
),
|
|
]
|
|
|
|
packet = build_adjudication_packet(
|
|
document_id="doc-1",
|
|
document_type="article",
|
|
document_chunks=chunks,
|
|
candidates=candidates,
|
|
conflicts=[],
|
|
questions=questions,
|
|
evidence=evidence,
|
|
)
|
|
|
|
# Decision references ev-3 which is NOT in the packet
|
|
decision = _make_decision(evidence_ids=["ev-1", "ev-3"])
|
|
missing = verify_evidence_references(decision, packet)
|
|
assert "ev-3" in missing
|
|
assert "ev-1" not in missing
|
|
|
|
def test_reject_unsupported_empty_evidence(self):
|
|
"""Task 35.2: Rejects decisions with empty evidence_ids."""
|
|
# Create decision manually bypassing validator
|
|
decision = AdjudicationDecision.model_construct(
|
|
decision_id="dec-1",
|
|
question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY,
|
|
verdict=DecisionVerdict.ACCEPT,
|
|
candidate_ids=["cand-1"],
|
|
evidence_ids=[],
|
|
reasoning="No evidence",
|
|
resolved_value={},
|
|
)
|
|
result = reject_unsupported_decisions(decision)
|
|
assert result.rejected is True
|
|
assert any("empty_evidence" in r.value for r in result.reasons)
|
|
|
|
def test_reject_unsupported_invalid_candidate_ref(self):
|
|
"""Task 35.2: Rejects decisions referencing invalid candidates."""
|
|
decision = _make_decision(candidate_ids=["cand-99"])
|
|
result = reject_unsupported_decisions(
|
|
decision,
|
|
valid_candidate_ids={"cand-1", "cand-2"},
|
|
)
|
|
assert result.rejected is True
|
|
|
|
def test_accept_valid_decision(self):
|
|
"""Task 35.2: Accepts schema-compatible decisions."""
|
|
decision = _make_decision(
|
|
evidence_ids=["ev-1"],
|
|
candidate_ids=["cand-1"],
|
|
)
|
|
result = reject_unsupported_decisions(
|
|
decision,
|
|
valid_candidate_ids={"cand-1"},
|
|
valid_evidence_ids={"ev-1"},
|
|
)
|
|
assert result.rejected is False
|
|
assert result.reasons == []
|
|
|
|
def test_preserve_pre_and_post(self):
|
|
"""Task 35.3: Stores both pre-candidates and post-decisions."""
|
|
candidates = [
|
|
_make_candidate("cand-1"),
|
|
_make_candidate("cand-2"),
|
|
]
|
|
decisions = [_make_decision("dec-1")]
|
|
|
|
record = preserve_pre_and_post(
|
|
document_id="doc-1",
|
|
pre_candidates=candidates,
|
|
post_decisions=decisions,
|
|
packet_evidence_ids=["ev-1", "ev-2"],
|
|
)
|
|
|
|
assert isinstance(record, AdjudicationRecord)
|
|
assert record.document_id == "doc-1"
|
|
assert len(record.pre_candidates) == 2
|
|
assert len(record.post_decisions) == 1
|
|
assert record.packet_evidence_ids == ["ev-1", "ev-2"]
|
|
assert record.timestamp is not None
|
|
|
|
def test_route_repeated_failures_to_review(self):
|
|
"""Task 35.4: Routes repeated failures to 'review'."""
|
|
assert route_repeated_failures(3, 3) == "review"
|
|
assert route_repeated_failures(5, 3) == "review"
|
|
assert route_repeated_failures(10, 5) == "review"
|
|
|
|
def test_route_never_returns_accept_repaired(self):
|
|
"""Task 35.4: Never returns 'accept_repaired'."""
|
|
# Even below threshold, should route to review
|
|
result = route_repeated_failures(1, 3)
|
|
assert result == "review"
|
|
assert result != "accept_repaired"
|
|
|
|
result = route_repeated_failures(0, 3)
|
|
assert result == "review"
|
|
assert result != "accept_repaired"
|