"""Tests for the adjudication layer of Intelligence Pipeline v3. Covers: - Schema models validate correctly - Packet building includes only relevant chunks - Evidence ID verification catches missing references - VRAM gate enforcement - Repeated failure routing to review """ from __future__ import annotations import pytest from services.intelligence_pipeline_v3.adjudication.deployment import ( APPROVED_MODEL, APPROVED_VLLM_VERSION, AlertConfig, ConcurrencySemaphore, check_vram_gate, verify_structured_output, ) from services.intelligence_pipeline_v3.adjudication.prompts import ( MAX_OUTPUT_TOKENS, PromptMetadata, build_adjudication_packet, build_request_payload, get_decision_json_schema, ) from services.intelligence_pipeline_v3.adjudication.schemas import ( AdjudicationCandidate, AdjudicationDecision, AdjudicationQuestion, CandidateType, ConflictDescription, ConflictType, DecisionVerdict, EvidencePacket, QuestionCode, ) from services.intelligence_pipeline_v3.adjudication.verification import ( AdjudicationRecord, preserve_pre_and_post, reject_unsupported_decisions, route_repeated_failures, verify_evidence_references, ) from services.intelligence_pipeline_v3.segmenter.models import DocumentChunk # --- Fixtures --- def _make_chunk(chunk_id: str, doc_id: str = "doc-1", text: str = "Sample text") -> DocumentChunk: return DocumentChunk( chunk_id=chunk_id, document_id=doc_id, document_type="article", start_char=0, end_char=len(text), text=text, ) def _make_evidence(evidence_id: str, chunk_id: str = "chunk-1") -> EvidencePacket: return EvidencePacket( evidence_id=evidence_id, chunk_id=chunk_id, start_char=0, end_char=10, text="Evidence text", source_document_id="doc-1", ) def _make_candidate( candidate_id: str, source_chunk_ids: list[str] | None = None, evidence_ids: list[str] | None = None, ) -> AdjudicationCandidate: return AdjudicationCandidate( candidate_id=candidate_id, candidate_type=CandidateType.ENTITY, label="Test Candidate", source_chunk_ids=source_chunk_ids or [], evidence_ids=evidence_ids or [], ) def _make_decision( decision_id: str = "dec-1", evidence_ids: list[str] | None = None, candidate_ids: list[str] | None = None, ) -> AdjudicationDecision: return AdjudicationDecision( decision_id=decision_id, question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY, verdict=DecisionVerdict.ACCEPT, candidate_ids=candidate_ids or ["cand-1"], evidence_ids=evidence_ids or ["ev-1"], reasoning="Test reasoning", ) # --- Task 32: Schema model validation tests --- class TestAdjudicationSchemas: """Test that schema models validate correctly.""" def test_adjudication_candidate_valid(self): candidate = AdjudicationCandidate( candidate_id="cand-1", candidate_type=CandidateType.ENTITY, label="Apple Inc.", source_chunk_ids=["chunk-1", "chunk-2"], evidence_ids=["ev-1"], score=0.85, ) assert candidate.candidate_id == "cand-1" assert candidate.candidate_type == CandidateType.ENTITY assert candidate.score == 0.85 def test_adjudication_candidate_score_bounds(self): with pytest.raises(Exception): AdjudicationCandidate( candidate_id="cand-1", candidate_type=CandidateType.ENTITY, label="Test", score=1.5, # Over 1.0 ) def test_conflict_description_requires_two_candidates(self): with pytest.raises(Exception): ConflictDescription( conflict_id="conf-1", conflict_type=ConflictType.CONTRADICTORY_VALUES, candidate_ids=["only-one"], # Needs at least 2 description="Test conflict", ) def test_conflict_description_valid(self): conflict = ConflictDescription( conflict_id="conf-1", conflict_type=ConflictType.AMBIGUOUS_IDENTITY, candidate_ids=["cand-1", "cand-2"], description="Two candidates for same entity", evidence_ids=["ev-1"], ) assert len(conflict.candidate_ids) == 2 def test_adjudication_question_valid(self): question = AdjudicationQuestion( question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY, description="Which company does AAPL refer to here?", candidate_ids=["cand-1", "cand-2"], ) assert question.question_code == QuestionCode.RESOLVE_ENTITY_IDENTITY def test_evidence_packet_valid(self): evidence = EvidencePacket( evidence_id="ev-1", chunk_id="chunk-1", start_char=10, end_char=50, text="Apple reported revenue of $94.8B", source_document_id="doc-1", ) assert evidence.start_char == 10 assert evidence.end_char == 50 def test_decision_excludes_confidence_novelty_impact_horizon(self): """Task 32.2: Decision model excludes confidence, novelty, impact, horizon.""" fields = set(AdjudicationDecision.model_fields.keys()) # These fields MUST NOT be in the decision model assert "confidence" not in fields assert "novelty" not in fields assert "impact" not in fields assert "impact_score" not in fields assert "horizon" not in fields assert "impact_horizon" not in fields def test_decision_requires_evidence_ids(self): """Task 32.3: Every decision requires evidence_ids.""" with pytest.raises(Exception): AdjudicationDecision( decision_id="dec-1", question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY, verdict=DecisionVerdict.ACCEPT, candidate_ids=["cand-1"], evidence_ids=[], # Empty — min_length=1 should reject reasoning="No evidence", ) def test_decision_valid_with_evidence(self): """Task 32.3: Decision with evidence_ids is accepted.""" decision = AdjudicationDecision( decision_id="dec-1", question_code=QuestionCode.RESOLVE_CAUSAL_DIRECTION, verdict=DecisionVerdict.MERGE, candidate_ids=["cand-1", "cand-2"], evidence_ids=["ev-1", "ev-2"], reasoning="Both refer to same event", resolved_value={"merged_event": "earnings_beat"}, ) assert len(decision.evidence_ids) == 2 assert decision.verdict == DecisionVerdict.MERGE # --- Task 33: Focused adjudication prompts tests --- class TestAdjudicationPrompts: """Test packet building and prompt configuration.""" def test_packet_includes_only_relevant_chunks(self): """Task 33.1: Packet includes only relevant chunks.""" chunks = [ _make_chunk("chunk-1", text="Relevant chunk about Apple"), _make_chunk("chunk-2", text="Irrelevant chunk about weather"), _make_chunk("chunk-3", text="Another relevant chunk"), ] candidates = [ _make_candidate("cand-1", source_chunk_ids=["chunk-1", "chunk-3"]), ] questions = [ AdjudicationQuestion( question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY, description="Resolve Apple identity", candidate_ids=["cand-1"], ), ] evidence = [_make_evidence("ev-1", "chunk-1")] packet = build_adjudication_packet( document_id="doc-1", document_type="article", document_chunks=chunks, candidates=candidates, conflicts=[], questions=questions, evidence=evidence, ) # Only chunk-1 and chunk-3 should be included chunk_ids = [c.chunk_id for c in packet.relevant_chunks] assert "chunk-1" in chunk_ids assert "chunk-3" in chunk_ids assert "chunk-2" not in chunk_ids def test_packet_uses_strict_json_schema(self): """Task 33.2: Uses strict JSON Schema and temperature zero.""" schema = get_decision_json_schema() assert schema["type"] == "object" assert "decisions" in schema["properties"] assert schema["additionalProperties"] is False # Verify required evidence_ids in decisions decision_schema = schema["properties"]["decisions"]["items"] assert "evidence_ids" in decision_schema["required"] def test_request_payload_temperature_zero(self): """Task 33.2: Temperature is zero for deterministic output.""" chunks = [_make_chunk("chunk-1")] candidates = [_make_candidate("cand-1", source_chunk_ids=["chunk-1"])] questions = [ AdjudicationQuestion( question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY, description="Resolve identity", ), ] evidence = [_make_evidence("ev-1")] packet = build_adjudication_packet( document_id="doc-1", document_type="article", document_chunks=chunks, candidates=candidates, conflicts=[], questions=questions, evidence=evidence, ) payload = build_request_payload(packet) assert payload["temperature"] == 0.0 assert payload["response_format"]["type"] == "json_schema" assert payload["response_format"]["json_schema"]["strict"] is True def test_bounded_output_budget(self): """Task 33.3: Bounded output budget max 1536 tokens.""" assert MAX_OUTPUT_TOKENS == 1536 chunks = [_make_chunk("chunk-1")] candidates = [_make_candidate("cand-1", source_chunk_ids=["chunk-1"])] questions = [ AdjudicationQuestion( question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY, description="Test", ), ] evidence = [_make_evidence("ev-1")] packet = build_adjudication_packet( document_id="doc-1", document_type="article", document_chunks=chunks, candidates=candidates, conflicts=[], questions=questions, evidence=evidence, ) payload = build_request_payload(packet) assert payload["max_tokens"] == 1536 def test_prompt_metadata_fields(self): """Task 33.4: PromptMetadata has version, schema_version, provider lineage.""" meta = PromptMetadata() assert meta.prompt_version == "1.0.0" assert meta.schema_version == "1.0.0" assert meta.provider_lineage == "adjudication_v3" assert meta.max_output_tokens == 1536 assert meta.temperature == 0.0 # --- Task 34: 9B deployment config tests --- class TestDeploymentConfig: """Test deployment constants and VRAM gating.""" def test_approved_model_constant(self): """Task 34.1: Pinned approved model.""" assert APPROVED_MODEL == "AxionML/Qwen3.5-9B-NVFP4" def test_approved_vllm_version(self): """Task 34.1: Pinned vLLM version.""" assert APPROVED_VLLM_VERSION == "0.8.5" def test_verify_structured_output_passes(self): """Task 34.2: Verify structured output with valid target.""" target = { "capabilities": {"json_schema": True}, "served_model_name": "stonks-adjudicator-9b", "vllm_version": "0.8.5", "model": "AxionML/Qwen3.5-9B-NVFP4", } assert verify_structured_output(target) is True def test_verify_structured_output_fails_no_schema(self): """Task 34.2: Fails without json_schema capability.""" target = { "capabilities": {"json_schema": False}, "served_model_name": "stonks-adjudicator-9b", "vllm_version": "0.8.5", } assert verify_structured_output(target) is False def test_verify_structured_output_fails_wrong_model(self): """Task 34.2: Fails with wrong model name.""" target = { "capabilities": {"json_schema": True}, "served_model_name": "stonks-adjudicator-9b", "vllm_version": "0.8.5", "model": "wrong-model/7B", } assert verify_structured_output(target) is False def test_vram_gate_within_limit(self): """Task 34.3: VRAM within +5% passes.""" baseline = 10000.0 # 10 GB peak = 10400.0 # 4% over -> passes assert check_vram_gate(peak, baseline) is True def test_vram_gate_at_limit(self): """Task 34.3: VRAM at exactly +5% passes.""" baseline = 10000.0 peak = 10500.0 # Exactly 5% assert check_vram_gate(peak, baseline) is True def test_vram_gate_over_limit(self): """Task 34.3: VRAM over +5% fails.""" baseline = 10000.0 peak = 10501.0 # Just over 5% assert check_vram_gate(peak, baseline) is False def test_vram_gate_zero_baseline(self): """Task 34.3: Zero baseline returns False.""" assert check_vram_gate(100.0, 0.0) is False def test_concurrency_semaphore_defaults(self): """Task 34.4: Semaphore defaults match vLLM max-num-seqs.""" sem_config = ConcurrencySemaphore() assert sem_config.max_concurrent == 8 assert sem_config.queue_timeout_seconds == 120.0 def test_concurrency_semaphore_creates_asyncio_semaphore(self): """Task 34.4: Can create an asyncio semaphore.""" sem_config = ConcurrencySemaphore(max_concurrent=4) sem = sem_config.create_semaphore() # asyncio.Semaphore has _value attribute assert sem._value == 4 def test_alert_config_defaults(self): """Task 34.5: Alert config has queue-depth and availability thresholds.""" config = AlertConfig() assert config.queue_depth_warning == 16 assert config.queue_depth_critical == 32 assert config.availability_threshold_percent == 95.0 assert config.consecutive_failures_alert == 3 def test_alert_config_custom(self): """Task 34.5: Alert config accepts custom values.""" config = AlertConfig( queue_depth_warning=8, queue_depth_critical=16, availability_threshold_percent=99.0, latency_p95_warning_ms=3000, ) assert config.queue_depth_warning == 8 assert config.latency_p95_warning_ms == 3000 # --- Task 35: Post-adjudication verification tests --- class TestPostAdjudicationVerification: """Test evidence verification and failure routing.""" def test_verify_evidence_references_all_present(self): """Task 35.1: No missing refs when all evidence IDs are in packet.""" chunks = [_make_chunk("chunk-1")] evidence = [_make_evidence("ev-1"), _make_evidence("ev-2")] candidates = [_make_candidate("cand-1", source_chunk_ids=["chunk-1"])] questions = [ AdjudicationQuestion( question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY, description="Test", ), ] packet = build_adjudication_packet( document_id="doc-1", document_type="article", document_chunks=chunks, candidates=candidates, conflicts=[], questions=questions, evidence=evidence, ) decision = _make_decision(evidence_ids=["ev-1", "ev-2"]) missing = verify_evidence_references(decision, packet) assert missing == [] def test_verify_evidence_references_catches_missing(self): """Task 35.1: Catches evidence IDs not in the packet.""" chunks = [_make_chunk("chunk-1")] evidence = [_make_evidence("ev-1")] candidates = [_make_candidate("cand-1", source_chunk_ids=["chunk-1"])] questions = [ AdjudicationQuestion( question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY, description="Test", ), ] packet = build_adjudication_packet( document_id="doc-1", document_type="article", document_chunks=chunks, candidates=candidates, conflicts=[], questions=questions, evidence=evidence, ) # Decision references ev-3 which is NOT in the packet decision = _make_decision(evidence_ids=["ev-1", "ev-3"]) missing = verify_evidence_references(decision, packet) assert "ev-3" in missing assert "ev-1" not in missing def test_reject_unsupported_empty_evidence(self): """Task 35.2: Rejects decisions with empty evidence_ids.""" # Create decision manually bypassing validator decision = AdjudicationDecision.model_construct( decision_id="dec-1", question_code=QuestionCode.RESOLVE_ENTITY_IDENTITY, verdict=DecisionVerdict.ACCEPT, candidate_ids=["cand-1"], evidence_ids=[], reasoning="No evidence", resolved_value={}, ) result = reject_unsupported_decisions(decision) assert result.rejected is True assert any("empty_evidence" in r.value for r in result.reasons) def test_reject_unsupported_invalid_candidate_ref(self): """Task 35.2: Rejects decisions referencing invalid candidates.""" decision = _make_decision(candidate_ids=["cand-99"]) result = reject_unsupported_decisions( decision, valid_candidate_ids={"cand-1", "cand-2"}, ) assert result.rejected is True def test_accept_valid_decision(self): """Task 35.2: Accepts schema-compatible decisions.""" decision = _make_decision( evidence_ids=["ev-1"], candidate_ids=["cand-1"], ) result = reject_unsupported_decisions( decision, valid_candidate_ids={"cand-1"}, valid_evidence_ids={"ev-1"}, ) assert result.rejected is False assert result.reasons == [] def test_preserve_pre_and_post(self): """Task 35.3: Stores both pre-candidates and post-decisions.""" candidates = [ _make_candidate("cand-1"), _make_candidate("cand-2"), ] decisions = [_make_decision("dec-1")] record = preserve_pre_and_post( document_id="doc-1", pre_candidates=candidates, post_decisions=decisions, packet_evidence_ids=["ev-1", "ev-2"], ) assert isinstance(record, AdjudicationRecord) assert record.document_id == "doc-1" assert len(record.pre_candidates) == 2 assert len(record.post_decisions) == 1 assert record.packet_evidence_ids == ["ev-1", "ev-2"] assert record.timestamp is not None def test_route_repeated_failures_to_review(self): """Task 35.4: Routes repeated failures to 'review'.""" assert route_repeated_failures(3, 3) == "review" assert route_repeated_failures(5, 3) == "review" assert route_repeated_failures(10, 5) == "review" def test_route_never_returns_accept_repaired(self): """Task 35.4: Never returns 'accept_repaired'.""" # Even below threshold, should route to review result = route_repeated_failures(1, 3) assert result == "review" assert result != "accept_repaired" result = route_repeated_failures(0, 3) assert result == "review" assert result != "accept_repaired"