"""Post-adjudication verification for Intelligence Pipeline v3. Ensures adjudication decisions are grounded in evidence, schema-compatible, and that repeated failures route to human review rather than accepting repaired defaults. """ from __future__ import annotations from datetime import datetime, timezone from enum import Enum from pydantic import BaseModel, Field from services.intelligence_pipeline_v3.adjudication.prompts import ( AdjudicationPacket, ) from services.intelligence_pipeline_v3.adjudication.schemas import ( AdjudicationCandidate, AdjudicationDecision, DecisionVerdict, QuestionCode, ) # --- Models --- class RejectionReason(str, Enum): """Reasons a decision can be rejected post-adjudication.""" MISSING_EVIDENCE_REFERENCE = "missing_evidence_reference" INVALID_CANDIDATE_REFERENCE = "invalid_candidate_reference" SCHEMA_INCOMPATIBLE = "schema_incompatible" EMPTY_EVIDENCE_IDS = "empty_evidence_ids" UNKNOWN_QUESTION_CODE = "unknown_question_code" UNKNOWN_VERDICT = "unknown_verdict" MISSING_REQUIRED_FIELD = "missing_required_field" class RejectionResult(BaseModel): """Result of rejecting an unsupported or schema-incompatible decision.""" rejected: bool = Field(description="Whether the decision was rejected") reasons: list[RejectionReason] = Field( default_factory=list, description="Reasons for rejection", ) decision_id: str = Field(default="", description="ID of the rejected decision") details: list[str] = Field( default_factory=list, description="Human-readable details about each rejection reason", ) class AdjudicationRecord(BaseModel): """Preserves both pre-adjudication candidates and post-adjudication decisions. This provides full audit trail showing what the pipeline proposed before adjudication and what the adjudicator decided. """ document_id: str = Field(description="Source document identifier") pre_candidates: list[AdjudicationCandidate] = Field( description="Candidates before adjudication", ) post_decisions: list[AdjudicationDecision] = Field( description="Decisions after adjudication", ) timestamp: datetime = Field( default_factory=lambda: datetime.now(timezone.utc), description="When the adjudication completed", ) packet_evidence_ids: list[str] = Field( default_factory=list, description="All evidence IDs that were in the adjudication packet", ) class FailureRoute(str, Enum): """Possible routes for repeated failures.""" REVIEW = "review" ACCEPT_REPAIRED = "accept_repaired" # --- Functions --- def verify_evidence_references( decision: AdjudicationDecision, packet: AdjudicationPacket, ) -> list[str]: """Check that all evidence IDs in the decision were present in the packet. Returns a list of evidence IDs that are referenced by the decision but were NOT included in the adjudication packet. An empty list means all references are valid. Args: decision: The adjudication decision to verify. packet: The adjudication packet that was sent to the model. Returns: List of evidence IDs that are missing from the packet (invalid refs). """ packet_evidence_ids = {e.evidence_id for e in packet.evidence} missing: list[str] = [] for eid in decision.evidence_ids: if eid not in packet_evidence_ids: missing.append(eid) return missing def reject_unsupported_decisions( decision: AdjudicationDecision, *, valid_candidate_ids: set[str] | None = None, valid_evidence_ids: set[str] | None = None, ) -> RejectionResult: """Reject schema-incompatible or unsupported decisions. Checks for: - Empty evidence_ids (every decision must cite evidence) - Invalid question codes - Invalid verdicts - References to non-existent candidates - References to non-existent evidence (if valid sets provided) Args: decision: The decision to validate. valid_candidate_ids: Optional set of valid candidate IDs. valid_evidence_ids: Optional set of valid evidence IDs from the packet. Returns: RejectionResult indicating whether and why the decision was rejected. """ reasons: list[RejectionReason] = [] details: list[str] = [] # Check evidence_ids is non-empty if not decision.evidence_ids: reasons.append(RejectionReason.EMPTY_EVIDENCE_IDS) details.append("Decision has no evidence_ids — every decision must cite evidence") # Check question_code validity try: QuestionCode(decision.question_code) except ValueError: reasons.append(RejectionReason.UNKNOWN_QUESTION_CODE) details.append(f"Unknown question_code: {decision.question_code}") # Check verdict validity try: DecisionVerdict(decision.verdict) except ValueError: reasons.append(RejectionReason.UNKNOWN_VERDICT) details.append(f"Unknown verdict: {decision.verdict}") # Check candidate references if valid set provided if valid_candidate_ids is not None: for cid in decision.candidate_ids: if cid not in valid_candidate_ids: reasons.append(RejectionReason.INVALID_CANDIDATE_REFERENCE) details.append(f"Candidate ID '{cid}' not in valid set") break # One invalid ref is enough to reject # Check evidence references if valid set provided if valid_evidence_ids is not None: for eid in decision.evidence_ids: if eid not in valid_evidence_ids: reasons.append(RejectionReason.MISSING_EVIDENCE_REFERENCE) details.append(f"Evidence ID '{eid}' not in valid set") break # One invalid ref is enough to reject # Check required fields if not decision.decision_id: reasons.append(RejectionReason.MISSING_REQUIRED_FIELD) details.append("decision_id is empty") if not decision.candidate_ids: reasons.append(RejectionReason.MISSING_REQUIRED_FIELD) details.append("candidate_ids is empty") return RejectionResult( rejected=len(reasons) > 0, reasons=reasons, decision_id=decision.decision_id, details=details, ) def preserve_pre_and_post( document_id: str, pre_candidates: list[AdjudicationCandidate], post_decisions: list[AdjudicationDecision], packet_evidence_ids: list[str] | None = None, ) -> AdjudicationRecord: """Store both pre-adjudication candidates and final decisions. Creates an immutable audit record preserving the full adjudication state for later review and quality assessment. Args: document_id: Source document identifier. pre_candidates: Candidates before adjudication. post_decisions: Decisions after adjudication. packet_evidence_ids: All evidence IDs from the packet. Returns: AdjudicationRecord with both pre and post states. """ return AdjudicationRecord( document_id=document_id, pre_candidates=pre_candidates, post_decisions=post_decisions, packet_evidence_ids=packet_evidence_ids or [], ) def route_repeated_failures(failure_count: int, threshold: int) -> str: """Route repeated adjudication failures to review. When the failure count meets or exceeds the threshold, routes to human review rather than accepting a repaired default. This prevents the system from silently accepting potentially incorrect outputs after repeated model failures. Args: failure_count: Number of consecutive adjudication failures. threshold: Failure count at which to escalate to review. Returns: "review" when threshold is met/exceeded, "review" always — never returns "accept_repaired" because accepting repaired defaults on repeated failures undermines evidence grounding. """ if failure_count >= threshold: return FailureRoute.REVIEW.value # Even below threshold, route to review for safety. # The adjudication system should never silently accept repaired defaults. return FailureRoute.REVIEW.value