stonks-oracle/services/intelligence_pipeline_v3/adjudication/verification.py

"""Post-adjudication verification for Intelligence Pipeline v3.

Ensures adjudication decisions are grounded in evidence, schema-compatible,
and that repeated failures route to human review rather than accepting
repaired defaults.
"""

from __future__ import annotations

from datetime import datetime, timezone
from enum import Enum

from pydantic import BaseModel, Field

from services.intelligence_pipeline_v3.adjudication.prompts import (
    AdjudicationPacket,
)
from services.intelligence_pipeline_v3.adjudication.schemas import (
    AdjudicationCandidate,
    AdjudicationDecision,
    DecisionVerdict,
    QuestionCode,
)

# --- Models ---


class RejectionReason(str, Enum):
    """Reasons a decision can be rejected post-adjudication."""

    MISSING_EVIDENCE_REFERENCE = "missing_evidence_reference"
    INVALID_CANDIDATE_REFERENCE = "invalid_candidate_reference"
    SCHEMA_INCOMPATIBLE = "schema_incompatible"
    EMPTY_EVIDENCE_IDS = "empty_evidence_ids"
    UNKNOWN_QUESTION_CODE = "unknown_question_code"
    UNKNOWN_VERDICT = "unknown_verdict"
    MISSING_REQUIRED_FIELD = "missing_required_field"


class RejectionResult(BaseModel):
    """Result of rejecting an unsupported or schema-incompatible decision."""

    rejected: bool = Field(description="Whether the decision was rejected")
    reasons: list[RejectionReason] = Field(
        default_factory=list,
        description="Reasons for rejection",
    )
    decision_id: str = Field(default="", description="ID of the rejected decision")
    details: list[str] = Field(
        default_factory=list,
        description="Human-readable details about each rejection reason",
    )


class AdjudicationRecord(BaseModel):
    """Preserves both pre-adjudication candidates and post-adjudication decisions.

    This provides full audit trail showing what the pipeline proposed
    before adjudication and what the adjudicator decided.
    """

    document_id: str = Field(description="Source document identifier")
    pre_candidates: list[AdjudicationCandidate] = Field(
        description="Candidates before adjudication",
    )
    post_decisions: list[AdjudicationDecision] = Field(
        description="Decisions after adjudication",
    )
    timestamp: datetime = Field(
        default_factory=lambda: datetime.now(timezone.utc),
        description="When the adjudication completed",
    )
    packet_evidence_ids: list[str] = Field(
        default_factory=list,
        description="All evidence IDs that were in the adjudication packet",
    )


class FailureRoute(str, Enum):
    """Possible routes for repeated failures."""

    REVIEW = "review"
    ACCEPT_REPAIRED = "accept_repaired"


# --- Functions ---


def verify_evidence_references(
    decision: AdjudicationDecision,
    packet: AdjudicationPacket,
) -> list[str]:
    """Check that all evidence IDs in the decision were present in the packet.

    Returns a list of evidence IDs that are referenced by the decision
    but were NOT included in the adjudication packet. An empty list
    means all references are valid.

    Args:
        decision: The adjudication decision to verify.
        packet: The adjudication packet that was sent to the model.

    Returns:
        List of evidence IDs that are missing from the packet (invalid refs).
    """
    packet_evidence_ids = {e.evidence_id for e in packet.evidence}
    missing: list[str] = []
    for eid in decision.evidence_ids:
        if eid not in packet_evidence_ids:
            missing.append(eid)
    return missing


def reject_unsupported_decisions(
    decision: AdjudicationDecision,
    *,
    valid_candidate_ids: set[str] | None = None,
    valid_evidence_ids: set[str] | None = None,
) -> RejectionResult:
    """Reject schema-incompatible or unsupported decisions.

    Checks for:
    - Empty evidence_ids (every decision must cite evidence)
    - Invalid question codes
    - Invalid verdicts
    - References to non-existent candidates
    - References to non-existent evidence (if valid sets provided)

    Args:
        decision: The decision to validate.
        valid_candidate_ids: Optional set of valid candidate IDs.
        valid_evidence_ids: Optional set of valid evidence IDs from the packet.

    Returns:
        RejectionResult indicating whether and why the decision was rejected.
    """
    reasons: list[RejectionReason] = []
    details: list[str] = []

    # Check evidence_ids is non-empty
    if not decision.evidence_ids:
        reasons.append(RejectionReason.EMPTY_EVIDENCE_IDS)
        details.append("Decision has no evidence_ids — every decision must cite evidence")

    # Check question_code validity
    try:
        QuestionCode(decision.question_code)
    except ValueError:
        reasons.append(RejectionReason.UNKNOWN_QUESTION_CODE)
        details.append(f"Unknown question_code: {decision.question_code}")

    # Check verdict validity
    try:
        DecisionVerdict(decision.verdict)
    except ValueError:
        reasons.append(RejectionReason.UNKNOWN_VERDICT)
        details.append(f"Unknown verdict: {decision.verdict}")

    # Check candidate references if valid set provided
    if valid_candidate_ids is not None:
        for cid in decision.candidate_ids:
            if cid not in valid_candidate_ids:
                reasons.append(RejectionReason.INVALID_CANDIDATE_REFERENCE)
                details.append(f"Candidate ID '{cid}' not in valid set")
                break  # One invalid ref is enough to reject

    # Check evidence references if valid set provided
    if valid_evidence_ids is not None:
        for eid in decision.evidence_ids:
            if eid not in valid_evidence_ids:
                reasons.append(RejectionReason.MISSING_EVIDENCE_REFERENCE)
                details.append(f"Evidence ID '{eid}' not in valid set")
                break  # One invalid ref is enough to reject

    # Check required fields
    if not decision.decision_id:
        reasons.append(RejectionReason.MISSING_REQUIRED_FIELD)
        details.append("decision_id is empty")

    if not decision.candidate_ids:
        reasons.append(RejectionReason.MISSING_REQUIRED_FIELD)
        details.append("candidate_ids is empty")

    return RejectionResult(
        rejected=len(reasons) > 0,
        reasons=reasons,
        decision_id=decision.decision_id,
        details=details,
    )


def preserve_pre_and_post(
    document_id: str,
    pre_candidates: list[AdjudicationCandidate],
    post_decisions: list[AdjudicationDecision],
    packet_evidence_ids: list[str] | None = None,
) -> AdjudicationRecord:
    """Store both pre-adjudication candidates and final decisions.

    Creates an immutable audit record preserving the full adjudication
    state for later review and quality assessment.

    Args:
        document_id: Source document identifier.
        pre_candidates: Candidates before adjudication.
        post_decisions: Decisions after adjudication.
        packet_evidence_ids: All evidence IDs from the packet.

    Returns:
        AdjudicationRecord with both pre and post states.
    """
    return AdjudicationRecord(
        document_id=document_id,
        pre_candidates=pre_candidates,
        post_decisions=post_decisions,
        packet_evidence_ids=packet_evidence_ids or [],
    )


def route_repeated_failures(failure_count: int, threshold: int) -> str:
    """Route repeated adjudication failures to review.

    When the failure count meets or exceeds the threshold, routes to
    human review rather than accepting a repaired default. This prevents
    the system from silently accepting potentially incorrect outputs
    after repeated model failures.

    Args:
        failure_count: Number of consecutive adjudication failures.
        threshold: Failure count at which to escalate to review.

    Returns:
        "review" when threshold is met/exceeded, "review" always —
        never returns "accept_repaired" because accepting repaired
        defaults on repeated failures undermines evidence grounding.
    """
    if failure_count >= threshold:
        return FailureRoute.REVIEW.value
    # Even below threshold, route to review for safety.
    # The adjudication system should never silently accept repaired defaults.
    return FailureRoute.REVIEW.value