feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,60 @@
|
||||
"""Adjudication layer for Intelligence Pipeline v3.
|
||||
|
||||
This package provides:
|
||||
- Schemas for adjudication candidates, conflicts, evidence, questions, and decisions
|
||||
- Focused adjudication prompt building with strict JSON Schema output
|
||||
- 9B adjudicator deployment configuration and VRAM gating
|
||||
- Post-adjudication verification ensuring evidence grounding
|
||||
"""
|
||||
|
||||
from services.intelligence_pipeline_v3.adjudication.deployment import (
|
||||
APPROVED_MODEL,
|
||||
APPROVED_VLLM_VERSION,
|
||||
AlertConfig,
|
||||
ConcurrencySemaphore,
|
||||
check_vram_gate,
|
||||
verify_structured_output,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.adjudication.prompts import (
|
||||
AdjudicationPacket,
|
||||
PromptMetadata,
|
||||
build_adjudication_packet,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.adjudication.schemas import (
|
||||
AdjudicationCandidate,
|
||||
AdjudicationDecision,
|
||||
AdjudicationQuestion,
|
||||
ConflictDescription,
|
||||
EvidencePacket,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.adjudication.verification import (
|
||||
AdjudicationRecord,
|
||||
RejectionResult,
|
||||
preserve_pre_and_post,
|
||||
reject_unsupported_decisions,
|
||||
route_repeated_failures,
|
||||
verify_evidence_references,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"APPROVED_MODEL",
|
||||
"APPROVED_VLLM_VERSION",
|
||||
"AdjudicationCandidate",
|
||||
"AdjudicationDecision",
|
||||
"AdjudicationPacket",
|
||||
"AdjudicationQuestion",
|
||||
"AdjudicationRecord",
|
||||
"AlertConfig",
|
||||
"ConcurrencySemaphore",
|
||||
"ConflictDescription",
|
||||
"EvidencePacket",
|
||||
"PromptMetadata",
|
||||
"RejectionResult",
|
||||
"build_adjudication_packet",
|
||||
"check_vram_gate",
|
||||
"preserve_pre_and_post",
|
||||
"reject_unsupported_decisions",
|
||||
"route_repeated_failures",
|
||||
"verify_evidence_references",
|
||||
"verify_structured_output",
|
||||
]
|
||||
@@ -0,0 +1,180 @@
|
||||
"""9B adjudicator deployment configuration for Intelligence Pipeline v3.
|
||||
|
||||
Manages the approved model/version pins, VRAM gating, concurrency
|
||||
semaphore configuration, and alerting thresholds for the 9B adjudicator
|
||||
running on RTX 4070 Ti SUPER via vLLM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# --- Pinned model and version constants ---
|
||||
|
||||
APPROVED_MODEL: str = "AxionML/Qwen3.5-9B-NVFP4"
|
||||
"""Approved 9B model for adjudication. NVFP4 quantization for 4070 Ti SUPER."""
|
||||
|
||||
APPROVED_VLLM_VERSION: str = "0.8.5"
|
||||
"""Approved vLLM version matching the cluster deployment."""
|
||||
|
||||
APPROVED_SERVED_NAME: str = "stonks-adjudicator-9b"
|
||||
"""The served model name exposed by the vLLM deployment."""
|
||||
|
||||
MAX_MODEL_LEN: int = 8192
|
||||
"""Maximum model context length configured for the deployment."""
|
||||
|
||||
MAX_NUM_SEQS: int = 8
|
||||
"""Maximum concurrent sequences for the vLLM deployment."""
|
||||
|
||||
GPU_MEMORY_UTILIZATION: float = 0.80
|
||||
"""Target GPU memory utilization fraction."""
|
||||
|
||||
VRAM_GATE_PERCENT: float = 5.0
|
||||
"""Maximum allowed VRAM increase over baseline (percentage)."""
|
||||
|
||||
|
||||
# --- Structured output verification ---
|
||||
|
||||
|
||||
def verify_structured_output(target: dict[str, Any]) -> bool:
|
||||
"""Verify that structured output works with the given deployment target.
|
||||
|
||||
Checks that the target deployment declares json_schema support in its
|
||||
capabilities and that the required configuration fields are present.
|
||||
|
||||
Args:
|
||||
target: Deployment target configuration dict containing at minimum:
|
||||
- capabilities: dict with json_schema boolean
|
||||
- served_model_name: str matching APPROVED_SERVED_NAME
|
||||
- vllm_version: str for version verification
|
||||
|
||||
Returns:
|
||||
True if strict schema output is expected to work, False otherwise.
|
||||
"""
|
||||
capabilities = target.get("capabilities", {})
|
||||
if not capabilities.get("json_schema", False):
|
||||
return False
|
||||
|
||||
# Verify model matches approved deployment
|
||||
served_name = target.get("served_model_name", "")
|
||||
if served_name and served_name != APPROVED_SERVED_NAME:
|
||||
return False
|
||||
|
||||
# Verify vLLM version compatibility
|
||||
vllm_version = target.get("vllm_version", "")
|
||||
if vllm_version and vllm_version != APPROVED_VLLM_VERSION:
|
||||
return False
|
||||
|
||||
# Verify the model is the approved one
|
||||
model = target.get("model", "")
|
||||
if model and model != APPROVED_MODEL:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
# --- VRAM gate ---
|
||||
|
||||
|
||||
def check_vram_gate(peak_mb: float, baseline_mb: float) -> bool:
|
||||
"""Check whether peak VRAM usage is within the +5% gate of baseline.
|
||||
|
||||
The gate ensures that no deployment update exceeds the measured current
|
||||
9B deployment VRAM by more than 5 percent.
|
||||
|
||||
Args:
|
||||
peak_mb: Measured peak VRAM in megabytes during test.
|
||||
baseline_mb: Baseline VRAM measurement in megabytes.
|
||||
|
||||
Returns:
|
||||
True if peak is within acceptable range, False if it exceeds the gate.
|
||||
"""
|
||||
if baseline_mb <= 0:
|
||||
return False
|
||||
if peak_mb <= 0:
|
||||
return False
|
||||
|
||||
max_allowed_mb = baseline_mb * (1.0 + VRAM_GATE_PERCENT / 100.0)
|
||||
return peak_mb <= max_allowed_mb
|
||||
|
||||
|
||||
# --- Concurrency semaphore ---
|
||||
|
||||
|
||||
class ConcurrencySemaphore(BaseModel):
|
||||
"""Configuration for the adjudication concurrency semaphore.
|
||||
|
||||
Limits concurrent adjudication requests to protect vLLM from
|
||||
overload. Aligned with max-num-seqs and KV-cache behavior.
|
||||
"""
|
||||
|
||||
max_concurrent: int = Field(
|
||||
default=MAX_NUM_SEQS,
|
||||
gt=0,
|
||||
description="Maximum concurrent adjudication requests",
|
||||
)
|
||||
queue_timeout_seconds: float = Field(
|
||||
default=120.0,
|
||||
gt=0,
|
||||
description="Maximum time to wait for semaphore acquisition",
|
||||
)
|
||||
backpressure_threshold: int = Field(
|
||||
default=MAX_NUM_SEQS * 4,
|
||||
ge=0,
|
||||
description="Queue depth at which backpressure signals are emitted",
|
||||
)
|
||||
|
||||
def create_semaphore(self) -> asyncio.Semaphore:
|
||||
"""Create an asyncio.Semaphore with the configured max_concurrent."""
|
||||
return asyncio.Semaphore(self.max_concurrent)
|
||||
|
||||
|
||||
# --- Alert configuration ---
|
||||
|
||||
|
||||
class AlertConfig(BaseModel):
|
||||
"""Alert thresholds for adjudicator monitoring.
|
||||
|
||||
Defines queue-depth and availability thresholds that trigger alerts
|
||||
when the adjudicator is overloaded or unavailable.
|
||||
"""
|
||||
|
||||
queue_depth_warning: int = Field(
|
||||
default=16,
|
||||
ge=1,
|
||||
description="Queue depth that triggers a warning alert",
|
||||
)
|
||||
queue_depth_critical: int = Field(
|
||||
default=32,
|
||||
ge=1,
|
||||
description="Queue depth that triggers a critical alert",
|
||||
)
|
||||
availability_threshold_percent: float = Field(
|
||||
default=95.0,
|
||||
gt=0.0,
|
||||
le=100.0,
|
||||
description="Minimum availability percentage before alerting",
|
||||
)
|
||||
latency_p95_warning_ms: int = Field(
|
||||
default=5000,
|
||||
gt=0,
|
||||
description="p95 latency (ms) that triggers a warning",
|
||||
)
|
||||
latency_p95_critical_ms: int = Field(
|
||||
default=15000,
|
||||
gt=0,
|
||||
description="p95 latency (ms) that triggers a critical alert",
|
||||
)
|
||||
consecutive_failures_alert: int = Field(
|
||||
default=3,
|
||||
ge=1,
|
||||
description="Number of consecutive failures before alerting",
|
||||
)
|
||||
health_check_interval_seconds: float = Field(
|
||||
default=30.0,
|
||||
gt=0,
|
||||
description="Interval between health checks in seconds",
|
||||
)
|
||||
@@ -0,0 +1,302 @@
|
||||
"""Focused adjudication prompt building for Intelligence Pipeline v3.
|
||||
|
||||
Builds adjudication packets containing only relevant chunks and candidates,
|
||||
uses strict JSON Schema with temperature zero, and enforces a bounded output
|
||||
budget (max 1536 tokens for decisions, not summaries).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from services.intelligence_pipeline_v3.adjudication.schemas import (
|
||||
AdjudicationCandidate,
|
||||
AdjudicationQuestion,
|
||||
ConflictDescription,
|
||||
EvidencePacket,
|
||||
QuestionCode,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.segmenter.models import DocumentChunk
|
||||
|
||||
# --- Constants ---
|
||||
|
||||
MAX_OUTPUT_TOKENS: int = 1536
|
||||
"""Maximum tokens for adjudication decisions output. Bounded to prevent
|
||||
long summaries — the adjudicator produces decisions, not narratives."""
|
||||
|
||||
TEMPERATURE: float = 0.0
|
||||
"""Temperature for adjudication requests. Zero for deterministic output."""
|
||||
|
||||
PROMPT_SCHEMA_VERSION: str = "1.0.0"
|
||||
"""Version of the adjudication prompt schema format."""
|
||||
|
||||
PROVIDER_LINEAGE_KEY: str = "adjudication_v3"
|
||||
"""Lineage identifier for adjudication prompts."""
|
||||
|
||||
|
||||
# --- Models ---
|
||||
|
||||
|
||||
class PromptMetadata(BaseModel):
|
||||
"""Metadata for the adjudication prompt including version and lineage.
|
||||
|
||||
Tracks prompt version, schema version, and provider lineage for
|
||||
reproducibility and auditing.
|
||||
"""
|
||||
|
||||
prompt_version: str = Field(
|
||||
default="1.0.0",
|
||||
description="Version of the prompt template",
|
||||
)
|
||||
schema_version: str = Field(
|
||||
default=PROMPT_SCHEMA_VERSION,
|
||||
description="Version of the JSON Schema format used",
|
||||
)
|
||||
provider_lineage: str = Field(
|
||||
default=PROVIDER_LINEAGE_KEY,
|
||||
description="Identifier for the prompt provider/pipeline stage",
|
||||
)
|
||||
max_output_tokens: int = Field(
|
||||
default=MAX_OUTPUT_TOKENS,
|
||||
description="Maximum output token budget for this prompt",
|
||||
)
|
||||
temperature: float = Field(
|
||||
default=TEMPERATURE,
|
||||
description="Generation temperature",
|
||||
)
|
||||
|
||||
|
||||
class AdjudicationPacket(BaseModel):
|
||||
"""Complete packet sent to the 9B adjudicator.
|
||||
|
||||
Contains only the information relevant to resolving the specific
|
||||
ambiguity — relevant chunks, candidates, conflicts, and questions.
|
||||
"""
|
||||
|
||||
document_id: str = Field(description="Source document identifier")
|
||||
document_type: str = Field(description="Type of document")
|
||||
relevant_chunks: list[DocumentChunk] = Field(
|
||||
description="Only chunks relevant to the adjudication questions",
|
||||
)
|
||||
candidates: list[AdjudicationCandidate] = Field(
|
||||
description="Candidates requiring adjudication",
|
||||
)
|
||||
conflicts: list[ConflictDescription] = Field(
|
||||
default_factory=list,
|
||||
description="Conflicts between candidates",
|
||||
)
|
||||
questions: list[AdjudicationQuestion] = Field(
|
||||
description="Specific questions the adjudicator must answer",
|
||||
)
|
||||
evidence: list[EvidencePacket] = Field(
|
||||
description="Evidence spans available for reference",
|
||||
)
|
||||
metadata: PromptMetadata = Field(
|
||||
default_factory=PromptMetadata,
|
||||
description="Prompt metadata for versioning and lineage",
|
||||
)
|
||||
|
||||
|
||||
# --- Output schema for strict JSON mode ---
|
||||
|
||||
|
||||
def get_decision_json_schema() -> dict[str, Any]:
|
||||
"""Return the strict JSON Schema for adjudication decisions.
|
||||
|
||||
Used as the `response_format.json_schema.schema` payload when
|
||||
calling the 9B model with strict structured output.
|
||||
"""
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"decisions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"decision_id": {"type": "string"},
|
||||
"question_code": {
|
||||
"type": "string",
|
||||
"enum": [code.value for code in QuestionCode],
|
||||
},
|
||||
"verdict": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"accept",
|
||||
"reject",
|
||||
"merge",
|
||||
"split",
|
||||
"reattribute",
|
||||
],
|
||||
},
|
||||
"candidate_ids": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"minItems": 1,
|
||||
},
|
||||
"evidence_ids": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"minItems": 1,
|
||||
},
|
||||
"reasoning": {"type": "string"},
|
||||
"resolved_value": {"type": "object"},
|
||||
},
|
||||
"required": [
|
||||
"decision_id",
|
||||
"question_code",
|
||||
"verdict",
|
||||
"candidate_ids",
|
||||
"evidence_ids",
|
||||
"reasoning",
|
||||
],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["decisions"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
|
||||
|
||||
# --- Packet builder ---
|
||||
|
||||
|
||||
def _get_relevant_chunk_ids(
|
||||
candidates: list[AdjudicationCandidate],
|
||||
conflicts: list[ConflictDescription],
|
||||
questions: list[AdjudicationQuestion],
|
||||
) -> set[str]:
|
||||
"""Collect chunk IDs referenced by candidates, conflicts, and questions."""
|
||||
chunk_ids: set[str] = set()
|
||||
for candidate in candidates:
|
||||
chunk_ids.update(candidate.source_chunk_ids)
|
||||
return chunk_ids
|
||||
|
||||
|
||||
def _filter_relevant_chunks(
|
||||
document_chunks: list[DocumentChunk],
|
||||
relevant_chunk_ids: set[str],
|
||||
) -> list[DocumentChunk]:
|
||||
"""Filter document chunks to include only those referenced by candidates."""
|
||||
if not relevant_chunk_ids:
|
||||
# If no specific chunks referenced, include all (fallback for
|
||||
# cases where chunk IDs weren't specified in candidates)
|
||||
return document_chunks
|
||||
return [c for c in document_chunks if c.chunk_id in relevant_chunk_ids]
|
||||
|
||||
|
||||
def _collect_evidence_ids(
|
||||
candidates: list[AdjudicationCandidate],
|
||||
conflicts: list[ConflictDescription],
|
||||
) -> set[str]:
|
||||
"""Collect all evidence IDs referenced by candidates and conflicts."""
|
||||
evidence_ids: set[str] = set()
|
||||
for candidate in candidates:
|
||||
evidence_ids.update(candidate.evidence_ids)
|
||||
for conflict in conflicts:
|
||||
evidence_ids.update(conflict.evidence_ids)
|
||||
return evidence_ids
|
||||
|
||||
|
||||
def build_adjudication_packet(
|
||||
document_id: str,
|
||||
document_type: str,
|
||||
document_chunks: list[DocumentChunk],
|
||||
candidates: list[AdjudicationCandidate],
|
||||
conflicts: list[ConflictDescription],
|
||||
questions: list[AdjudicationQuestion],
|
||||
evidence: list[EvidencePacket],
|
||||
*,
|
||||
question_codes: list[str] | None = None,
|
||||
) -> AdjudicationPacket:
|
||||
"""Build an adjudication packet with only relevant chunks and evidence.
|
||||
|
||||
Filters document_chunks to include only those referenced by the
|
||||
candidates being adjudicated. Ensures the packet is focused and
|
||||
within the bounded context the adjudicator expects.
|
||||
|
||||
Args:
|
||||
document_id: Source document identifier.
|
||||
document_type: Type of document (article, filing, transcript, etc.).
|
||||
document_chunks: All available chunks for the document.
|
||||
candidates: Candidates requiring adjudication.
|
||||
conflicts: Conflicts between candidates.
|
||||
questions: Specific questions to resolve.
|
||||
evidence: Available evidence spans.
|
||||
question_codes: Optional filter to limit questions by code.
|
||||
|
||||
Returns:
|
||||
AdjudicationPacket with only relevant chunks included.
|
||||
"""
|
||||
# Filter questions by code if specified
|
||||
filtered_questions = questions
|
||||
if question_codes:
|
||||
code_set = set(question_codes)
|
||||
filtered_questions = [
|
||||
q for q in questions if q.question_code.value in code_set
|
||||
]
|
||||
|
||||
# Determine which chunks are relevant
|
||||
relevant_chunk_ids = _get_relevant_chunk_ids(
|
||||
candidates, conflicts, filtered_questions
|
||||
)
|
||||
relevant_chunks = _filter_relevant_chunks(document_chunks, relevant_chunk_ids)
|
||||
|
||||
# Filter evidence to only include those referenced by candidates/conflicts
|
||||
referenced_evidence_ids = _collect_evidence_ids(candidates, conflicts)
|
||||
if referenced_evidence_ids:
|
||||
relevant_evidence = [
|
||||
e for e in evidence if e.evidence_id in referenced_evidence_ids
|
||||
]
|
||||
else:
|
||||
# Include all evidence if none specifically referenced
|
||||
relevant_evidence = evidence
|
||||
|
||||
return AdjudicationPacket(
|
||||
document_id=document_id,
|
||||
document_type=document_type,
|
||||
relevant_chunks=relevant_chunks,
|
||||
candidates=candidates,
|
||||
conflicts=conflicts,
|
||||
questions=filtered_questions,
|
||||
evidence=relevant_evidence,
|
||||
metadata=PromptMetadata(),
|
||||
)
|
||||
|
||||
|
||||
def build_request_payload(packet: AdjudicationPacket) -> dict[str, Any]:
|
||||
"""Build the full inference request payload for the adjudicator.
|
||||
|
||||
Returns a dict suitable for passing to the inference gateway,
|
||||
including strict JSON Schema response format and temperature zero.
|
||||
"""
|
||||
system_prompt = (
|
||||
"You are a semantic adjudicator for financial document extraction. "
|
||||
"Resolve the ambiguities described in the questions using ONLY the "
|
||||
"provided evidence spans. Every decision MUST reference evidence_ids "
|
||||
"from the provided evidence. Do NOT estimate confidence, novelty, "
|
||||
"impact magnitude, or time horizon — those are computed by separate "
|
||||
"calibrated pipelines. Output valid JSON matching the required schema."
|
||||
)
|
||||
|
||||
user_content = packet.model_dump_json()
|
||||
|
||||
return {
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_content},
|
||||
],
|
||||
"temperature": packet.metadata.temperature,
|
||||
"max_tokens": packet.metadata.max_output_tokens,
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "adjudication_response",
|
||||
"strict": True,
|
||||
"schema": get_decision_json_schema(),
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Adjudication schemas for Intelligence Pipeline v3.
|
||||
|
||||
Defines Pydantic models for the adjudication layer:
|
||||
- AdjudicationCandidate: a proposed entity/fact/event requiring adjudication
|
||||
- ConflictDescription: describes a conflict between candidates
|
||||
- AdjudicationQuestion: a specific question the adjudicator must resolve
|
||||
- EvidencePacket: evidence spans provided to the adjudicator
|
||||
- AdjudicationDecision: the adjudicator's resolution (excludes confidence,
|
||||
novelty, impact, and horizon — those come from calibrated pipelines)
|
||||
|
||||
Every decision requires evidence_ids linking back to packet evidence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class CandidateType(str, Enum):
|
||||
"""Type of candidate being adjudicated."""
|
||||
|
||||
ENTITY = "entity"
|
||||
EVENT = "event"
|
||||
FACT = "fact"
|
||||
RELATION = "relation"
|
||||
SENTIMENT = "sentiment"
|
||||
|
||||
|
||||
class AdjudicationCandidate(BaseModel):
|
||||
"""A proposed extraction candidate that requires adjudication.
|
||||
|
||||
Represents an entity, event, fact, relation, or sentiment that the
|
||||
fast-path could not resolve with sufficient confidence.
|
||||
"""
|
||||
|
||||
candidate_id: str = Field(description="Unique identifier for this candidate")
|
||||
candidate_type: CandidateType = Field(description="Type of candidate")
|
||||
label: str = Field(description="Human-readable label or description")
|
||||
source_chunk_ids: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Chunk IDs where this candidate was found",
|
||||
)
|
||||
evidence_ids: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Evidence span IDs supporting this candidate",
|
||||
)
|
||||
metadata: dict[str, Any] = Field(
|
||||
default_factory=dict,
|
||||
description="Additional type-specific metadata",
|
||||
)
|
||||
score: float = Field(
|
||||
default=0.0,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Specialist extraction score (0.0-1.0)",
|
||||
)
|
||||
|
||||
|
||||
class ConflictType(str, Enum):
|
||||
"""Type of conflict between candidates."""
|
||||
|
||||
CONTRADICTORY_VALUES = "contradictory_values"
|
||||
AMBIGUOUS_IDENTITY = "ambiguous_identity"
|
||||
OPPOSING_SENTIMENT = "opposing_sentiment"
|
||||
OVERLAPPING_EVENTS = "overlapping_events"
|
||||
CAUSAL_AMBIGUITY = "causal_ambiguity"
|
||||
|
||||
|
||||
class ConflictDescription(BaseModel):
|
||||
"""Describes a conflict between two or more candidates.
|
||||
|
||||
Used to inform the adjudicator about what needs resolution.
|
||||
"""
|
||||
|
||||
conflict_id: str = Field(description="Unique identifier for this conflict")
|
||||
conflict_type: ConflictType = Field(description="Type of conflict")
|
||||
candidate_ids: list[str] = Field(
|
||||
min_length=2,
|
||||
description="IDs of conflicting candidates",
|
||||
)
|
||||
description: str = Field(description="Human-readable conflict description")
|
||||
evidence_ids: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Evidence IDs relevant to this conflict",
|
||||
)
|
||||
|
||||
|
||||
class QuestionCode(str, Enum):
|
||||
"""Codes representing specific adjudication questions."""
|
||||
|
||||
RESOLVE_ENTITY_IDENTITY = "RESOLVE_ENTITY_IDENTITY"
|
||||
RESOLVE_EVENT_TYPE = "RESOLVE_EVENT_TYPE"
|
||||
RESOLVE_CAUSAL_DIRECTION = "RESOLVE_CAUSAL_DIRECTION"
|
||||
RESOLVE_NUMERIC_CONFLICT = "RESOLVE_NUMERIC_CONFLICT"
|
||||
RESOLVE_SENTIMENT_DIRECTION = "RESOLVE_SENTIMENT_DIRECTION"
|
||||
RESOLVE_TEMPORAL_ORDERING = "RESOLVE_TEMPORAL_ORDERING"
|
||||
RESOLVE_COMPANY_ATTRIBUTION = "RESOLVE_COMPANY_ATTRIBUTION"
|
||||
CONFIRM_CROSS_CHUNK_RELATION = "CONFIRM_CROSS_CHUNK_RELATION"
|
||||
|
||||
|
||||
class AdjudicationQuestion(BaseModel):
|
||||
"""A specific question the adjudicator must answer.
|
||||
|
||||
Each question references candidates and conflicts that need resolution.
|
||||
"""
|
||||
|
||||
question_code: QuestionCode = Field(description="Structured question code")
|
||||
description: str = Field(description="Natural language question for the adjudicator")
|
||||
candidate_ids: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Candidate IDs this question applies to",
|
||||
)
|
||||
conflict_ids: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Conflict IDs this question resolves",
|
||||
)
|
||||
|
||||
|
||||
class EvidencePacket(BaseModel):
|
||||
"""Evidence spans provided to the adjudicator.
|
||||
|
||||
Contains the exact text and location of evidence the adjudicator
|
||||
can reference in its decisions.
|
||||
"""
|
||||
|
||||
evidence_id: str = Field(description="Unique identifier for this evidence span")
|
||||
chunk_id: str = Field(description="Source chunk identifier")
|
||||
start_char: int = Field(ge=0, description="Start character offset within chunk")
|
||||
end_char: int = Field(gt=0, description="End character offset within chunk")
|
||||
text: str = Field(min_length=1, description="Evidence text content")
|
||||
source_document_id: str = Field(description="Parent document identifier")
|
||||
|
||||
|
||||
class DecisionVerdict(str, Enum):
|
||||
"""Possible verdicts for an adjudication decision."""
|
||||
|
||||
ACCEPT = "accept"
|
||||
REJECT = "reject"
|
||||
MERGE = "merge"
|
||||
SPLIT = "split"
|
||||
REATTRIBUTE = "reattribute"
|
||||
|
||||
|
||||
class AdjudicationDecision(BaseModel):
|
||||
"""The adjudicator's resolution for one or more candidates.
|
||||
|
||||
IMPORTANT: This model intentionally EXCLUDES:
|
||||
- authoritative confidence (comes from calibration pipeline)
|
||||
- novelty (comes from retrieval-based novelty stage)
|
||||
- impact (comes from stock-specific impact model)
|
||||
- horizon (comes from impact model)
|
||||
|
||||
The adjudicator resolves candidate identity, relationships, event
|
||||
interpretation, and supported qualitative direction only. Every
|
||||
decision MUST reference evidence_ids from the provided packet.
|
||||
"""
|
||||
|
||||
decision_id: str = Field(description="Unique identifier for this decision")
|
||||
question_code: QuestionCode = Field(description="Which question this resolves")
|
||||
verdict: DecisionVerdict = Field(description="The adjudication verdict")
|
||||
candidate_ids: list[str] = Field(
|
||||
min_length=1,
|
||||
description="Candidate IDs this decision applies to",
|
||||
)
|
||||
evidence_ids: list[str] = Field(
|
||||
min_length=1,
|
||||
description="Evidence IDs supporting this decision (required, non-empty)",
|
||||
)
|
||||
reasoning: str = Field(
|
||||
description="Brief reasoning for the decision",
|
||||
)
|
||||
resolved_value: dict[str, Any] = Field(
|
||||
default_factory=dict,
|
||||
description="The resolved value(s) if applicable",
|
||||
)
|
||||
@@ -0,0 +1,241 @@
|
||||
"""Post-adjudication verification for Intelligence Pipeline v3.
|
||||
|
||||
Ensures adjudication decisions are grounded in evidence, schema-compatible,
|
||||
and that repeated failures route to human review rather than accepting
|
||||
repaired defaults.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from services.intelligence_pipeline_v3.adjudication.prompts import (
|
||||
AdjudicationPacket,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.adjudication.schemas import (
|
||||
AdjudicationCandidate,
|
||||
AdjudicationDecision,
|
||||
DecisionVerdict,
|
||||
QuestionCode,
|
||||
)
|
||||
|
||||
# --- Models ---
|
||||
|
||||
|
||||
class RejectionReason(str, Enum):
|
||||
"""Reasons a decision can be rejected post-adjudication."""
|
||||
|
||||
MISSING_EVIDENCE_REFERENCE = "missing_evidence_reference"
|
||||
INVALID_CANDIDATE_REFERENCE = "invalid_candidate_reference"
|
||||
SCHEMA_INCOMPATIBLE = "schema_incompatible"
|
||||
EMPTY_EVIDENCE_IDS = "empty_evidence_ids"
|
||||
UNKNOWN_QUESTION_CODE = "unknown_question_code"
|
||||
UNKNOWN_VERDICT = "unknown_verdict"
|
||||
MISSING_REQUIRED_FIELD = "missing_required_field"
|
||||
|
||||
|
||||
class RejectionResult(BaseModel):
|
||||
"""Result of rejecting an unsupported or schema-incompatible decision."""
|
||||
|
||||
rejected: bool = Field(description="Whether the decision was rejected")
|
||||
reasons: list[RejectionReason] = Field(
|
||||
default_factory=list,
|
||||
description="Reasons for rejection",
|
||||
)
|
||||
decision_id: str = Field(default="", description="ID of the rejected decision")
|
||||
details: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Human-readable details about each rejection reason",
|
||||
)
|
||||
|
||||
|
||||
class AdjudicationRecord(BaseModel):
|
||||
"""Preserves both pre-adjudication candidates and post-adjudication decisions.
|
||||
|
||||
This provides full audit trail showing what the pipeline proposed
|
||||
before adjudication and what the adjudicator decided.
|
||||
"""
|
||||
|
||||
document_id: str = Field(description="Source document identifier")
|
||||
pre_candidates: list[AdjudicationCandidate] = Field(
|
||||
description="Candidates before adjudication",
|
||||
)
|
||||
post_decisions: list[AdjudicationDecision] = Field(
|
||||
description="Decisions after adjudication",
|
||||
)
|
||||
timestamp: datetime = Field(
|
||||
default_factory=lambda: datetime.now(timezone.utc),
|
||||
description="When the adjudication completed",
|
||||
)
|
||||
packet_evidence_ids: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="All evidence IDs that were in the adjudication packet",
|
||||
)
|
||||
|
||||
|
||||
class FailureRoute(str, Enum):
|
||||
"""Possible routes for repeated failures."""
|
||||
|
||||
REVIEW = "review"
|
||||
ACCEPT_REPAIRED = "accept_repaired"
|
||||
|
||||
|
||||
# --- Functions ---
|
||||
|
||||
|
||||
def verify_evidence_references(
|
||||
decision: AdjudicationDecision,
|
||||
packet: AdjudicationPacket,
|
||||
) -> list[str]:
|
||||
"""Check that all evidence IDs in the decision were present in the packet.
|
||||
|
||||
Returns a list of evidence IDs that are referenced by the decision
|
||||
but were NOT included in the adjudication packet. An empty list
|
||||
means all references are valid.
|
||||
|
||||
Args:
|
||||
decision: The adjudication decision to verify.
|
||||
packet: The adjudication packet that was sent to the model.
|
||||
|
||||
Returns:
|
||||
List of evidence IDs that are missing from the packet (invalid refs).
|
||||
"""
|
||||
packet_evidence_ids = {e.evidence_id for e in packet.evidence}
|
||||
missing: list[str] = []
|
||||
for eid in decision.evidence_ids:
|
||||
if eid not in packet_evidence_ids:
|
||||
missing.append(eid)
|
||||
return missing
|
||||
|
||||
|
||||
def reject_unsupported_decisions(
|
||||
decision: AdjudicationDecision,
|
||||
*,
|
||||
valid_candidate_ids: set[str] | None = None,
|
||||
valid_evidence_ids: set[str] | None = None,
|
||||
) -> RejectionResult:
|
||||
"""Reject schema-incompatible or unsupported decisions.
|
||||
|
||||
Checks for:
|
||||
- Empty evidence_ids (every decision must cite evidence)
|
||||
- Invalid question codes
|
||||
- Invalid verdicts
|
||||
- References to non-existent candidates
|
||||
- References to non-existent evidence (if valid sets provided)
|
||||
|
||||
Args:
|
||||
decision: The decision to validate.
|
||||
valid_candidate_ids: Optional set of valid candidate IDs.
|
||||
valid_evidence_ids: Optional set of valid evidence IDs from the packet.
|
||||
|
||||
Returns:
|
||||
RejectionResult indicating whether and why the decision was rejected.
|
||||
"""
|
||||
reasons: list[RejectionReason] = []
|
||||
details: list[str] = []
|
||||
|
||||
# Check evidence_ids is non-empty
|
||||
if not decision.evidence_ids:
|
||||
reasons.append(RejectionReason.EMPTY_EVIDENCE_IDS)
|
||||
details.append("Decision has no evidence_ids — every decision must cite evidence")
|
||||
|
||||
# Check question_code validity
|
||||
try:
|
||||
QuestionCode(decision.question_code)
|
||||
except ValueError:
|
||||
reasons.append(RejectionReason.UNKNOWN_QUESTION_CODE)
|
||||
details.append(f"Unknown question_code: {decision.question_code}")
|
||||
|
||||
# Check verdict validity
|
||||
try:
|
||||
DecisionVerdict(decision.verdict)
|
||||
except ValueError:
|
||||
reasons.append(RejectionReason.UNKNOWN_VERDICT)
|
||||
details.append(f"Unknown verdict: {decision.verdict}")
|
||||
|
||||
# Check candidate references if valid set provided
|
||||
if valid_candidate_ids is not None:
|
||||
for cid in decision.candidate_ids:
|
||||
if cid not in valid_candidate_ids:
|
||||
reasons.append(RejectionReason.INVALID_CANDIDATE_REFERENCE)
|
||||
details.append(f"Candidate ID '{cid}' not in valid set")
|
||||
break # One invalid ref is enough to reject
|
||||
|
||||
# Check evidence references if valid set provided
|
||||
if valid_evidence_ids is not None:
|
||||
for eid in decision.evidence_ids:
|
||||
if eid not in valid_evidence_ids:
|
||||
reasons.append(RejectionReason.MISSING_EVIDENCE_REFERENCE)
|
||||
details.append(f"Evidence ID '{eid}' not in valid set")
|
||||
break # One invalid ref is enough to reject
|
||||
|
||||
# Check required fields
|
||||
if not decision.decision_id:
|
||||
reasons.append(RejectionReason.MISSING_REQUIRED_FIELD)
|
||||
details.append("decision_id is empty")
|
||||
|
||||
if not decision.candidate_ids:
|
||||
reasons.append(RejectionReason.MISSING_REQUIRED_FIELD)
|
||||
details.append("candidate_ids is empty")
|
||||
|
||||
return RejectionResult(
|
||||
rejected=len(reasons) > 0,
|
||||
reasons=reasons,
|
||||
decision_id=decision.decision_id,
|
||||
details=details,
|
||||
)
|
||||
|
||||
|
||||
def preserve_pre_and_post(
|
||||
document_id: str,
|
||||
pre_candidates: list[AdjudicationCandidate],
|
||||
post_decisions: list[AdjudicationDecision],
|
||||
packet_evidence_ids: list[str] | None = None,
|
||||
) -> AdjudicationRecord:
|
||||
"""Store both pre-adjudication candidates and final decisions.
|
||||
|
||||
Creates an immutable audit record preserving the full adjudication
|
||||
state for later review and quality assessment.
|
||||
|
||||
Args:
|
||||
document_id: Source document identifier.
|
||||
pre_candidates: Candidates before adjudication.
|
||||
post_decisions: Decisions after adjudication.
|
||||
packet_evidence_ids: All evidence IDs from the packet.
|
||||
|
||||
Returns:
|
||||
AdjudicationRecord with both pre and post states.
|
||||
"""
|
||||
return AdjudicationRecord(
|
||||
document_id=document_id,
|
||||
pre_candidates=pre_candidates,
|
||||
post_decisions=post_decisions,
|
||||
packet_evidence_ids=packet_evidence_ids or [],
|
||||
)
|
||||
|
||||
|
||||
def route_repeated_failures(failure_count: int, threshold: int) -> str:
|
||||
"""Route repeated adjudication failures to review.
|
||||
|
||||
When the failure count meets or exceeds the threshold, routes to
|
||||
human review rather than accepting a repaired default. This prevents
|
||||
the system from silently accepting potentially incorrect outputs
|
||||
after repeated model failures.
|
||||
|
||||
Args:
|
||||
failure_count: Number of consecutive adjudication failures.
|
||||
threshold: Failure count at which to escalate to review.
|
||||
|
||||
Returns:
|
||||
"review" when threshold is met/exceeded, "review" always —
|
||||
never returns "accept_repaired" because accepting repaired
|
||||
defaults on repeated failures undermines evidence grounding.
|
||||
"""
|
||||
if failure_count >= threshold:
|
||||
return FailureRoute.REVIEW.value
|
||||
# Even below threshold, route to review for safety.
|
||||
# The adjudication system should never silently accept repaired defaults.
|
||||
return FailureRoute.REVIEW.value
|
||||
Reference in New Issue
Block a user