feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,32 @@
|
||||
"""Deterministic routing engine for Intelligence Pipeline v3.
|
||||
|
||||
Routes documents to fast-path or adjudication based on hard ambiguity/conflict
|
||||
rules and calibrated confidence thresholds. Every routing decision is stored
|
||||
with the full feature snapshot for audit and calibration feedback.
|
||||
"""
|
||||
|
||||
from services.intelligence_pipeline_v3.routing.reasons import (
|
||||
RouteDecision,
|
||||
RoutingReason,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.routing.router import (
|
||||
RoutingDecision,
|
||||
RoutingEngine,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.routing.rules import evaluate_hard_rules
|
||||
from services.intelligence_pipeline_v3.routing.store import RoutingDecisionStore
|
||||
from services.intelligence_pipeline_v3.routing.thresholds import (
|
||||
FastPathThresholds,
|
||||
evaluate_thresholds,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"FastPathThresholds",
|
||||
"RouteDecision",
|
||||
"RoutingDecision",
|
||||
"RoutingEngine",
|
||||
"RoutingReason",
|
||||
"RoutingDecisionStore",
|
||||
"evaluate_hard_rules",
|
||||
"evaluate_thresholds",
|
||||
]
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Routing reason enums for the deterministic routing engine.
|
||||
|
||||
RoutingReason captures *why* a document was routed to adjudication or accepted
|
||||
on the fast path. RouteDecision is the binary outcome.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class RoutingReason(str, Enum):
|
||||
"""Structured reason codes explaining a routing decision.
|
||||
|
||||
Any triggered reason (except FAST_PATH_ACCEPTED) forces adjudication.
|
||||
These codes are stored in the database as TEXT[] and must remain stable
|
||||
across versions for audit queries.
|
||||
"""
|
||||
|
||||
UNRESOLVED_ALIAS = "UNRESOLVED_ALIAS"
|
||||
MULTIPLE_PRIMARY_COMPANIES = "MULTIPLE_PRIMARY_COMPANIES"
|
||||
CONTRADICTORY_NUMERIC_FACTS = "CONTRADICTORY_NUMERIC_FACTS"
|
||||
CONFLICTING_SENTIMENT = "CONFLICTING_SENTIMENT"
|
||||
IMPLIED_CAUSAL_IMPACT = "IMPLIED_CAUSAL_IMPACT"
|
||||
GUIDANCE_VS_CONSENSUS_REQUIRES_REASONING = (
|
||||
"GUIDANCE_VS_CONSENSUS_REQUIRES_REASONING"
|
||||
)
|
||||
MATERIAL_FIELD_MISSING = "MATERIAL_FIELD_MISSING"
|
||||
EVIDENCE_COVERAGE_BELOW_THRESHOLD = "EVIDENCE_COVERAGE_BELOW_THRESHOLD"
|
||||
CALIBRATED_CONFIDENCE_BELOW_THRESHOLD = "CALIBRATED_CONFIDENCE_BELOW_THRESHOLD"
|
||||
LONG_DOCUMENT_CROSS_CHUNK_RELATION = "LONG_DOCUMENT_CROSS_CHUNK_RELATION"
|
||||
FAST_PATH_ACCEPTED = "FAST_PATH_ACCEPTED"
|
||||
|
||||
|
||||
class RouteDecision(str, Enum):
|
||||
"""Binary routing outcome."""
|
||||
|
||||
FAST_PATH = "fast_path"
|
||||
ADJUDICATION = "adjudication"
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Deterministic routing engine for Intelligence Pipeline v3.
|
||||
|
||||
The RoutingEngine combines hard ambiguity/conflict rules with calibrated
|
||||
confidence thresholds to produce a deterministic route decision. The same
|
||||
inputs always produce the same output — no randomness or side effects.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from services.intelligence_pipeline_v3.routing.reasons import (
|
||||
RouteDecision,
|
||||
RoutingReason,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.routing.rules import evaluate_hard_rules
|
||||
from services.intelligence_pipeline_v3.routing.thresholds import (
|
||||
FastPathThresholds,
|
||||
evaluate_thresholds,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RoutingDecision:
|
||||
"""Immutable record of a routing decision with full context.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
id:
|
||||
Unique identifier for this decision.
|
||||
pipeline_run_id:
|
||||
The pipeline run this decision belongs to.
|
||||
document_id:
|
||||
The document being routed.
|
||||
route:
|
||||
The binary routing outcome (fast_path or adjudication).
|
||||
reasons:
|
||||
List of routing reasons explaining the decision.
|
||||
confidence_snapshot:
|
||||
Full feature snapshot at decision time for audit and recalibration.
|
||||
decided_at:
|
||||
UTC timestamp of the decision.
|
||||
"""
|
||||
|
||||
id: UUID
|
||||
pipeline_run_id: UUID
|
||||
document_id: UUID
|
||||
route: RouteDecision
|
||||
reasons: list[RoutingReason]
|
||||
confidence_snapshot: dict[str, Any]
|
||||
decided_at: datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class RoutingEngine:
|
||||
"""Deterministic routing engine.
|
||||
|
||||
Evaluates hard rules first, then applies confidence thresholds.
|
||||
Same inputs always produce the same route — no randomness, no external
|
||||
state dependency beyond the provided arguments.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
thresholds:
|
||||
Fast-path threshold configuration. Defaults to conservative values.
|
||||
"""
|
||||
|
||||
thresholds: FastPathThresholds = field(default_factory=FastPathThresholds)
|
||||
|
||||
def route(
|
||||
self,
|
||||
pipeline_run_id: UUID,
|
||||
document_id: UUID,
|
||||
confidence_features: dict[str, Any],
|
||||
ambiguity_markers: dict[str, Any],
|
||||
document_type: str,
|
||||
event_type: str | None = None,
|
||||
) -> RoutingDecision:
|
||||
"""Produce a deterministic routing decision.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
pipeline_run_id:
|
||||
The pipeline run identifier.
|
||||
document_id:
|
||||
The document being routed.
|
||||
confidence_features:
|
||||
Field-level confidence features from the confidence pipeline.
|
||||
Must include ``calibrated_confidence`` (float 0-1).
|
||||
ambiguity_markers:
|
||||
Structural ambiguity markers from candidate generation.
|
||||
document_type:
|
||||
The document type (article, filing, transcript, etc.).
|
||||
event_type:
|
||||
Optional event type detected in the document.
|
||||
|
||||
Returns
|
||||
-------
|
||||
RoutingDecision
|
||||
Immutable decision record with route, reasons, and feature snapshot.
|
||||
"""
|
||||
# Step 1: Evaluate hard rules (any trigger = adjudication)
|
||||
hard_reasons = evaluate_hard_rules(confidence_features, ambiguity_markers)
|
||||
|
||||
if hard_reasons:
|
||||
return self._build_decision(
|
||||
pipeline_run_id=pipeline_run_id,
|
||||
document_id=document_id,
|
||||
route=RouteDecision.ADJUDICATION,
|
||||
reasons=hard_reasons,
|
||||
confidence_features=confidence_features,
|
||||
ambiguity_markers=ambiguity_markers,
|
||||
)
|
||||
|
||||
# Step 2: Evaluate confidence thresholds
|
||||
calibrated_confidence = confidence_features.get("calibrated_confidence", 0.0)
|
||||
|
||||
# Check evidence coverage threshold (hard threshold, not configurable per doc type)
|
||||
evidence_coverage = confidence_features.get("evidence_coverage", 1.0)
|
||||
if evidence_coverage < 0.5:
|
||||
return self._build_decision(
|
||||
pipeline_run_id=pipeline_run_id,
|
||||
document_id=document_id,
|
||||
route=RouteDecision.ADJUDICATION,
|
||||
reasons=[RoutingReason.EVIDENCE_COVERAGE_BELOW_THRESHOLD],
|
||||
confidence_features=confidence_features,
|
||||
ambiguity_markers=ambiguity_markers,
|
||||
)
|
||||
|
||||
# Apply calibrated confidence threshold
|
||||
threshold_decision = evaluate_thresholds(
|
||||
confidence=calibrated_confidence,
|
||||
document_type=document_type,
|
||||
event_type=event_type,
|
||||
thresholds=self.thresholds,
|
||||
)
|
||||
|
||||
if threshold_decision == RouteDecision.ADJUDICATION:
|
||||
return self._build_decision(
|
||||
pipeline_run_id=pipeline_run_id,
|
||||
document_id=document_id,
|
||||
route=RouteDecision.ADJUDICATION,
|
||||
reasons=[RoutingReason.CALIBRATED_CONFIDENCE_BELOW_THRESHOLD],
|
||||
confidence_features=confidence_features,
|
||||
ambiguity_markers=ambiguity_markers,
|
||||
)
|
||||
|
||||
# All checks passed — fast path accepted
|
||||
return self._build_decision(
|
||||
pipeline_run_id=pipeline_run_id,
|
||||
document_id=document_id,
|
||||
route=RouteDecision.FAST_PATH,
|
||||
reasons=[RoutingReason.FAST_PATH_ACCEPTED],
|
||||
confidence_features=confidence_features,
|
||||
ambiguity_markers=ambiguity_markers,
|
||||
)
|
||||
|
||||
def _build_decision(
|
||||
self,
|
||||
pipeline_run_id: UUID,
|
||||
document_id: UUID,
|
||||
route: RouteDecision,
|
||||
reasons: list[RoutingReason],
|
||||
confidence_features: dict[str, Any],
|
||||
ambiguity_markers: dict[str, Any],
|
||||
) -> RoutingDecision:
|
||||
"""Build an immutable routing decision with full snapshot."""
|
||||
return RoutingDecision(
|
||||
id=uuid4(),
|
||||
pipeline_run_id=pipeline_run_id,
|
||||
document_id=document_id,
|
||||
route=route,
|
||||
reasons=reasons,
|
||||
confidence_snapshot={
|
||||
"confidence_features": confidence_features,
|
||||
"ambiguity_markers": ambiguity_markers,
|
||||
"thresholds_version": self.thresholds.version,
|
||||
},
|
||||
decided_at=datetime.now(timezone.utc),
|
||||
)
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Hard ambiguity and conflict rules for routing decisions.
|
||||
|
||||
These rules check structural markers in extraction output that indicate
|
||||
the document *requires* semantic reasoning by the 9B adjudicator. Any
|
||||
triggered rule forces ADJUDICATION regardless of confidence scores.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from services.intelligence_pipeline_v3.routing.reasons import RoutingReason
|
||||
|
||||
|
||||
def evaluate_hard_rules(
|
||||
confidence_features: dict[str, Any],
|
||||
ambiguity_markers: dict[str, Any],
|
||||
) -> list[RoutingReason]:
|
||||
"""Evaluate hard ambiguity/conflict rules against extraction output.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
confidence_features:
|
||||
Field-level confidence features from the confidence pipeline.
|
||||
Expected keys include:
|
||||
- ``evidence_coverage``: float 0-1
|
||||
- ``material_fields_present``: bool
|
||||
- ``cross_chunk_relations``: bool (relations span multiple chunks)
|
||||
|
||||
ambiguity_markers:
|
||||
Structural ambiguity markers from candidate generation and resolution.
|
||||
Expected keys include:
|
||||
- ``unresolved_aliases``: int (count of unresolved entity aliases)
|
||||
- ``primary_company_count``: int (number of primary companies detected)
|
||||
- ``contradictory_numeric_facts``: bool
|
||||
- ``conflicting_sentiment``: bool
|
||||
- ``implied_causal_impact``: bool
|
||||
- ``guidance_vs_consensus``: bool
|
||||
- ``long_document_cross_chunk``: bool
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[RoutingReason]
|
||||
List of triggered reasons. Empty list means no hard rules triggered.
|
||||
"""
|
||||
triggered: list[RoutingReason] = []
|
||||
|
||||
# Unresolved entity aliases require contextual disambiguation
|
||||
if ambiguity_markers.get("unresolved_aliases", 0) > 0:
|
||||
triggered.append(RoutingReason.UNRESOLVED_ALIAS)
|
||||
|
||||
# Multiple primary companies need reasoning about which is the subject
|
||||
if ambiguity_markers.get("primary_company_count", 0) > 1:
|
||||
triggered.append(RoutingReason.MULTIPLE_PRIMARY_COMPANIES)
|
||||
|
||||
# Contradictory numeric facts (e.g., conflicting revenue figures)
|
||||
if ambiguity_markers.get("contradictory_numeric_facts", False):
|
||||
triggered.append(RoutingReason.CONTRADICTORY_NUMERIC_FACTS)
|
||||
|
||||
# Conflicting sentiment across evidence groups for the same company
|
||||
if ambiguity_markers.get("conflicting_sentiment", False):
|
||||
triggered.append(RoutingReason.CONFLICTING_SENTIMENT)
|
||||
|
||||
# Implied causal impact requiring reasoning (not explicit statement)
|
||||
if ambiguity_markers.get("implied_causal_impact", False):
|
||||
triggered.append(RoutingReason.IMPLIED_CAUSAL_IMPACT)
|
||||
|
||||
# Guidance vs consensus comparison requires model reasoning
|
||||
if ambiguity_markers.get("guidance_vs_consensus", False):
|
||||
triggered.append(RoutingReason.GUIDANCE_VS_CONSENSUS_REQUIRES_REASONING)
|
||||
|
||||
# Material fields missing from extraction output
|
||||
if not confidence_features.get("material_fields_present", True):
|
||||
triggered.append(RoutingReason.MATERIAL_FIELD_MISSING)
|
||||
|
||||
# Cross-chunk relations in long documents need broader context
|
||||
if ambiguity_markers.get("long_document_cross_chunk", False):
|
||||
triggered.append(RoutingReason.LONG_DOCUMENT_CROSS_CHUNK_RELATION)
|
||||
|
||||
return triggered
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Routing decision storage.
|
||||
|
||||
Stores every routing decision with the full feature snapshot for audit,
|
||||
recalibration, and explainability. Backed by the v3_routing_decisions table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from uuid import UUID
|
||||
|
||||
from services.intelligence_pipeline_v3.routing.router import RoutingDecision
|
||||
|
||||
|
||||
@dataclass
|
||||
class RoutingDecisionStore:
|
||||
"""In-memory store for routing decisions.
|
||||
|
||||
In production, this would be backed by the ``v3_routing_decisions`` table.
|
||||
This implementation provides the storage interface for use in the pipeline
|
||||
orchestrator and for testing.
|
||||
|
||||
The store is append-only — decisions are immutable once stored.
|
||||
"""
|
||||
|
||||
_decisions: list[RoutingDecision] = field(default_factory=list)
|
||||
_by_pipeline_run: dict[UUID, list[RoutingDecision]] = field(default_factory=dict)
|
||||
|
||||
def store(self, decision: RoutingDecision) -> None:
|
||||
"""Store a routing decision.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
decision:
|
||||
The routing decision to persist. Must have a unique id.
|
||||
"""
|
||||
self._decisions.append(decision)
|
||||
run_decisions = self._by_pipeline_run.setdefault(
|
||||
decision.pipeline_run_id, []
|
||||
)
|
||||
run_decisions.append(decision)
|
||||
|
||||
def get_by_pipeline_run(self, run_id: UUID) -> list[RoutingDecision]:
|
||||
"""Retrieve all routing decisions for a pipeline run.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
run_id:
|
||||
The pipeline run identifier.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[RoutingDecision]
|
||||
All decisions for the given run, in insertion order.
|
||||
Returns empty list if no decisions exist for the run.
|
||||
"""
|
||||
return list(self._by_pipeline_run.get(run_id, []))
|
||||
|
||||
def get_all(self) -> list[RoutingDecision]:
|
||||
"""Retrieve all stored decisions in insertion order."""
|
||||
return list(self._decisions)
|
||||
|
||||
def count(self) -> int:
|
||||
"""Return the total number of stored decisions."""
|
||||
return len(self._decisions)
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Calibrated fast-path thresholds by document type and event type.
|
||||
|
||||
Thresholds represent the minimum calibrated confidence required for
|
||||
fast-path acceptance. Documents/events below these thresholds are routed
|
||||
to adjudication. Thresholds are versioned and can be updated as
|
||||
calibration data improves.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from services.intelligence_pipeline_v3.routing.reasons import RouteDecision
|
||||
|
||||
# Default confidence thresholds per document type.
|
||||
# These are initial conservative values; calibration on the Gold Corpus
|
||||
# will refine them over time.
|
||||
DEFAULT_DOCUMENT_THRESHOLDS: dict[str, float] = {
|
||||
"article": 0.80,
|
||||
"press_release": 0.80,
|
||||
"filing": 0.70,
|
||||
"transcript": 0.75,
|
||||
"macro_event": 0.75,
|
||||
}
|
||||
|
||||
# Default confidence thresholds per event type (override document-type defaults).
|
||||
DEFAULT_EVENT_THRESHOLDS: dict[str, float] = {
|
||||
"earnings_beat": 0.75,
|
||||
"earnings_miss": 0.75,
|
||||
"guidance_change": 0.65,
|
||||
"management_change": 0.70,
|
||||
"merger_acquisition": 0.60,
|
||||
"regulatory_action": 0.65,
|
||||
"product_launch": 0.80,
|
||||
"legal_action": 0.65,
|
||||
"rating_change": 0.75,
|
||||
"supply_chain": 0.70,
|
||||
}
|
||||
|
||||
# Fallback threshold when document_type or event_type is unknown.
|
||||
DEFAULT_FALLBACK_THRESHOLD: float = 0.80
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FastPathThresholds:
|
||||
"""Configuration for fast-path acceptance thresholds.
|
||||
|
||||
Resolution order:
|
||||
1. Event-type-specific threshold (if event_type is provided and known).
|
||||
2. Document-type-specific threshold.
|
||||
3. Fallback threshold.
|
||||
|
||||
Higher thresholds are more conservative (more documents go to adjudication).
|
||||
"""
|
||||
|
||||
document_thresholds: dict[str, float] = field(
|
||||
default_factory=lambda: dict(DEFAULT_DOCUMENT_THRESHOLDS)
|
||||
)
|
||||
event_thresholds: dict[str, float] = field(
|
||||
default_factory=lambda: dict(DEFAULT_EVENT_THRESHOLDS)
|
||||
)
|
||||
fallback_threshold: float = DEFAULT_FALLBACK_THRESHOLD
|
||||
version: str = "1.0.0"
|
||||
|
||||
def resolve_threshold(
|
||||
self,
|
||||
document_type: str,
|
||||
event_type: str | None = None,
|
||||
) -> float:
|
||||
"""Resolve the applicable threshold for a document/event combination.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
document_type:
|
||||
The document type (article, filing, transcript, etc.).
|
||||
event_type:
|
||||
Optional event type detected in the document.
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
The minimum calibrated confidence required for fast-path acceptance.
|
||||
"""
|
||||
# Event-type threshold takes priority when available
|
||||
if event_type and event_type in self.event_thresholds:
|
||||
return self.event_thresholds[event_type]
|
||||
|
||||
# Document-type threshold
|
||||
if document_type in self.document_thresholds:
|
||||
return self.document_thresholds[document_type]
|
||||
|
||||
# Fallback
|
||||
return self.fallback_threshold
|
||||
|
||||
|
||||
def evaluate_thresholds(
|
||||
confidence: float,
|
||||
document_type: str,
|
||||
event_type: str | None,
|
||||
thresholds: FastPathThresholds,
|
||||
) -> RouteDecision:
|
||||
"""Evaluate whether calibrated confidence meets the fast-path threshold.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
confidence:
|
||||
Calibrated confidence score (0.0 to 1.0).
|
||||
document_type:
|
||||
The document type being processed.
|
||||
event_type:
|
||||
Optional event type detected in the document.
|
||||
thresholds:
|
||||
Threshold configuration to use.
|
||||
|
||||
Returns
|
||||
-------
|
||||
RouteDecision
|
||||
FAST_PATH if confidence >= threshold, ADJUDICATION otherwise.
|
||||
|
||||
Notes
|
||||
-----
|
||||
The comparison uses ``>=`` (greater-than-or-equal). A confidence value
|
||||
exactly at the threshold is accepted on the fast path. This boundary
|
||||
behavior is deterministic and tested by property tests.
|
||||
"""
|
||||
threshold = thresholds.resolve_threshold(document_type, event_type)
|
||||
|
||||
if confidence >= threshold:
|
||||
return RouteDecision.FAST_PATH
|
||||
return RouteDecision.ADJUDICATION
|
||||
Reference in New Issue
Block a user