feat: Intelligence Pipeline v3 — full implementation

Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
@@ -0,0 +1,210 @@
+"""Holdout evaluation and promotion gate checking for fine-tuned models.
+
+Evaluates against frozen holdout and production artifact. A model is
+promoted only when correctness gates pass — not merely when adjudication
+rate falls.
+"""
+
+from __future__ import annotations
+
+import enum
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+from uuid import UUID, uuid4
+
+
+class PromotionDecision(str, enum.Enum):
+    """Decision on whether to promote a fine-tuned model."""
+
+    PROMOTE = "promote"
+    REJECT = "reject"
+    NEEDS_REVIEW = "needs_review"
+
+
+@dataclass
+class EvaluationResult:
+    """Results of evaluating a fine-tuned model against holdout data."""
+
+    evaluation_id: UUID
+    training_run_id: UUID
+    model_version: str
+    evaluated_at: datetime
+
+    # Correctness metrics (what matters for promotion)
+    entity_f1: float = 0.0
+    entity_precision: float = 0.0
+    entity_recall: float = 0.0
+    event_f1: float = 0.0
+    relation_f1: float = 0.0
+    fact_exact_match: float = 0.0
+
+    # Calibration metrics
+    calibration_ece: float = 0.0
+    brier_score: float = 0.0
+
+    # Comparison with production model
+    production_entity_f1: float = 0.0
+    production_event_f1: float = 0.0
+    entity_f1_delta: float = 0.0
+    event_f1_delta: float = 0.0
+
+    # Adjudication impact (reported but not a gate)
+    adjudication_rate_before: float = 0.0
+    adjudication_rate_after: float = 0.0
+    adjudication_rate_delta: float = 0.0
+
+    # Holdout details
+    holdout_size: int = 0
+    holdout_version: str = ""
+
+    @classmethod
+    def create(
+        cls,
+        training_run_id: UUID,
+        model_version: str,
+        **kwargs: Any,
+    ) -> EvaluationResult:
+        return cls(
+            evaluation_id=uuid4(),
+            training_run_id=training_run_id,
+            model_version=model_version,
+            evaluated_at=datetime.now(timezone.utc),
+            **kwargs,
+        )
+
+    def passes_correctness_gates(
+        self,
+        min_entity_f1_delta: float = 0.0,
+        min_event_f1_delta: float = -0.02,  # Allow tiny regression on events
+        max_calibration_ece: float = 0.08,
+    ) -> bool:
+        """Check if correctness gates pass.
+
+        Note: adjudication rate reduction is NOT a promotion gate.
+        A model must pass field-level correctness regardless of
+        adjudication impact.
+        """
+        # Entity F1 must not regress
+        if self.entity_f1_delta < min_entity_f1_delta:
+            return False
+
+        # Event F1 must not regress significantly
+        if self.event_f1_delta < min_event_f1_delta:
+            return False
+
+        # Calibration must remain acceptable
+        if self.calibration_ece > max_calibration_ece:
+            return False
+
+        return True
+
+    def promotion_decision(self) -> PromotionDecision:
+        """Determine promotion decision based on gates."""
+        if not self.passes_correctness_gates():
+            return PromotionDecision.REJECT
+
+        # If adjudication rate actually increases, flag for review
+        if self.adjudication_rate_delta > 0.05:
+            return PromotionDecision.NEEDS_REVIEW
+
+        return PromotionDecision.PROMOTE
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "evaluation_id": str(self.evaluation_id),
+            "model_version": self.model_version,
+            "entity_f1": self.entity_f1,
+            "event_f1": self.event_f1,
+            "relation_f1": self.relation_f1,
+            "calibration_ece": self.calibration_ece,
+            "entity_f1_delta": self.entity_f1_delta,
+            "event_f1_delta": self.event_f1_delta,
+            "adjudication_rate_delta": self.adjudication_rate_delta,
+            "passes_correctness_gates": self.passes_correctness_gates(),
+            "promotion_decision": self.promotion_decision().value,
+        }
+
+
+@dataclass
+class ModelCard:
+    """Model card for a trained specialist model artifact.
+
+    Contains training range, dataset version, intended use, limitations,
+    and evaluation results as required by Requirement 17.6.
+    """
+
+    card_id: UUID
+    model_version: str
+    base_model: str
+    training_run_id: UUID
+    created_at: datetime
+
+    # Training details
+    training_range: str = ""
+    dataset_version: str = ""
+    schema_version: str = ""
+    total_training_examples: int = 0
+
+    # Intended use
+    intended_use: str = "Entity and event extraction for financial documents"
+    entity_types: list[str] = field(default_factory=list)
+
+    # Limitations
+    limitations: list[str] = field(default_factory=lambda: [
+        "Trained on English-language financial documents only",
+        "Requires recalibration when new entity types are added",
+        "Performance may degrade on document types not in training set",
+    ])
+
+    # Evaluation
+    evaluation_results: EvaluationResult | None = None
+
+    # Registry
+    promoted: bool = False
+    promoted_at: datetime | None = None
+    deprecated: bool = False
+    deprecated_at: datetime | None = None
+
+    @classmethod
+    def create(
+        cls,
+        model_version: str,
+        base_model: str,
+        training_run_id: UUID,
+        **kwargs: Any,
+    ) -> ModelCard:
+        return cls(
+            card_id=uuid4(),
+            model_version=model_version,
+            base_model=base_model,
+            training_run_id=training_run_id,
+            created_at=datetime.now(timezone.utc),
+            **kwargs,
+        )
+
+    def promote(self) -> None:
+        """Mark this model as promoted to production."""
+        self.promoted = True
+        self.promoted_at = datetime.now(timezone.utc)
+
+    def deprecate(self) -> None:
+        """Mark this model as deprecated."""
+        self.deprecated = True
+        self.deprecated_at = datetime.now(timezone.utc)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "card_id": str(self.card_id),
+            "model_version": self.model_version,
+            "base_model": self.base_model,
+            "training_range": self.training_range,
+            "dataset_version": self.dataset_version,
+            "schema_version": self.schema_version,
+            "total_training_examples": self.total_training_examples,
+            "intended_use": self.intended_use,
+            "entity_types": self.entity_types,
+            "limitations": self.limitations,
+            "promoted": self.promoted,
+            "deprecated": self.deprecated,
+        }