feat: Intelligence Pipeline v3 — full implementation

Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
@@ -0,0 +1,140 @@
+"""Replay runner — executes pipeline configurations against the Gold Corpus.
+
+Compares every required system configuration on identical inputs and
+produces structured output for report generation and gate evaluation.
+"""
+
+from __future__ import annotations
+
+import enum
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+from uuid import UUID, uuid4
+
+
+class ReplayMode(str, enum.Enum):
+    """Pipeline configurations to compare."""
+
+    CURRENT_V2 = "current_v2"
+    CURRENT_V2_STRICT = "current_v2_strict"  # Temperature 0 + strict schema
+    V3_FAST_PATH = "v3_fast_path"
+    V3_FULL = "v3_full"  # Fast path + adjudication
+    V3_SPECIALIST_ONLY = "v3_specialist_only"
+
+
+@dataclass(frozen=True)
+class ReplayConfig:
+    """Configuration for a replay run."""
+
+    config_id: UUID
+    mode: ReplayMode
+    corpus_version: str
+    pipeline_version: str
+    model_version: str | None = None
+    temperature: float = 0.0
+    strict_schema: bool = True
+    description: str = ""
+
+    @classmethod
+    def create(
+        cls,
+        mode: ReplayMode,
+        corpus_version: str = "1.0",
+        pipeline_version: str = "v3",
+        **kwargs: Any,
+    ) -> ReplayConfig:
+        return cls(
+            config_id=uuid4(),
+            mode=mode,
+            corpus_version=corpus_version,
+            pipeline_version=pipeline_version,
+            **kwargs,
+        )
+
+
+@dataclass
+class ReplayResult:
+    """Result of processing a single document in replay mode."""
+
+    document_id: str
+    config_id: UUID
+    success: bool
+    latency_ms: float
+    tokens_used: int = 0
+    gpu_seconds: float = 0.0
+    cpu_seconds: float = 0.0
+    extracted_entities: int = 0
+    extracted_facts: int = 0
+    evidence_spans: int = 0
+    schema_valid: bool = True
+    errors: list[str] = field(default_factory=list)
+    field_scores: dict[str, float] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class ReplayRunner:
+    """Executes replay runs against a corpus.
+
+    Processes documents through the specified pipeline configuration
+    and collects results for comparison and reporting.
+    """
+
+    config: ReplayConfig
+    _results: list[ReplayResult] = field(default_factory=list)
+    started_at: datetime | None = None
+    completed_at: datetime | None = None
+
+    def start(self) -> None:
+        """Mark the replay as started."""
+        self.started_at = datetime.now(timezone.utc)
+
+    def complete(self) -> None:
+        """Mark the replay as completed."""
+        self.completed_at = datetime.now(timezone.utc)
+
+    def record_result(self, result: ReplayResult) -> None:
+        """Add a document processing result."""
+        self._results.append(result)
+
+    @property
+    def results(self) -> list[ReplayResult]:
+        return list(self._results)
+
+    @property
+    def total_documents(self) -> int:
+        return len(self._results)
+
+    @property
+    def success_count(self) -> int:
+        return sum(1 for r in self._results if r.success)
+
+    @property
+    def failure_count(self) -> int:
+        return sum(1 for r in self._results if not r.success)
+
+    @property
+    def success_rate(self) -> float:
+        if not self._results:
+            return 0.0
+        return self.success_count / len(self._results)
+
+    @property
+    def avg_latency_ms(self) -> float:
+        if not self._results:
+            return 0.0
+        return sum(r.latency_ms for r in self._results) / len(self._results)
+
+    @property
+    def total_gpu_seconds(self) -> float:
+        return sum(r.gpu_seconds for r in self._results)
+
+    @property
+    def schema_validity_rate(self) -> float:
+        if not self._results:
+            return 0.0
+        return sum(1 for r in self._results if r.schema_valid) / len(self._results)
+
+    def is_complete(self) -> bool:
+        return self.completed_at is not None