Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
141 lines
3.8 KiB
Python
141 lines
3.8 KiB
Python
"""Replay runner — executes pipeline configurations against the Gold Corpus.
|
|
|
|
Compares every required system configuration on identical inputs and
|
|
produces structured output for report generation and gate evaluation.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import enum
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from typing import Any
|
|
from uuid import UUID, uuid4
|
|
|
|
|
|
class ReplayMode(str, enum.Enum):
|
|
"""Pipeline configurations to compare."""
|
|
|
|
CURRENT_V2 = "current_v2"
|
|
CURRENT_V2_STRICT = "current_v2_strict" # Temperature 0 + strict schema
|
|
V3_FAST_PATH = "v3_fast_path"
|
|
V3_FULL = "v3_full" # Fast path + adjudication
|
|
V3_SPECIALIST_ONLY = "v3_specialist_only"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReplayConfig:
|
|
"""Configuration for a replay run."""
|
|
|
|
config_id: UUID
|
|
mode: ReplayMode
|
|
corpus_version: str
|
|
pipeline_version: str
|
|
model_version: str | None = None
|
|
temperature: float = 0.0
|
|
strict_schema: bool = True
|
|
description: str = ""
|
|
|
|
@classmethod
|
|
def create(
|
|
cls,
|
|
mode: ReplayMode,
|
|
corpus_version: str = "1.0",
|
|
pipeline_version: str = "v3",
|
|
**kwargs: Any,
|
|
) -> ReplayConfig:
|
|
return cls(
|
|
config_id=uuid4(),
|
|
mode=mode,
|
|
corpus_version=corpus_version,
|
|
pipeline_version=pipeline_version,
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class ReplayResult:
|
|
"""Result of processing a single document in replay mode."""
|
|
|
|
document_id: str
|
|
config_id: UUID
|
|
success: bool
|
|
latency_ms: float
|
|
tokens_used: int = 0
|
|
gpu_seconds: float = 0.0
|
|
cpu_seconds: float = 0.0
|
|
extracted_entities: int = 0
|
|
extracted_facts: int = 0
|
|
evidence_spans: int = 0
|
|
schema_valid: bool = True
|
|
errors: list[str] = field(default_factory=list)
|
|
field_scores: dict[str, float] = field(default_factory=dict)
|
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
@dataclass
|
|
class ReplayRunner:
|
|
"""Executes replay runs against a corpus.
|
|
|
|
Processes documents through the specified pipeline configuration
|
|
and collects results for comparison and reporting.
|
|
"""
|
|
|
|
config: ReplayConfig
|
|
_results: list[ReplayResult] = field(default_factory=list)
|
|
started_at: datetime | None = None
|
|
completed_at: datetime | None = None
|
|
|
|
def start(self) -> None:
|
|
"""Mark the replay as started."""
|
|
self.started_at = datetime.now(timezone.utc)
|
|
|
|
def complete(self) -> None:
|
|
"""Mark the replay as completed."""
|
|
self.completed_at = datetime.now(timezone.utc)
|
|
|
|
def record_result(self, result: ReplayResult) -> None:
|
|
"""Add a document processing result."""
|
|
self._results.append(result)
|
|
|
|
@property
|
|
def results(self) -> list[ReplayResult]:
|
|
return list(self._results)
|
|
|
|
@property
|
|
def total_documents(self) -> int:
|
|
return len(self._results)
|
|
|
|
@property
|
|
def success_count(self) -> int:
|
|
return sum(1 for r in self._results if r.success)
|
|
|
|
@property
|
|
def failure_count(self) -> int:
|
|
return sum(1 for r in self._results if not r.success)
|
|
|
|
@property
|
|
def success_rate(self) -> float:
|
|
if not self._results:
|
|
return 0.0
|
|
return self.success_count / len(self._results)
|
|
|
|
@property
|
|
def avg_latency_ms(self) -> float:
|
|
if not self._results:
|
|
return 0.0
|
|
return sum(r.latency_ms for r in self._results) / len(self._results)
|
|
|
|
@property
|
|
def total_gpu_seconds(self) -> float:
|
|
return sum(r.gpu_seconds for r in self._results)
|
|
|
|
@property
|
|
def schema_validity_rate(self) -> float:
|
|
if not self._results:
|
|
return 0.0
|
|
return sum(1 for r in self._results if r.schema_valid) / len(self._results)
|
|
|
|
def is_complete(self) -> bool:
|
|
return self.completed_at is not None
|