feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,291 @@
|
||||
"""Stage metrics, dashboards, and alert definitions for the v3 pipeline.
|
||||
|
||||
Tracks latency, errors, batch size, queue depth, route metrics, field
|
||||
accuracy, evidence coverage, calibration, fast-path rate, adjudication
|
||||
reasons, GPU memory, GPU utilization, and GPU-seconds per document.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
|
||||
class AlertSeverity(str, enum.Enum):
|
||||
"""Alert severity levels."""
|
||||
|
||||
INFO = "info"
|
||||
WARNING = "warning"
|
||||
CRITICAL = "critical"
|
||||
|
||||
|
||||
class MetricType(str, enum.Enum):
|
||||
"""Types of collected metrics."""
|
||||
|
||||
COUNTER = "counter"
|
||||
GAUGE = "gauge"
|
||||
HISTOGRAM = "histogram"
|
||||
SUMMARY = "summary"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MetricAlert:
|
||||
"""Alert definition for pipeline metrics."""
|
||||
|
||||
name: str
|
||||
metric_name: str
|
||||
condition: str # e.g., "> 0.05", "< 0.6"
|
||||
severity: AlertSeverity
|
||||
description: str
|
||||
threshold: float
|
||||
window_seconds: int = 300
|
||||
|
||||
def evaluate(self, current_value: float) -> bool:
|
||||
"""Check if the alert condition is triggered.
|
||||
|
||||
Returns True if the alert should fire.
|
||||
"""
|
||||
if self.condition.startswith(">"):
|
||||
return current_value > self.threshold
|
||||
elif self.condition.startswith("<"):
|
||||
return current_value < self.threshold
|
||||
elif self.condition.startswith(">="):
|
||||
return current_value >= self.threshold
|
||||
elif self.condition.startswith("<="):
|
||||
return current_value <= self.threshold
|
||||
return False
|
||||
|
||||
|
||||
@dataclass
|
||||
class StageMetrics:
|
||||
"""Metrics for a single pipeline stage."""
|
||||
|
||||
stage_name: str
|
||||
total_invocations: int = 0
|
||||
total_errors: int = 0
|
||||
total_latency_ms: float = 0.0
|
||||
max_latency_ms: float = 0.0
|
||||
min_latency_ms: float = float("inf")
|
||||
total_tokens_in: int = 0
|
||||
total_tokens_out: int = 0
|
||||
total_batch_items: int = 0
|
||||
total_batches: int = 0
|
||||
gpu_seconds: float = 0.0
|
||||
gpu_memory_peak_mb: float = 0.0
|
||||
gpu_utilization_avg: float = 0.0
|
||||
|
||||
def record_invocation(
|
||||
self,
|
||||
latency_ms: float,
|
||||
tokens_in: int = 0,
|
||||
tokens_out: int = 0,
|
||||
error: bool = False,
|
||||
batch_size: int = 1,
|
||||
gpu_seconds: float = 0.0,
|
||||
gpu_memory_mb: float = 0.0,
|
||||
gpu_utilization: float = 0.0,
|
||||
) -> None:
|
||||
"""Record a single stage invocation."""
|
||||
self.total_invocations += 1
|
||||
self.total_latency_ms += latency_ms
|
||||
self.max_latency_ms = max(self.max_latency_ms, latency_ms)
|
||||
self.min_latency_ms = min(self.min_latency_ms, latency_ms)
|
||||
self.total_tokens_in += tokens_in
|
||||
self.total_tokens_out += tokens_out
|
||||
self.total_batch_items += batch_size
|
||||
self.total_batches += 1
|
||||
self.gpu_seconds += gpu_seconds
|
||||
self.gpu_memory_peak_mb = max(self.gpu_memory_peak_mb, gpu_memory_mb)
|
||||
|
||||
if error:
|
||||
self.total_errors += 1
|
||||
|
||||
# Running average for GPU utilization
|
||||
if gpu_utilization > 0:
|
||||
n = self.total_invocations
|
||||
self.gpu_utilization_avg = (
|
||||
self.gpu_utilization_avg * (n - 1) + gpu_utilization
|
||||
) / n
|
||||
|
||||
@property
|
||||
def avg_latency_ms(self) -> float:
|
||||
if self.total_invocations == 0:
|
||||
return 0.0
|
||||
return self.total_latency_ms / self.total_invocations
|
||||
|
||||
@property
|
||||
def error_rate(self) -> float:
|
||||
if self.total_invocations == 0:
|
||||
return 0.0
|
||||
return self.total_errors / self.total_invocations
|
||||
|
||||
@property
|
||||
def avg_batch_size(self) -> float:
|
||||
if self.total_batches == 0:
|
||||
return 0.0
|
||||
return self.total_batch_items / self.total_batches
|
||||
|
||||
@property
|
||||
def avg_tokens_per_doc(self) -> float:
|
||||
if self.total_invocations == 0:
|
||||
return 0.0
|
||||
return (self.total_tokens_in + self.total_tokens_out) / self.total_invocations
|
||||
|
||||
@property
|
||||
def gpu_seconds_per_doc(self) -> float:
|
||||
if self.total_invocations == 0:
|
||||
return 0.0
|
||||
return self.gpu_seconds / self.total_invocations
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"stage_name": self.stage_name,
|
||||
"total_invocations": self.total_invocations,
|
||||
"total_errors": self.total_errors,
|
||||
"error_rate": self.error_rate,
|
||||
"avg_latency_ms": self.avg_latency_ms,
|
||||
"max_latency_ms": self.max_latency_ms,
|
||||
"avg_batch_size": self.avg_batch_size,
|
||||
"gpu_seconds_per_doc": self.gpu_seconds_per_doc,
|
||||
"gpu_memory_peak_mb": self.gpu_memory_peak_mb,
|
||||
}
|
||||
|
||||
|
||||
# Default alert definitions for the v3 pipeline
|
||||
DEFAULT_ALERTS: list[MetricAlert] = [
|
||||
MetricAlert(
|
||||
name="schema_failure_rate_high",
|
||||
metric_name="schema_failures",
|
||||
condition="> 0.05",
|
||||
severity=AlertSeverity.CRITICAL,
|
||||
description="Schema validation failure rate exceeds 5%",
|
||||
threshold=0.05,
|
||||
),
|
||||
MetricAlert(
|
||||
name="unsupported_claims_high",
|
||||
metric_name="unsupported_claim_rate",
|
||||
condition="> 0.10",
|
||||
severity=AlertSeverity.WARNING,
|
||||
description="Unsupported claim rate exceeds 10%",
|
||||
threshold=0.10,
|
||||
),
|
||||
MetricAlert(
|
||||
name="calibration_drift",
|
||||
metric_name="calibration_ece",
|
||||
condition="> 0.08",
|
||||
severity=AlertSeverity.WARNING,
|
||||
description="Calibration ECE exceeds 8%",
|
||||
threshold=0.08,
|
||||
),
|
||||
MetricAlert(
|
||||
name="queue_saturation",
|
||||
metric_name="queue_saturation_ratio",
|
||||
condition="> 0.90",
|
||||
severity=AlertSeverity.CRITICAL,
|
||||
description="Queue saturation exceeds 90%",
|
||||
threshold=0.90,
|
||||
),
|
||||
MetricAlert(
|
||||
name="provider_probe_failure",
|
||||
metric_name="probe_failure_rate",
|
||||
condition="> 0.0",
|
||||
severity=AlertSeverity.CRITICAL,
|
||||
description="Provider capability probe failed",
|
||||
threshold=0.0,
|
||||
),
|
||||
MetricAlert(
|
||||
name="gpu_memory_high",
|
||||
metric_name="gpu_memory_utilization",
|
||||
condition="> 0.85",
|
||||
severity=AlertSeverity.WARNING,
|
||||
description="GPU memory utilization exceeds 85%",
|
||||
threshold=0.85,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetricsCollector:
|
||||
"""Collects and aggregates metrics across pipeline stages.
|
||||
|
||||
In production, this would export to Prometheus/Grafana.
|
||||
This implementation provides the collection logic for testing.
|
||||
"""
|
||||
|
||||
_stages: dict[str, StageMetrics] = field(default_factory=dict)
|
||||
_alerts: list[MetricAlert] = field(default_factory=list)
|
||||
_counters: dict[str, float] = field(default_factory=dict)
|
||||
_fired_alerts: list[tuple[MetricAlert, float, datetime]] = field(
|
||||
default_factory=list
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self._alerts:
|
||||
self._alerts = list(DEFAULT_ALERTS)
|
||||
|
||||
def get_stage(self, stage_name: str) -> StageMetrics:
|
||||
"""Get or create metrics for a stage."""
|
||||
if stage_name not in self._stages:
|
||||
self._stages[stage_name] = StageMetrics(stage_name=stage_name)
|
||||
return self._stages[stage_name]
|
||||
|
||||
def record_stage(
|
||||
self,
|
||||
stage_name: str,
|
||||
latency_ms: float,
|
||||
tokens_in: int = 0,
|
||||
tokens_out: int = 0,
|
||||
error: bool = False,
|
||||
batch_size: int = 1,
|
||||
gpu_seconds: float = 0.0,
|
||||
gpu_memory_mb: float = 0.0,
|
||||
gpu_utilization: float = 0.0,
|
||||
) -> None:
|
||||
"""Record a stage invocation."""
|
||||
stage = self.get_stage(stage_name)
|
||||
stage.record_invocation(
|
||||
latency_ms=latency_ms,
|
||||
tokens_in=tokens_in,
|
||||
tokens_out=tokens_out,
|
||||
error=error,
|
||||
batch_size=batch_size,
|
||||
gpu_seconds=gpu_seconds,
|
||||
gpu_memory_mb=gpu_memory_mb,
|
||||
gpu_utilization=gpu_utilization,
|
||||
)
|
||||
|
||||
def increment_counter(self, name: str, value: float = 1.0) -> None:
|
||||
"""Increment a named counter."""
|
||||
self._counters[name] = self._counters.get(name, 0.0) + value
|
||||
|
||||
def get_counter(self, name: str) -> float:
|
||||
"""Get current counter value."""
|
||||
return self._counters.get(name, 0.0)
|
||||
|
||||
def check_alerts(self) -> list[tuple[MetricAlert, float]]:
|
||||
"""Evaluate all alert conditions. Returns (alert, value) for fired alerts."""
|
||||
fired: list[tuple[MetricAlert, float]] = []
|
||||
for alert in self._alerts:
|
||||
value = self._counters.get(alert.metric_name, 0.0)
|
||||
if alert.evaluate(value):
|
||||
fired.append((alert, value))
|
||||
self._fired_alerts.append(
|
||||
(alert, value, datetime.now(timezone.utc))
|
||||
)
|
||||
return fired
|
||||
|
||||
@property
|
||||
def stage_names(self) -> list[str]:
|
||||
return list(self._stages.keys())
|
||||
|
||||
def summary(self) -> dict[str, Any]:
|
||||
"""Generate a metrics summary for dashboard display."""
|
||||
return {
|
||||
"stages": {
|
||||
name: stage.to_dict() for name, stage in self._stages.items()
|
||||
},
|
||||
"counters": dict(self._counters),
|
||||
"fired_alerts": len(self._fired_alerts),
|
||||
}
|
||||
Reference in New Issue
Block a user