Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
292 lines
9.0 KiB
Python
292 lines
9.0 KiB
Python
"""Stage metrics, dashboards, and alert definitions for the v3 pipeline.
|
|
|
|
Tracks latency, errors, batch size, queue depth, route metrics, field
|
|
accuracy, evidence coverage, calibration, fast-path rate, adjudication
|
|
reasons, GPU memory, GPU utilization, and GPU-seconds per document.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import enum
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from typing import Any
|
|
|
|
|
|
class AlertSeverity(str, enum.Enum):
|
|
"""Alert severity levels."""
|
|
|
|
INFO = "info"
|
|
WARNING = "warning"
|
|
CRITICAL = "critical"
|
|
|
|
|
|
class MetricType(str, enum.Enum):
|
|
"""Types of collected metrics."""
|
|
|
|
COUNTER = "counter"
|
|
GAUGE = "gauge"
|
|
HISTOGRAM = "histogram"
|
|
SUMMARY = "summary"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MetricAlert:
|
|
"""Alert definition for pipeline metrics."""
|
|
|
|
name: str
|
|
metric_name: str
|
|
condition: str # e.g., "> 0.05", "< 0.6"
|
|
severity: AlertSeverity
|
|
description: str
|
|
threshold: float
|
|
window_seconds: int = 300
|
|
|
|
def evaluate(self, current_value: float) -> bool:
|
|
"""Check if the alert condition is triggered.
|
|
|
|
Returns True if the alert should fire.
|
|
"""
|
|
if self.condition.startswith(">"):
|
|
return current_value > self.threshold
|
|
elif self.condition.startswith("<"):
|
|
return current_value < self.threshold
|
|
elif self.condition.startswith(">="):
|
|
return current_value >= self.threshold
|
|
elif self.condition.startswith("<="):
|
|
return current_value <= self.threshold
|
|
return False
|
|
|
|
|
|
@dataclass
|
|
class StageMetrics:
|
|
"""Metrics for a single pipeline stage."""
|
|
|
|
stage_name: str
|
|
total_invocations: int = 0
|
|
total_errors: int = 0
|
|
total_latency_ms: float = 0.0
|
|
max_latency_ms: float = 0.0
|
|
min_latency_ms: float = float("inf")
|
|
total_tokens_in: int = 0
|
|
total_tokens_out: int = 0
|
|
total_batch_items: int = 0
|
|
total_batches: int = 0
|
|
gpu_seconds: float = 0.0
|
|
gpu_memory_peak_mb: float = 0.0
|
|
gpu_utilization_avg: float = 0.0
|
|
|
|
def record_invocation(
|
|
self,
|
|
latency_ms: float,
|
|
tokens_in: int = 0,
|
|
tokens_out: int = 0,
|
|
error: bool = False,
|
|
batch_size: int = 1,
|
|
gpu_seconds: float = 0.0,
|
|
gpu_memory_mb: float = 0.0,
|
|
gpu_utilization: float = 0.0,
|
|
) -> None:
|
|
"""Record a single stage invocation."""
|
|
self.total_invocations += 1
|
|
self.total_latency_ms += latency_ms
|
|
self.max_latency_ms = max(self.max_latency_ms, latency_ms)
|
|
self.min_latency_ms = min(self.min_latency_ms, latency_ms)
|
|
self.total_tokens_in += tokens_in
|
|
self.total_tokens_out += tokens_out
|
|
self.total_batch_items += batch_size
|
|
self.total_batches += 1
|
|
self.gpu_seconds += gpu_seconds
|
|
self.gpu_memory_peak_mb = max(self.gpu_memory_peak_mb, gpu_memory_mb)
|
|
|
|
if error:
|
|
self.total_errors += 1
|
|
|
|
# Running average for GPU utilization
|
|
if gpu_utilization > 0:
|
|
n = self.total_invocations
|
|
self.gpu_utilization_avg = (
|
|
self.gpu_utilization_avg * (n - 1) + gpu_utilization
|
|
) / n
|
|
|
|
@property
|
|
def avg_latency_ms(self) -> float:
|
|
if self.total_invocations == 0:
|
|
return 0.0
|
|
return self.total_latency_ms / self.total_invocations
|
|
|
|
@property
|
|
def error_rate(self) -> float:
|
|
if self.total_invocations == 0:
|
|
return 0.0
|
|
return self.total_errors / self.total_invocations
|
|
|
|
@property
|
|
def avg_batch_size(self) -> float:
|
|
if self.total_batches == 0:
|
|
return 0.0
|
|
return self.total_batch_items / self.total_batches
|
|
|
|
@property
|
|
def avg_tokens_per_doc(self) -> float:
|
|
if self.total_invocations == 0:
|
|
return 0.0
|
|
return (self.total_tokens_in + self.total_tokens_out) / self.total_invocations
|
|
|
|
@property
|
|
def gpu_seconds_per_doc(self) -> float:
|
|
if self.total_invocations == 0:
|
|
return 0.0
|
|
return self.gpu_seconds / self.total_invocations
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"stage_name": self.stage_name,
|
|
"total_invocations": self.total_invocations,
|
|
"total_errors": self.total_errors,
|
|
"error_rate": self.error_rate,
|
|
"avg_latency_ms": self.avg_latency_ms,
|
|
"max_latency_ms": self.max_latency_ms,
|
|
"avg_batch_size": self.avg_batch_size,
|
|
"gpu_seconds_per_doc": self.gpu_seconds_per_doc,
|
|
"gpu_memory_peak_mb": self.gpu_memory_peak_mb,
|
|
}
|
|
|
|
|
|
# Default alert definitions for the v3 pipeline
|
|
DEFAULT_ALERTS: list[MetricAlert] = [
|
|
MetricAlert(
|
|
name="schema_failure_rate_high",
|
|
metric_name="schema_failures",
|
|
condition="> 0.05",
|
|
severity=AlertSeverity.CRITICAL,
|
|
description="Schema validation failure rate exceeds 5%",
|
|
threshold=0.05,
|
|
),
|
|
MetricAlert(
|
|
name="unsupported_claims_high",
|
|
metric_name="unsupported_claim_rate",
|
|
condition="> 0.10",
|
|
severity=AlertSeverity.WARNING,
|
|
description="Unsupported claim rate exceeds 10%",
|
|
threshold=0.10,
|
|
),
|
|
MetricAlert(
|
|
name="calibration_drift",
|
|
metric_name="calibration_ece",
|
|
condition="> 0.08",
|
|
severity=AlertSeverity.WARNING,
|
|
description="Calibration ECE exceeds 8%",
|
|
threshold=0.08,
|
|
),
|
|
MetricAlert(
|
|
name="queue_saturation",
|
|
metric_name="queue_saturation_ratio",
|
|
condition="> 0.90",
|
|
severity=AlertSeverity.CRITICAL,
|
|
description="Queue saturation exceeds 90%",
|
|
threshold=0.90,
|
|
),
|
|
MetricAlert(
|
|
name="provider_probe_failure",
|
|
metric_name="probe_failure_rate",
|
|
condition="> 0.0",
|
|
severity=AlertSeverity.CRITICAL,
|
|
description="Provider capability probe failed",
|
|
threshold=0.0,
|
|
),
|
|
MetricAlert(
|
|
name="gpu_memory_high",
|
|
metric_name="gpu_memory_utilization",
|
|
condition="> 0.85",
|
|
severity=AlertSeverity.WARNING,
|
|
description="GPU memory utilization exceeds 85%",
|
|
threshold=0.85,
|
|
),
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class MetricsCollector:
|
|
"""Collects and aggregates metrics across pipeline stages.
|
|
|
|
In production, this would export to Prometheus/Grafana.
|
|
This implementation provides the collection logic for testing.
|
|
"""
|
|
|
|
_stages: dict[str, StageMetrics] = field(default_factory=dict)
|
|
_alerts: list[MetricAlert] = field(default_factory=list)
|
|
_counters: dict[str, float] = field(default_factory=dict)
|
|
_fired_alerts: list[tuple[MetricAlert, float, datetime]] = field(
|
|
default_factory=list
|
|
)
|
|
|
|
def __post_init__(self) -> None:
|
|
if not self._alerts:
|
|
self._alerts = list(DEFAULT_ALERTS)
|
|
|
|
def get_stage(self, stage_name: str) -> StageMetrics:
|
|
"""Get or create metrics for a stage."""
|
|
if stage_name not in self._stages:
|
|
self._stages[stage_name] = StageMetrics(stage_name=stage_name)
|
|
return self._stages[stage_name]
|
|
|
|
def record_stage(
|
|
self,
|
|
stage_name: str,
|
|
latency_ms: float,
|
|
tokens_in: int = 0,
|
|
tokens_out: int = 0,
|
|
error: bool = False,
|
|
batch_size: int = 1,
|
|
gpu_seconds: float = 0.0,
|
|
gpu_memory_mb: float = 0.0,
|
|
gpu_utilization: float = 0.0,
|
|
) -> None:
|
|
"""Record a stage invocation."""
|
|
stage = self.get_stage(stage_name)
|
|
stage.record_invocation(
|
|
latency_ms=latency_ms,
|
|
tokens_in=tokens_in,
|
|
tokens_out=tokens_out,
|
|
error=error,
|
|
batch_size=batch_size,
|
|
gpu_seconds=gpu_seconds,
|
|
gpu_memory_mb=gpu_memory_mb,
|
|
gpu_utilization=gpu_utilization,
|
|
)
|
|
|
|
def increment_counter(self, name: str, value: float = 1.0) -> None:
|
|
"""Increment a named counter."""
|
|
self._counters[name] = self._counters.get(name, 0.0) + value
|
|
|
|
def get_counter(self, name: str) -> float:
|
|
"""Get current counter value."""
|
|
return self._counters.get(name, 0.0)
|
|
|
|
def check_alerts(self) -> list[tuple[MetricAlert, float]]:
|
|
"""Evaluate all alert conditions. Returns (alert, value) for fired alerts."""
|
|
fired: list[tuple[MetricAlert, float]] = []
|
|
for alert in self._alerts:
|
|
value = self._counters.get(alert.metric_name, 0.0)
|
|
if alert.evaluate(value):
|
|
fired.append((alert, value))
|
|
self._fired_alerts.append(
|
|
(alert, value, datetime.now(timezone.utc))
|
|
)
|
|
return fired
|
|
|
|
@property
|
|
def stage_names(self) -> list[str]:
|
|
return list(self._stages.keys())
|
|
|
|
def summary(self) -> dict[str, Any]:
|
|
"""Generate a metrics summary for dashboard display."""
|
|
return {
|
|
"stages": {
|
|
name: stage.to_dict() for name, stage in self._stages.items()
|
|
},
|
|
"counters": dict(self._counters),
|
|
"fired_alerts": len(self._fired_alerts),
|
|
}
|