feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,28 @@
|
||||
"""Observability module — distributed tracing, stage metrics, dashboards, and alerts.
|
||||
|
||||
Provides unified tracing across pipeline stages, metric collection for
|
||||
latency/errors/batch-size/queue-depth/routing, and alert definitions
|
||||
for operational monitoring.
|
||||
"""
|
||||
|
||||
from services.intelligence_pipeline_v3.observability.metrics import (
|
||||
AlertSeverity,
|
||||
MetricAlert,
|
||||
MetricsCollector,
|
||||
StageMetrics,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.observability.tracing import (
|
||||
PipelineTrace,
|
||||
StageSpan,
|
||||
TraceCollector,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AlertSeverity",
|
||||
"MetricAlert",
|
||||
"MetricsCollector",
|
||||
"PipelineTrace",
|
||||
"StageMetrics",
|
||||
"StageSpan",
|
||||
"TraceCollector",
|
||||
]
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Stage metrics, dashboards, and alert definitions for the v3 pipeline.
|
||||
|
||||
Tracks latency, errors, batch size, queue depth, route metrics, field
|
||||
accuracy, evidence coverage, calibration, fast-path rate, adjudication
|
||||
reasons, GPU memory, GPU utilization, and GPU-seconds per document.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
|
||||
class AlertSeverity(str, enum.Enum):
|
||||
"""Alert severity levels."""
|
||||
|
||||
INFO = "info"
|
||||
WARNING = "warning"
|
||||
CRITICAL = "critical"
|
||||
|
||||
|
||||
class MetricType(str, enum.Enum):
|
||||
"""Types of collected metrics."""
|
||||
|
||||
COUNTER = "counter"
|
||||
GAUGE = "gauge"
|
||||
HISTOGRAM = "histogram"
|
||||
SUMMARY = "summary"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MetricAlert:
|
||||
"""Alert definition for pipeline metrics."""
|
||||
|
||||
name: str
|
||||
metric_name: str
|
||||
condition: str # e.g., "> 0.05", "< 0.6"
|
||||
severity: AlertSeverity
|
||||
description: str
|
||||
threshold: float
|
||||
window_seconds: int = 300
|
||||
|
||||
def evaluate(self, current_value: float) -> bool:
|
||||
"""Check if the alert condition is triggered.
|
||||
|
||||
Returns True if the alert should fire.
|
||||
"""
|
||||
if self.condition.startswith(">"):
|
||||
return current_value > self.threshold
|
||||
elif self.condition.startswith("<"):
|
||||
return current_value < self.threshold
|
||||
elif self.condition.startswith(">="):
|
||||
return current_value >= self.threshold
|
||||
elif self.condition.startswith("<="):
|
||||
return current_value <= self.threshold
|
||||
return False
|
||||
|
||||
|
||||
@dataclass
|
||||
class StageMetrics:
|
||||
"""Metrics for a single pipeline stage."""
|
||||
|
||||
stage_name: str
|
||||
total_invocations: int = 0
|
||||
total_errors: int = 0
|
||||
total_latency_ms: float = 0.0
|
||||
max_latency_ms: float = 0.0
|
||||
min_latency_ms: float = float("inf")
|
||||
total_tokens_in: int = 0
|
||||
total_tokens_out: int = 0
|
||||
total_batch_items: int = 0
|
||||
total_batches: int = 0
|
||||
gpu_seconds: float = 0.0
|
||||
gpu_memory_peak_mb: float = 0.0
|
||||
gpu_utilization_avg: float = 0.0
|
||||
|
||||
def record_invocation(
|
||||
self,
|
||||
latency_ms: float,
|
||||
tokens_in: int = 0,
|
||||
tokens_out: int = 0,
|
||||
error: bool = False,
|
||||
batch_size: int = 1,
|
||||
gpu_seconds: float = 0.0,
|
||||
gpu_memory_mb: float = 0.0,
|
||||
gpu_utilization: float = 0.0,
|
||||
) -> None:
|
||||
"""Record a single stage invocation."""
|
||||
self.total_invocations += 1
|
||||
self.total_latency_ms += latency_ms
|
||||
self.max_latency_ms = max(self.max_latency_ms, latency_ms)
|
||||
self.min_latency_ms = min(self.min_latency_ms, latency_ms)
|
||||
self.total_tokens_in += tokens_in
|
||||
self.total_tokens_out += tokens_out
|
||||
self.total_batch_items += batch_size
|
||||
self.total_batches += 1
|
||||
self.gpu_seconds += gpu_seconds
|
||||
self.gpu_memory_peak_mb = max(self.gpu_memory_peak_mb, gpu_memory_mb)
|
||||
|
||||
if error:
|
||||
self.total_errors += 1
|
||||
|
||||
# Running average for GPU utilization
|
||||
if gpu_utilization > 0:
|
||||
n = self.total_invocations
|
||||
self.gpu_utilization_avg = (
|
||||
self.gpu_utilization_avg * (n - 1) + gpu_utilization
|
||||
) / n
|
||||
|
||||
@property
|
||||
def avg_latency_ms(self) -> float:
|
||||
if self.total_invocations == 0:
|
||||
return 0.0
|
||||
return self.total_latency_ms / self.total_invocations
|
||||
|
||||
@property
|
||||
def error_rate(self) -> float:
|
||||
if self.total_invocations == 0:
|
||||
return 0.0
|
||||
return self.total_errors / self.total_invocations
|
||||
|
||||
@property
|
||||
def avg_batch_size(self) -> float:
|
||||
if self.total_batches == 0:
|
||||
return 0.0
|
||||
return self.total_batch_items / self.total_batches
|
||||
|
||||
@property
|
||||
def avg_tokens_per_doc(self) -> float:
|
||||
if self.total_invocations == 0:
|
||||
return 0.0
|
||||
return (self.total_tokens_in + self.total_tokens_out) / self.total_invocations
|
||||
|
||||
@property
|
||||
def gpu_seconds_per_doc(self) -> float:
|
||||
if self.total_invocations == 0:
|
||||
return 0.0
|
||||
return self.gpu_seconds / self.total_invocations
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"stage_name": self.stage_name,
|
||||
"total_invocations": self.total_invocations,
|
||||
"total_errors": self.total_errors,
|
||||
"error_rate": self.error_rate,
|
||||
"avg_latency_ms": self.avg_latency_ms,
|
||||
"max_latency_ms": self.max_latency_ms,
|
||||
"avg_batch_size": self.avg_batch_size,
|
||||
"gpu_seconds_per_doc": self.gpu_seconds_per_doc,
|
||||
"gpu_memory_peak_mb": self.gpu_memory_peak_mb,
|
||||
}
|
||||
|
||||
|
||||
# Default alert definitions for the v3 pipeline
|
||||
DEFAULT_ALERTS: list[MetricAlert] = [
|
||||
MetricAlert(
|
||||
name="schema_failure_rate_high",
|
||||
metric_name="schema_failures",
|
||||
condition="> 0.05",
|
||||
severity=AlertSeverity.CRITICAL,
|
||||
description="Schema validation failure rate exceeds 5%",
|
||||
threshold=0.05,
|
||||
),
|
||||
MetricAlert(
|
||||
name="unsupported_claims_high",
|
||||
metric_name="unsupported_claim_rate",
|
||||
condition="> 0.10",
|
||||
severity=AlertSeverity.WARNING,
|
||||
description="Unsupported claim rate exceeds 10%",
|
||||
threshold=0.10,
|
||||
),
|
||||
MetricAlert(
|
||||
name="calibration_drift",
|
||||
metric_name="calibration_ece",
|
||||
condition="> 0.08",
|
||||
severity=AlertSeverity.WARNING,
|
||||
description="Calibration ECE exceeds 8%",
|
||||
threshold=0.08,
|
||||
),
|
||||
MetricAlert(
|
||||
name="queue_saturation",
|
||||
metric_name="queue_saturation_ratio",
|
||||
condition="> 0.90",
|
||||
severity=AlertSeverity.CRITICAL,
|
||||
description="Queue saturation exceeds 90%",
|
||||
threshold=0.90,
|
||||
),
|
||||
MetricAlert(
|
||||
name="provider_probe_failure",
|
||||
metric_name="probe_failure_rate",
|
||||
condition="> 0.0",
|
||||
severity=AlertSeverity.CRITICAL,
|
||||
description="Provider capability probe failed",
|
||||
threshold=0.0,
|
||||
),
|
||||
MetricAlert(
|
||||
name="gpu_memory_high",
|
||||
metric_name="gpu_memory_utilization",
|
||||
condition="> 0.85",
|
||||
severity=AlertSeverity.WARNING,
|
||||
description="GPU memory utilization exceeds 85%",
|
||||
threshold=0.85,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetricsCollector:
|
||||
"""Collects and aggregates metrics across pipeline stages.
|
||||
|
||||
In production, this would export to Prometheus/Grafana.
|
||||
This implementation provides the collection logic for testing.
|
||||
"""
|
||||
|
||||
_stages: dict[str, StageMetrics] = field(default_factory=dict)
|
||||
_alerts: list[MetricAlert] = field(default_factory=list)
|
||||
_counters: dict[str, float] = field(default_factory=dict)
|
||||
_fired_alerts: list[tuple[MetricAlert, float, datetime]] = field(
|
||||
default_factory=list
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self._alerts:
|
||||
self._alerts = list(DEFAULT_ALERTS)
|
||||
|
||||
def get_stage(self, stage_name: str) -> StageMetrics:
|
||||
"""Get or create metrics for a stage."""
|
||||
if stage_name not in self._stages:
|
||||
self._stages[stage_name] = StageMetrics(stage_name=stage_name)
|
||||
return self._stages[stage_name]
|
||||
|
||||
def record_stage(
|
||||
self,
|
||||
stage_name: str,
|
||||
latency_ms: float,
|
||||
tokens_in: int = 0,
|
||||
tokens_out: int = 0,
|
||||
error: bool = False,
|
||||
batch_size: int = 1,
|
||||
gpu_seconds: float = 0.0,
|
||||
gpu_memory_mb: float = 0.0,
|
||||
gpu_utilization: float = 0.0,
|
||||
) -> None:
|
||||
"""Record a stage invocation."""
|
||||
stage = self.get_stage(stage_name)
|
||||
stage.record_invocation(
|
||||
latency_ms=latency_ms,
|
||||
tokens_in=tokens_in,
|
||||
tokens_out=tokens_out,
|
||||
error=error,
|
||||
batch_size=batch_size,
|
||||
gpu_seconds=gpu_seconds,
|
||||
gpu_memory_mb=gpu_memory_mb,
|
||||
gpu_utilization=gpu_utilization,
|
||||
)
|
||||
|
||||
def increment_counter(self, name: str, value: float = 1.0) -> None:
|
||||
"""Increment a named counter."""
|
||||
self._counters[name] = self._counters.get(name, 0.0) + value
|
||||
|
||||
def get_counter(self, name: str) -> float:
|
||||
"""Get current counter value."""
|
||||
return self._counters.get(name, 0.0)
|
||||
|
||||
def check_alerts(self) -> list[tuple[MetricAlert, float]]:
|
||||
"""Evaluate all alert conditions. Returns (alert, value) for fired alerts."""
|
||||
fired: list[tuple[MetricAlert, float]] = []
|
||||
for alert in self._alerts:
|
||||
value = self._counters.get(alert.metric_name, 0.0)
|
||||
if alert.evaluate(value):
|
||||
fired.append((alert, value))
|
||||
self._fired_alerts.append(
|
||||
(alert, value, datetime.now(timezone.utc))
|
||||
)
|
||||
return fired
|
||||
|
||||
@property
|
||||
def stage_names(self) -> list[str]:
|
||||
return list(self._stages.keys())
|
||||
|
||||
def summary(self) -> dict[str, Any]:
|
||||
"""Generate a metrics summary for dashboard display."""
|
||||
return {
|
||||
"stages": {
|
||||
name: stage.to_dict() for name, stage in self._stages.items()
|
||||
},
|
||||
"counters": dict(self._counters),
|
||||
"fired_alerts": len(self._fired_alerts),
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Distributed tracing for the v3 intelligence pipeline.
|
||||
|
||||
Every document gets one trace ID that covers preprocessing, specialist
|
||||
stages, routing, adjudication, impact prediction, and persistence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
|
||||
class SpanStatus(str, enum.Enum):
|
||||
"""Status of a trace span."""
|
||||
|
||||
RUNNING = "running"
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
SKIPPED = "skipped"
|
||||
|
||||
|
||||
@dataclass
|
||||
class StageSpan:
|
||||
"""A single stage span within a pipeline trace."""
|
||||
|
||||
span_id: UUID
|
||||
trace_id: UUID
|
||||
stage_name: str
|
||||
parent_span_id: UUID | None
|
||||
started_at: datetime
|
||||
ended_at: datetime | None = None
|
||||
status: SpanStatus = SpanStatus.RUNNING
|
||||
duration_ms: float = 0.0
|
||||
attributes: dict[str, Any] = field(default_factory=dict)
|
||||
error_message: str | None = None
|
||||
|
||||
def finish(
|
||||
self,
|
||||
status: SpanStatus = SpanStatus.SUCCEEDED,
|
||||
error_message: str | None = None,
|
||||
) -> None:
|
||||
"""Mark the span as complete."""
|
||||
self.ended_at = datetime.now(timezone.utc)
|
||||
self.status = status
|
||||
self.error_message = error_message
|
||||
if self.started_at and self.ended_at:
|
||||
self.duration_ms = (
|
||||
self.ended_at - self.started_at
|
||||
).total_seconds() * 1000
|
||||
|
||||
def set_attribute(self, key: str, value: Any) -> None:
|
||||
"""Add a span attribute."""
|
||||
self.attributes[key] = value
|
||||
|
||||
|
||||
@dataclass
|
||||
class PipelineTrace:
|
||||
"""Complete distributed trace for one document through the pipeline."""
|
||||
|
||||
trace_id: UUID
|
||||
document_id: str
|
||||
run_id: UUID
|
||||
started_at: datetime
|
||||
ended_at: datetime | None = None
|
||||
spans: list[StageSpan] = field(default_factory=list)
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def create(cls, document_id: str, run_id: UUID) -> PipelineTrace:
|
||||
return cls(
|
||||
trace_id=uuid4(),
|
||||
document_id=document_id,
|
||||
run_id=run_id,
|
||||
started_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
def start_span(
|
||||
self,
|
||||
stage_name: str,
|
||||
parent_span_id: UUID | None = None,
|
||||
attributes: dict[str, Any] | None = None,
|
||||
) -> StageSpan:
|
||||
"""Start a new span for a pipeline stage."""
|
||||
span = StageSpan(
|
||||
span_id=uuid4(),
|
||||
trace_id=self.trace_id,
|
||||
stage_name=stage_name,
|
||||
parent_span_id=parent_span_id,
|
||||
started_at=datetime.now(timezone.utc),
|
||||
attributes=attributes or {},
|
||||
)
|
||||
self.spans.append(span)
|
||||
return span
|
||||
|
||||
def finish(self) -> None:
|
||||
"""Mark the trace as complete."""
|
||||
self.ended_at = datetime.now(timezone.utc)
|
||||
|
||||
@property
|
||||
def total_duration_ms(self) -> float:
|
||||
if self.started_at and self.ended_at:
|
||||
return (self.ended_at - self.started_at).total_seconds() * 1000
|
||||
return 0.0
|
||||
|
||||
@property
|
||||
def failed_spans(self) -> list[StageSpan]:
|
||||
return [s for s in self.spans if s.status == SpanStatus.FAILED]
|
||||
|
||||
@property
|
||||
def is_complete(self) -> bool:
|
||||
return self.ended_at is not None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Serialize trace for export/storage."""
|
||||
return {
|
||||
"trace_id": str(self.trace_id),
|
||||
"document_id": self.document_id,
|
||||
"run_id": str(self.run_id),
|
||||
"started_at": self.started_at.isoformat(),
|
||||
"ended_at": self.ended_at.isoformat() if self.ended_at else None,
|
||||
"total_duration_ms": self.total_duration_ms,
|
||||
"span_count": len(self.spans),
|
||||
"failed_span_count": len(self.failed_spans),
|
||||
"metadata": self.metadata,
|
||||
"spans": [
|
||||
{
|
||||
"span_id": str(s.span_id),
|
||||
"stage_name": s.stage_name,
|
||||
"status": s.status.value,
|
||||
"duration_ms": s.duration_ms,
|
||||
"attributes": s.attributes,
|
||||
"error_message": s.error_message,
|
||||
}
|
||||
for s in self.spans
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class TraceCollector:
|
||||
"""Collects and stores pipeline traces.
|
||||
|
||||
In production, this would export to an observability backend
|
||||
(Jaeger, Tempo, etc.). This implementation provides the collection
|
||||
logic for testing and local development.
|
||||
"""
|
||||
|
||||
_traces: dict[UUID, PipelineTrace] = field(default_factory=dict)
|
||||
max_stored: int = 10000
|
||||
|
||||
def start_trace(self, document_id: str, run_id: UUID) -> PipelineTrace:
|
||||
"""Create and store a new trace."""
|
||||
trace = PipelineTrace.create(document_id, run_id)
|
||||
self._traces[trace.trace_id] = trace
|
||||
# Evict oldest if over limit
|
||||
if len(self._traces) > self.max_stored:
|
||||
oldest_key = next(iter(self._traces))
|
||||
del self._traces[oldest_key]
|
||||
return trace
|
||||
|
||||
def get_trace(self, trace_id: UUID) -> PipelineTrace | None:
|
||||
return self._traces.get(trace_id)
|
||||
|
||||
def get_by_document(self, document_id: str) -> list[PipelineTrace]:
|
||||
return [
|
||||
t for t in self._traces.values() if t.document_id == document_id
|
||||
]
|
||||
|
||||
def get_by_run(self, run_id: UUID) -> PipelineTrace | None:
|
||||
for t in self._traces.values():
|
||||
if t.run_id == run_id:
|
||||
return t
|
||||
return None
|
||||
|
||||
@property
|
||||
def trace_count(self) -> int:
|
||||
return len(self._traces)
|
||||
Reference in New Issue
Block a user