feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,180 @@
|
||||
"""Distributed tracing for the v3 intelligence pipeline.
|
||||
|
||||
Every document gets one trace ID that covers preprocessing, specialist
|
||||
stages, routing, adjudication, impact prediction, and persistence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
|
||||
class SpanStatus(str, enum.Enum):
|
||||
"""Status of a trace span."""
|
||||
|
||||
RUNNING = "running"
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
SKIPPED = "skipped"
|
||||
|
||||
|
||||
@dataclass
|
||||
class StageSpan:
|
||||
"""A single stage span within a pipeline trace."""
|
||||
|
||||
span_id: UUID
|
||||
trace_id: UUID
|
||||
stage_name: str
|
||||
parent_span_id: UUID | None
|
||||
started_at: datetime
|
||||
ended_at: datetime | None = None
|
||||
status: SpanStatus = SpanStatus.RUNNING
|
||||
duration_ms: float = 0.0
|
||||
attributes: dict[str, Any] = field(default_factory=dict)
|
||||
error_message: str | None = None
|
||||
|
||||
def finish(
|
||||
self,
|
||||
status: SpanStatus = SpanStatus.SUCCEEDED,
|
||||
error_message: str | None = None,
|
||||
) -> None:
|
||||
"""Mark the span as complete."""
|
||||
self.ended_at = datetime.now(timezone.utc)
|
||||
self.status = status
|
||||
self.error_message = error_message
|
||||
if self.started_at and self.ended_at:
|
||||
self.duration_ms = (
|
||||
self.ended_at - self.started_at
|
||||
).total_seconds() * 1000
|
||||
|
||||
def set_attribute(self, key: str, value: Any) -> None:
|
||||
"""Add a span attribute."""
|
||||
self.attributes[key] = value
|
||||
|
||||
|
||||
@dataclass
|
||||
class PipelineTrace:
|
||||
"""Complete distributed trace for one document through the pipeline."""
|
||||
|
||||
trace_id: UUID
|
||||
document_id: str
|
||||
run_id: UUID
|
||||
started_at: datetime
|
||||
ended_at: datetime | None = None
|
||||
spans: list[StageSpan] = field(default_factory=list)
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def create(cls, document_id: str, run_id: UUID) -> PipelineTrace:
|
||||
return cls(
|
||||
trace_id=uuid4(),
|
||||
document_id=document_id,
|
||||
run_id=run_id,
|
||||
started_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
def start_span(
|
||||
self,
|
||||
stage_name: str,
|
||||
parent_span_id: UUID | None = None,
|
||||
attributes: dict[str, Any] | None = None,
|
||||
) -> StageSpan:
|
||||
"""Start a new span for a pipeline stage."""
|
||||
span = StageSpan(
|
||||
span_id=uuid4(),
|
||||
trace_id=self.trace_id,
|
||||
stage_name=stage_name,
|
||||
parent_span_id=parent_span_id,
|
||||
started_at=datetime.now(timezone.utc),
|
||||
attributes=attributes or {},
|
||||
)
|
||||
self.spans.append(span)
|
||||
return span
|
||||
|
||||
def finish(self) -> None:
|
||||
"""Mark the trace as complete."""
|
||||
self.ended_at = datetime.now(timezone.utc)
|
||||
|
||||
@property
|
||||
def total_duration_ms(self) -> float:
|
||||
if self.started_at and self.ended_at:
|
||||
return (self.ended_at - self.started_at).total_seconds() * 1000
|
||||
return 0.0
|
||||
|
||||
@property
|
||||
def failed_spans(self) -> list[StageSpan]:
|
||||
return [s for s in self.spans if s.status == SpanStatus.FAILED]
|
||||
|
||||
@property
|
||||
def is_complete(self) -> bool:
|
||||
return self.ended_at is not None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Serialize trace for export/storage."""
|
||||
return {
|
||||
"trace_id": str(self.trace_id),
|
||||
"document_id": self.document_id,
|
||||
"run_id": str(self.run_id),
|
||||
"started_at": self.started_at.isoformat(),
|
||||
"ended_at": self.ended_at.isoformat() if self.ended_at else None,
|
||||
"total_duration_ms": self.total_duration_ms,
|
||||
"span_count": len(self.spans),
|
||||
"failed_span_count": len(self.failed_spans),
|
||||
"metadata": self.metadata,
|
||||
"spans": [
|
||||
{
|
||||
"span_id": str(s.span_id),
|
||||
"stage_name": s.stage_name,
|
||||
"status": s.status.value,
|
||||
"duration_ms": s.duration_ms,
|
||||
"attributes": s.attributes,
|
||||
"error_message": s.error_message,
|
||||
}
|
||||
for s in self.spans
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class TraceCollector:
|
||||
"""Collects and stores pipeline traces.
|
||||
|
||||
In production, this would export to an observability backend
|
||||
(Jaeger, Tempo, etc.). This implementation provides the collection
|
||||
logic for testing and local development.
|
||||
"""
|
||||
|
||||
_traces: dict[UUID, PipelineTrace] = field(default_factory=dict)
|
||||
max_stored: int = 10000
|
||||
|
||||
def start_trace(self, document_id: str, run_id: UUID) -> PipelineTrace:
|
||||
"""Create and store a new trace."""
|
||||
trace = PipelineTrace.create(document_id, run_id)
|
||||
self._traces[trace.trace_id] = trace
|
||||
# Evict oldest if over limit
|
||||
if len(self._traces) > self.max_stored:
|
||||
oldest_key = next(iter(self._traces))
|
||||
del self._traces[oldest_key]
|
||||
return trace
|
||||
|
||||
def get_trace(self, trace_id: UUID) -> PipelineTrace | None:
|
||||
return self._traces.get(trace_id)
|
||||
|
||||
def get_by_document(self, document_id: str) -> list[PipelineTrace]:
|
||||
return [
|
||||
t for t in self._traces.values() if t.document_id == document_id
|
||||
]
|
||||
|
||||
def get_by_run(self, run_id: UUID) -> PipelineTrace | None:
|
||||
for t in self._traces.values():
|
||||
if t.run_id == run_id:
|
||||
return t
|
||||
return None
|
||||
|
||||
@property
|
||||
def trace_count(self) -> int:
|
||||
return len(self._traces)
|
||||
Reference in New Issue
Block a user