feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,162 @@
|
||||
"""Input/output models for the v3→v2 compatibility adapter.
|
||||
|
||||
V3IntelligenceRecord represents the full v3 pipeline output.
|
||||
V2IntelligenceRecord / V2ImpactRecord match the current document_intelligence
|
||||
and document_impact_records database schemas.
|
||||
AdapterLineage captures version and stage provenance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# V3 Pipeline Output (input to adapter)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class V3SentimentDistribution(BaseModel):
|
||||
"""Per-company calibrated sentiment probabilities."""
|
||||
|
||||
positive: float = Field(ge=0.0, le=1.0)
|
||||
negative: float = Field(ge=0.0, le=1.0)
|
||||
neutral: float = Field(ge=0.0, le=1.0)
|
||||
|
||||
|
||||
class V3HorizonProbabilities(BaseModel):
|
||||
"""Probability distribution over impact horizons."""
|
||||
|
||||
intraday: float = Field(ge=0.0, le=1.0, default=0.0)
|
||||
one_day: float = Field(ge=0.0, le=1.0, default=0.0)
|
||||
seven_day: float = Field(ge=0.0, le=1.0, default=0.0)
|
||||
thirty_day: float = Field(ge=0.0, le=1.0, default=0.0)
|
||||
ninety_day: float = Field(ge=0.0, le=1.0, default=0.0)
|
||||
|
||||
|
||||
class V3DirectionProbabilities(BaseModel):
|
||||
"""Probability distribution over market direction."""
|
||||
|
||||
positive: float = Field(ge=0.0, le=1.0, default=0.0)
|
||||
negative: float = Field(ge=0.0, le=1.0, default=0.0)
|
||||
neutral: float = Field(ge=0.0, le=1.0, default=0.0)
|
||||
|
||||
|
||||
class V3CompanySignal(BaseModel):
|
||||
"""A single company's signal from the v3 pipeline."""
|
||||
|
||||
company_id: str
|
||||
ticker: str
|
||||
relevance_probability: float = Field(ge=0.0, le=1.0)
|
||||
event_classes: list[str] = Field(default_factory=list)
|
||||
sentiment: V3SentimentDistribution
|
||||
direction_probabilities: V3DirectionProbabilities
|
||||
horizon_probabilities: V3HorizonProbabilities
|
||||
expected_magnitude: float | None = None
|
||||
evidence_spans: list[str] = Field(default_factory=list)
|
||||
adjudicated: bool = False
|
||||
|
||||
|
||||
class V3StageRun(BaseModel):
|
||||
"""Lineage for a single pipeline stage execution."""
|
||||
|
||||
stage: str
|
||||
endpoint_id: str | None = None
|
||||
deployment_id: str | None = None
|
||||
model_version: str | None = None
|
||||
schema_version: str = "1.0.0"
|
||||
calibration_version: str | None = None
|
||||
started_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc))
|
||||
duration_ms: int = 0
|
||||
status: str = "completed"
|
||||
|
||||
|
||||
class V3IntelligenceRecord(BaseModel):
|
||||
"""Complete v3 pipeline output for a single document.
|
||||
|
||||
This is the adapter's input — the full v3 record with probabilities,
|
||||
evidence, and stage lineage.
|
||||
"""
|
||||
|
||||
document_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
||||
document_type: str = "article"
|
||||
summary: str = ""
|
||||
macro_themes: list[str] = Field(default_factory=list)
|
||||
novelty_score: float = Field(ge=0.0, le=1.0, default=0.5)
|
||||
confidence: float = Field(ge=0.0, le=1.0, default=0.5)
|
||||
company_signals: list[V3CompanySignal] = Field(default_factory=list)
|
||||
stage_runs: list[V3StageRun] = Field(default_factory=list)
|
||||
pipeline_version: str = "3.0.0"
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# V2 Output (adapter output — matches current DB schema)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class V2ImpactRecord(BaseModel):
|
||||
"""Maps to document_impact_records table.
|
||||
|
||||
Fields match the columns: relevance, sentiment (enum string),
|
||||
impact_score (float), impact_horizon (string), catalyst_type,
|
||||
key_facts, risks, evidence_spans.
|
||||
"""
|
||||
|
||||
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
||||
company_id: str
|
||||
ticker: str
|
||||
relevance: float = Field(ge=0.0, le=1.0)
|
||||
sentiment: Literal["positive", "negative", "neutral", "mixed"]
|
||||
impact_score: float = Field(ge=-1.0, le=1.0)
|
||||
impact_horizon: Literal["intraday", "1d", "7d", "30d", "90d"]
|
||||
catalyst_type: str = "other"
|
||||
key_facts: list[str] = Field(default_factory=list)
|
||||
risks: list[str] = Field(default_factory=list)
|
||||
evidence_spans: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class V2IntelligenceRecord(BaseModel):
|
||||
"""Maps to document_intelligence table.
|
||||
|
||||
Fields match columns: summary, macro_themes, novelty_score,
|
||||
source_credibility, confidence, model_provider, model_name,
|
||||
prompt_version, schema_version, plus associated impact records.
|
||||
"""
|
||||
|
||||
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
||||
document_id: str
|
||||
summary: str = ""
|
||||
macro_themes: list[str] = Field(default_factory=list)
|
||||
novelty_score: float = Field(ge=0.0, le=1.0)
|
||||
source_credibility: float = Field(ge=0.0, le=1.0, default=0.5)
|
||||
confidence: float = Field(ge=0.0, le=1.0)
|
||||
model_provider: str = "hybrid"
|
||||
model_name: str = "intelligence-pipeline-v3"
|
||||
prompt_version: str = ""
|
||||
schema_version: str = "3.0.0"
|
||||
impact_records: list[V2ImpactRecord] = Field(default_factory=list)
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Adapter Lineage
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class AdapterLineage(BaseModel):
|
||||
"""Records which adapter version produced the v2 record and from what v3 data.
|
||||
|
||||
Stored separately so v3 provenance is never lost.
|
||||
"""
|
||||
|
||||
adapter_version: str = "1.0.0"
|
||||
pipeline_version: str = "3.0.0"
|
||||
v3_document_id: str
|
||||
v2_intelligence_id: str
|
||||
stage_runs: list[V3StageRun] = Field(default_factory=list)
|
||||
mapped_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc))
|
||||
mapping_notes: list[str] = Field(default_factory=list)
|
||||
Reference in New Issue
Block a user