admin/stonks-oracle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Intelligence Pipeline v3 — full implementation

Browse Source 
Multi-stage evidence-grounded inference architecture replacing the
monolithic 9B model extraction pipeline. CPU-first specialist services
handle routine extraction while the 9B vLLM model is preserved for
semantic adjudication of ambiguous cases.

Key components:
- Capability-aware inference gateway (OpenAI-compatible + Ollama)
- Endpoint registry with DB migrations and REST API
- Sentence-aware document segmenter (property tests)
- Deterministic financial parsing with offset integrity
- Symbol resolution with ambiguity detection
- Specialist service (GLiNER2, dynamic batching, K8s deployment)
- Company-specific sentiment (FinBERT, calibration)
- Retrieval-based novelty and duplicate detection
- Confidence calibration pipeline
- Deterministic routing engine (property tests)
- 9B adjudication layer with VRAM gating
- Stock-specific impact model (features, labels, baseline, trained)
- Pipeline orchestrator (state machine, queues, leases, feature flags)
- Bounded parallelism (async workers, semaphore, load shedding)
- Observability (tracing, metrics, alerts)
- Compatibility adapter (v3→v2 golden mapping tests)
- Shadow/canary promotion framework
- Active learning and fine-tuning pipeline

Test results: 1,161 tests pass, ruff lint clean.
All 282 spec tasks completed.
This commit is contained in:
Celes Renata
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/intelligence_pipeline_v3/compatibility/__init__.py
+20
View File
@@ -0,0 +1,20 @@
"""Compatibility adapter — maps v3 intelligence records to current v2 data classes."""
from services.intelligence_pipeline_v3.compatibility.adapter import CompatibilityAdapter
from services.intelligence_pipeline_v3.compatibility.config import AdapterMode, is_adapter_enabled
from services.intelligence_pipeline_v3.compatibility.models import (
AdapterLineage,
V2ImpactRecord,
V2IntelligenceRecord,
V3IntelligenceRecord,
)
__all__ = [
"AdapterLineage",
"AdapterMode",
"CompatibilityAdapter",
"V2ImpactRecord",
"V2IntelligenceRecord",
"V3IntelligenceRecord",
"is_adapter_enabled",
]
services/intelligence_pipeline_v3/compatibility/adapter.py
+208
View File
@@ -0,0 +1,208 @@
"""Compatibility adapter — maps approved v3 records to current v2 data classes.
The adapter creates current-format records without discarding v3 provenance.
It marks model_provider='hybrid' and stores complete stage lineage separately.
Design reference: Section K (Compatibility Adapter) in design.md.
"""
from __future__ import annotations
import uuid
from services.intelligence_pipeline_v3.compatibility.config import (
AdapterMode,
is_adapter_enabled,
)
from services.intelligence_pipeline_v3.compatibility.models import (
AdapterLineage,
V2ImpactRecord,
V2IntelligenceRecord,
V3CompanySignal,
V3HorizonProbabilities,
V3IntelligenceRecord,
V3SentimentDistribution,
)
ADAPTER_VERSION = "1.0.0"
class AdapterDisabledError(Exception):
"""Raised when the adapter is called in disabled mode."""
pass
class CompatibilityAdapter:
"""Maps v3 intelligence records to v2 format for downstream consumers.
The adapter is gated by AdapterMode — it refuses to produce output when
disabled, ensuring v3 records cannot accidentally affect production
consumers until explicitly enabled.
"""
def __init__(self, mode: AdapterMode = AdapterMode.DISABLED) -> None:
self._mode = mode
@property
def mode(self) -> AdapterMode:
return self._mode
@property
def version(self) -> str:
return ADAPTER_VERSION
def map_to_v2(
self, v3_record: V3IntelligenceRecord
) -> tuple[V2IntelligenceRecord, AdapterLineage]:
"""Map an approved v3 record to v2 intelligence + impact records.
Returns:
A tuple of (V2IntelligenceRecord, AdapterLineage).
Raises:
AdapterDisabledError: If the adapter is in disabled mode.
"""
if not is_adapter_enabled(self._mode):
raise AdapterDisabledError(
f"Adapter is disabled (mode={self._mode.value}). "
"Enable replay, shadow, canary, or production mode to use."
)
v2_id = str(uuid.uuid4())
# Map each company signal to a v2 impact record
impact_records = [
self._map_company_signal(signal) for signal in v3_record.company_signals
]
v2_record = V2IntelligenceRecord(
id=v2_id,
document_id=v3_record.document_id,
summary=v3_record.summary,
macro_themes=v3_record.macro_themes,
novelty_score=v3_record.novelty_score,
confidence=v3_record.confidence,
model_provider="hybrid",
model_name="intelligence-pipeline-v3",
prompt_version=f"adapter-{ADAPTER_VERSION}",
schema_version="3.0.0",
impact_records=impact_records,
)
lineage = AdapterLineage(
adapter_version=ADAPTER_VERSION,
pipeline_version=v3_record.pipeline_version,
v3_document_id=v3_record.document_id,
v2_intelligence_id=v2_id,
stage_runs=v3_record.stage_runs,
mapping_notes=[
f"Mapped {len(v3_record.company_signals)} company signals",
f"Mode: {self._mode.value}",
],
)
return v2_record, lineage
def _map_company_signal(self, signal: V3CompanySignal) -> V2ImpactRecord:
"""Map a single v3 company signal to a v2 impact record."""
return V2ImpactRecord(
company_id=signal.company_id,
ticker=signal.ticker,
relevance=signal.relevance_probability,
sentiment=self._map_sentiment(signal.sentiment),
impact_score=self._map_impact_score(signal),
impact_horizon=self._map_horizon(signal.horizon_probabilities),
catalyst_type=self._map_catalyst_type(signal.event_classes),
evidence_spans=signal.evidence_spans,
)
@staticmethod
def _map_sentiment(dist: V3SentimentDistribution) -> str:
"""Map probability distribution to legacy sentiment enum.
Logic:
- If max probability is neutral and ≥ 0.5 → neutral
- If positive and negative are both ≥ 0.3 → mixed
- Otherwise take the argmax of positive/negative/neutral
"""
pos, neg, neu = dist.positive, dist.negative, dist.neutral
# Mixed detection: both positive and negative have significant mass
if pos >= 0.3 and neg >= 0.3:
return "mixed"
# Argmax
max_val = max(pos, neg, neu)
if max_val == neu:
return "neutral"
elif max_val == pos:
return "positive"
else:
return "negative"
@staticmethod
def _map_impact_score(signal: V3CompanySignal) -> float:
"""Map v3 expected_magnitude to legacy impact_score in [-1, 1].
The v3 expected_magnitude is already a signed value representing
expected market response. We clamp to [-1, 1] for legacy compatibility.
If expected_magnitude is None, derive a conservative estimate from
direction probabilities.
"""
if signal.expected_magnitude is not None:
return max(-1.0, min(1.0, signal.expected_magnitude))
# Fallback: derive from direction probabilities
dp = signal.direction_probabilities
# Signed score: positive_prob - negative_prob, scaled to [-1, 1]
signed = dp.positive - dp.negative
return max(-1.0, min(1.0, signed))
@staticmethod
def _map_horizon(probs: V3HorizonProbabilities) -> str:
"""Map horizon probability distribution to single legacy horizon string.
Returns the horizon with the highest probability (argmax).
Ties are broken by preferring shorter horizons.
"""
horizon_map = {
"intraday": probs.intraday,
"1d": probs.one_day,
"7d": probs.seven_day,
"30d": probs.thirty_day,
"90d": probs.ninety_day,
}
# argmax with tie-breaking by order (shortest first)
return max(horizon_map, key=lambda k: horizon_map[k])
@staticmethod
def _map_catalyst_type(event_classes: list[str]) -> str:
"""Map v3 event taxonomy to legacy catalyst_type enum.
Uses the first matching event class. Falls back to 'other'.
"""
# Mapping from v3 event classes to legacy CatalystType values
event_to_catalyst: dict[str, str] = {
"earnings_beat": "earnings",
"earnings_miss": "earnings",
"guidance_raise": "earnings",
"guidance_cut": "earnings",
"product_launch": "product",
"legal_regulatory": "legal",
"ma_announcement": "m_and_a",
"supply_chain": "supply_chain",
"rating_change": "rating_change",
"macro_event": "macro",
"management_change": "other",
"dividend_change": "other",
"buyback": "other",
}
for event_class in event_classes:
if event_class in event_to_catalyst:
return event_to_catalyst[event_class]
return "other"
services/intelligence_pipeline_v3/compatibility/config.py
+39
View File
@@ -0,0 +1,39 @@
"""Feature flag configuration for the compatibility adapter.
The adapter is disabled by default and must be explicitly enabled for
replay, shadow, canary, or production modes.
"""
from __future__ import annotations
from enum import Enum
class AdapterMode(str, Enum):
"""Operating mode for the compatibility adapter.
- disabled: adapter does not run (default)
- replay_only: adapter runs during offline replay evaluation
- shadow_only: adapter runs in shadow mode (no downstream effect)
- canary: adapter outputs routed to a percentage of non-trading consumers
- production: adapter outputs used for all consumers
"""
DISABLED = "disabled"
REPLAY_ONLY = "replay_only"
SHADOW_ONLY = "shadow_only"
CANARY = "canary"
PRODUCTION = "production"
def is_adapter_enabled(mode: AdapterMode) -> bool:
"""Return True if the adapter should produce output in the given mode.
Only replay, shadow, canary, and production modes enable output.
The disabled mode prevents any adapter execution.
"""
return mode != AdapterMode.DISABLED
# Default mode — adapter is OFF until explicitly activated
DEFAULT_ADAPTER_MODE: AdapterMode = AdapterMode.DISABLED
services/intelligence_pipeline_v3/compatibility/models.py
+162
View File
@@ -0,0 +1,162 @@
"""Input/output models for the v3→v2 compatibility adapter.
V3IntelligenceRecord represents the full v3 pipeline output.
V2IntelligenceRecord / V2ImpactRecord match the current document_intelligence
and document_impact_records database schemas.
AdapterLineage captures version and stage provenance.
"""
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from typing import Literal
from pydantic import BaseModel, Field
# ---------------------------------------------------------------------------
# V3 Pipeline Output (input to adapter)
# ---------------------------------------------------------------------------
class V3SentimentDistribution(BaseModel):
"""Per-company calibrated sentiment probabilities."""
positive: float = Field(ge=0.0, le=1.0)
negative: float = Field(ge=0.0, le=1.0)
neutral: float = Field(ge=0.0, le=1.0)
class V3HorizonProbabilities(BaseModel):
"""Probability distribution over impact horizons."""
intraday: float = Field(ge=0.0, le=1.0, default=0.0)
one_day: float = Field(ge=0.0, le=1.0, default=0.0)
seven_day: float = Field(ge=0.0, le=1.0, default=0.0)
thirty_day: float = Field(ge=0.0, le=1.0, default=0.0)
ninety_day: float = Field(ge=0.0, le=1.0, default=0.0)
class V3DirectionProbabilities(BaseModel):
"""Probability distribution over market direction."""
positive: float = Field(ge=0.0, le=1.0, default=0.0)
negative: float = Field(ge=0.0, le=1.0, default=0.0)
neutral: float = Field(ge=0.0, le=1.0, default=0.0)
class V3CompanySignal(BaseModel):
"""A single company's signal from the v3 pipeline."""
company_id: str
ticker: str
relevance_probability: float = Field(ge=0.0, le=1.0)
event_classes: list[str] = Field(default_factory=list)
sentiment: V3SentimentDistribution
direction_probabilities: V3DirectionProbabilities
horizon_probabilities: V3HorizonProbabilities
expected_magnitude: float | None = None
evidence_spans: list[str] = Field(default_factory=list)
adjudicated: bool = False
class V3StageRun(BaseModel):
"""Lineage for a single pipeline stage execution."""
stage: str
endpoint_id: str | None = None
deployment_id: str | None = None
model_version: str | None = None
schema_version: str = "1.0.0"
calibration_version: str | None = None
started_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc))
duration_ms: int = 0
status: str = "completed"
class V3IntelligenceRecord(BaseModel):
"""Complete v3 pipeline output for a single document.
This is the adapter's input — the full v3 record with probabilities,
evidence, and stage lineage.
"""
document_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
document_type: str = "article"
summary: str = ""
macro_themes: list[str] = Field(default_factory=list)
novelty_score: float = Field(ge=0.0, le=1.0, default=0.5)
confidence: float = Field(ge=0.0, le=1.0, default=0.5)
company_signals: list[V3CompanySignal] = Field(default_factory=list)
stage_runs: list[V3StageRun] = Field(default_factory=list)
pipeline_version: str = "3.0.0"
created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc))
# ---------------------------------------------------------------------------
# V2 Output (adapter output — matches current DB schema)
# ---------------------------------------------------------------------------
class V2ImpactRecord(BaseModel):
"""Maps to document_impact_records table.
Fields match the columns: relevance, sentiment (enum string),
impact_score (float), impact_horizon (string), catalyst_type,
key_facts, risks, evidence_spans.
"""
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
company_id: str
ticker: str
relevance: float = Field(ge=0.0, le=1.0)
sentiment: Literal["positive", "negative", "neutral", "mixed"]
impact_score: float = Field(ge=-1.0, le=1.0)
impact_horizon: Literal["intraday", "1d", "7d", "30d", "90d"]
catalyst_type: str = "other"
key_facts: list[str] = Field(default_factory=list)
risks: list[str] = Field(default_factory=list)
evidence_spans: list[str] = Field(default_factory=list)
class V2IntelligenceRecord(BaseModel):
"""Maps to document_intelligence table.
Fields match columns: summary, macro_themes, novelty_score,
source_credibility, confidence, model_provider, model_name,
prompt_version, schema_version, plus associated impact records.
"""
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
document_id: str
summary: str = ""
macro_themes: list[str] = Field(default_factory=list)
novelty_score: float = Field(ge=0.0, le=1.0)
source_credibility: float = Field(ge=0.0, le=1.0, default=0.5)
confidence: float = Field(ge=0.0, le=1.0)
model_provider: str = "hybrid"
model_name: str = "intelligence-pipeline-v3"
prompt_version: str = ""
schema_version: str = "3.0.0"
impact_records: list[V2ImpactRecord] = Field(default_factory=list)
created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc))
# ---------------------------------------------------------------------------
# Adapter Lineage
# ---------------------------------------------------------------------------
class AdapterLineage(BaseModel):
"""Records which adapter version produced the v2 record and from what v3 data.
Stored separately so v3 provenance is never lost.
"""
adapter_version: str = "1.0.0"
pipeline_version: str = "3.0.0"
v3_document_id: str
v2_intelligence_id: str
stage_runs: list[V3StageRun] = Field(default_factory=list)
mapped_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc))
mapping_notes: list[str] = Field(default_factory=list)