Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
163 lines
5.8 KiB
Python
163 lines
5.8 KiB
Python
"""Input/output models for the v3→v2 compatibility adapter.
|
|
|
|
V3IntelligenceRecord represents the full v3 pipeline output.
|
|
V2IntelligenceRecord / V2ImpactRecord match the current document_intelligence
|
|
and document_impact_records database schemas.
|
|
AdapterLineage captures version and stage provenance.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from typing import Literal
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# V3 Pipeline Output (input to adapter)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class V3SentimentDistribution(BaseModel):
|
|
"""Per-company calibrated sentiment probabilities."""
|
|
|
|
positive: float = Field(ge=0.0, le=1.0)
|
|
negative: float = Field(ge=0.0, le=1.0)
|
|
neutral: float = Field(ge=0.0, le=1.0)
|
|
|
|
|
|
class V3HorizonProbabilities(BaseModel):
|
|
"""Probability distribution over impact horizons."""
|
|
|
|
intraday: float = Field(ge=0.0, le=1.0, default=0.0)
|
|
one_day: float = Field(ge=0.0, le=1.0, default=0.0)
|
|
seven_day: float = Field(ge=0.0, le=1.0, default=0.0)
|
|
thirty_day: float = Field(ge=0.0, le=1.0, default=0.0)
|
|
ninety_day: float = Field(ge=0.0, le=1.0, default=0.0)
|
|
|
|
|
|
class V3DirectionProbabilities(BaseModel):
|
|
"""Probability distribution over market direction."""
|
|
|
|
positive: float = Field(ge=0.0, le=1.0, default=0.0)
|
|
negative: float = Field(ge=0.0, le=1.0, default=0.0)
|
|
neutral: float = Field(ge=0.0, le=1.0, default=0.0)
|
|
|
|
|
|
class V3CompanySignal(BaseModel):
|
|
"""A single company's signal from the v3 pipeline."""
|
|
|
|
company_id: str
|
|
ticker: str
|
|
relevance_probability: float = Field(ge=0.0, le=1.0)
|
|
event_classes: list[str] = Field(default_factory=list)
|
|
sentiment: V3SentimentDistribution
|
|
direction_probabilities: V3DirectionProbabilities
|
|
horizon_probabilities: V3HorizonProbabilities
|
|
expected_magnitude: float | None = None
|
|
evidence_spans: list[str] = Field(default_factory=list)
|
|
adjudicated: bool = False
|
|
|
|
|
|
class V3StageRun(BaseModel):
|
|
"""Lineage for a single pipeline stage execution."""
|
|
|
|
stage: str
|
|
endpoint_id: str | None = None
|
|
deployment_id: str | None = None
|
|
model_version: str | None = None
|
|
schema_version: str = "1.0.0"
|
|
calibration_version: str | None = None
|
|
started_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc))
|
|
duration_ms: int = 0
|
|
status: str = "completed"
|
|
|
|
|
|
class V3IntelligenceRecord(BaseModel):
|
|
"""Complete v3 pipeline output for a single document.
|
|
|
|
This is the adapter's input — the full v3 record with probabilities,
|
|
evidence, and stage lineage.
|
|
"""
|
|
|
|
document_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
|
document_type: str = "article"
|
|
summary: str = ""
|
|
macro_themes: list[str] = Field(default_factory=list)
|
|
novelty_score: float = Field(ge=0.0, le=1.0, default=0.5)
|
|
confidence: float = Field(ge=0.0, le=1.0, default=0.5)
|
|
company_signals: list[V3CompanySignal] = Field(default_factory=list)
|
|
stage_runs: list[V3StageRun] = Field(default_factory=list)
|
|
pipeline_version: str = "3.0.0"
|
|
created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# V2 Output (adapter output — matches current DB schema)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class V2ImpactRecord(BaseModel):
|
|
"""Maps to document_impact_records table.
|
|
|
|
Fields match the columns: relevance, sentiment (enum string),
|
|
impact_score (float), impact_horizon (string), catalyst_type,
|
|
key_facts, risks, evidence_spans.
|
|
"""
|
|
|
|
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
|
company_id: str
|
|
ticker: str
|
|
relevance: float = Field(ge=0.0, le=1.0)
|
|
sentiment: Literal["positive", "negative", "neutral", "mixed"]
|
|
impact_score: float = Field(ge=-1.0, le=1.0)
|
|
impact_horizon: Literal["intraday", "1d", "7d", "30d", "90d"]
|
|
catalyst_type: str = "other"
|
|
key_facts: list[str] = Field(default_factory=list)
|
|
risks: list[str] = Field(default_factory=list)
|
|
evidence_spans: list[str] = Field(default_factory=list)
|
|
|
|
|
|
class V2IntelligenceRecord(BaseModel):
|
|
"""Maps to document_intelligence table.
|
|
|
|
Fields match columns: summary, macro_themes, novelty_score,
|
|
source_credibility, confidence, model_provider, model_name,
|
|
prompt_version, schema_version, plus associated impact records.
|
|
"""
|
|
|
|
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
|
document_id: str
|
|
summary: str = ""
|
|
macro_themes: list[str] = Field(default_factory=list)
|
|
novelty_score: float = Field(ge=0.0, le=1.0)
|
|
source_credibility: float = Field(ge=0.0, le=1.0, default=0.5)
|
|
confidence: float = Field(ge=0.0, le=1.0)
|
|
model_provider: str = "hybrid"
|
|
model_name: str = "intelligence-pipeline-v3"
|
|
prompt_version: str = ""
|
|
schema_version: str = "3.0.0"
|
|
impact_records: list[V2ImpactRecord] = Field(default_factory=list)
|
|
created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Adapter Lineage
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class AdapterLineage(BaseModel):
|
|
"""Records which adapter version produced the v2 record and from what v3 data.
|
|
|
|
Stored separately so v3 provenance is never lost.
|
|
"""
|
|
|
|
adapter_version: str = "1.0.0"
|
|
pipeline_version: str = "3.0.0"
|
|
v3_document_id: str
|
|
v2_intelligence_id: str
|
|
stage_runs: list[V3StageRun] = Field(default_factory=list)
|
|
mapped_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc))
|
|
mapping_notes: list[str] = Field(default_factory=list)
|