admin/stonks-oracle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Intelligence Pipeline v3 — full implementation

Browse Source 
Multi-stage evidence-grounded inference architecture replacing the
monolithic 9B model extraction pipeline. CPU-first specialist services
handle routine extraction while the 9B vLLM model is preserved for
semantic adjudication of ambiguous cases.

Key components:
- Capability-aware inference gateway (OpenAI-compatible + Ollama)
- Endpoint registry with DB migrations and REST API
- Sentence-aware document segmenter (property tests)
- Deterministic financial parsing with offset integrity
- Symbol resolution with ambiguity detection
- Specialist service (GLiNER2, dynamic batching, K8s deployment)
- Company-specific sentiment (FinBERT, calibration)
- Retrieval-based novelty and duplicate detection
- Confidence calibration pipeline
- Deterministic routing engine (property tests)
- 9B adjudication layer with VRAM gating
- Stock-specific impact model (features, labels, baseline, trained)
- Pipeline orchestrator (state machine, queues, leases, feature flags)
- Bounded parallelism (async workers, semaphore, load shedding)
- Observability (tracing, metrics, alerts)
- Compatibility adapter (v3→v2 golden mapping tests)
- Shadow/canary promotion framework
- Active learning and fine-tuning pipeline

Test results: 1,161 tests pass, ruff lint clean.
All 282 spec tasks completed.
This commit is contained in:
Celes Renata
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/intelligence_pipeline_v3/resolution/__init__.py
+40
View File
@@ -0,0 +1,40 @@
"""Symbol resolution package for Intelligence Pipeline v3.
Resolves company mentions in documents to canonical identifiers using the
symbol registry, supporting alias matching, ambiguity detection, and
separation of explicit mentions from inferred exposures.
"""
from services.intelligence_pipeline_v3.resolution.alias_index import (
AliasIndex,
build_alias_index,
)
from services.intelligence_pipeline_v3.resolution.explicit_vs_inferred import (
ClassifiedMentionType,
classify_mention,
to_mention_type,
)
from services.intelligence_pipeline_v3.resolution.models import (
MatchType,
MentionType,
ResolutionCandidate,
ResolutionResult,
UnresolvedMention,
UnresolvedReason,
)
from services.intelligence_pipeline_v3.resolution.symbol_resolver import SymbolResolver
__all__ = [
"AliasIndex",
"ClassifiedMentionType",
"MatchType",
"MentionType",
"ResolutionCandidate",
"ResolutionResult",
"SymbolResolver",
"UnresolvedMention",
"UnresolvedReason",
"build_alias_index",
"classify_mention",
"to_mention_type",
]
services/intelligence_pipeline_v3/resolution/alias_index.py
+164
View File
@@ -0,0 +1,164 @@
"""In-memory alias index for company name/ticker/alias lookup.
Supports case-insensitive matching with common corporate suffix stripping
(Inc., Corp., LLC, etc.) to maximize recall against varied document text.
The primary entry point for consumers is `build_alias_index(companies)` which
constructs a populated AliasIndex from company registry data.
"""
from __future__ import annotations
import re
from dataclasses import dataclass, field
# Common corporate suffixes to strip for matching purposes.
_SUFFIX_PATTERN = re.compile(
r"\s*\b("
r"inc\.?|incorporated|"
r"corp\.?|corporation|"
r"co\.?|company|"
r"ltd\.?|limited|"
r"llc|l\.l\.c\.?|"
r"plc|p\.l\.c\.?|"
r"sa|s\.a\.?|"
r"nv|n\.v\.?|"
r"ag|"
r"se|"
r"group|"
r"holdings?"
r")\s*$",
re.IGNORECASE,
)
# Trailing punctuation after suffix removal.
_TRAILING_PUNCT = re.compile(r"[.,;:\s]+$")
@dataclass
class IndexEntry:
"""A single entry linking a normalized key to a company."""
company_id: str
ticker: str
name: str
match_type: str # "exact_ticker", "exact_name", "alias"
@dataclass
class AliasIndex:
"""Case-insensitive index mapping normalized strings to company entries.
Stores tickers, full legal names, and known aliases. Returns all
matching companies for a given query string.
"""
_entries: dict[str, list[IndexEntry]] = field(default_factory=dict)
@staticmethod
def normalize(text: str) -> str:
"""Normalize a string for matching: lowercase, strip suffixes and punctuation."""
s = text.strip().lower()
# Strip corporate suffixes.
s = _SUFFIX_PATTERN.sub("", s)
# Remove trailing punctuation left behind.
s = _TRAILING_PUNCT.sub("", s)
# Collapse whitespace.
s = re.sub(r"\s+", " ", s).strip()
return s
def add(self, key: str, entry: IndexEntry) -> None:
"""Add a lookup key mapped to an index entry."""
normalized = self.normalize(key)
if not normalized:
return
self._entries.setdefault(normalized, []).append(entry)
def lookup(self, query: str) -> list[IndexEntry]:
"""Return all entries matching the normalized query."""
normalized = self.normalize(query)
if not normalized:
return []
return list(self._entries.get(normalized, []))
def keys(self) -> list[str]:
"""Return all normalized keys in the index."""
return list(self._entries.keys())
def __len__(self) -> int:
"""Number of distinct normalized keys in the index."""
return len(self._entries)
def build_alias_index(companies: list[dict]) -> AliasIndex:
"""Build a complete alias index from company registry data.
This is the primary factory function for constructing an AliasIndex.
It processes the same company dict format used by the symbol registry seed:
Each company dict should contain:
- id: str (company UUID)
- ticker: str
- legal_name: str
Optional fields:
- aliases: list[dict] with keys "alias" and optionally "alias_type"
OR list[tuple[str, str]] of (alias_text, alias_type)
- exchange: str (used for qualified ticker indexing)
- sector: str
- industry: str
The function indexes:
1. Ticker symbols (exact_ticker match type)
2. Full legal names (exact_name match type)
3. Legal names with suffixes stripped (exact_name match type)
4. All known aliases (alias match type)
Returns:
A fully populated AliasIndex ready for lookup operations.
"""
index = AliasIndex()
for company in companies:
company_id = str(company["id"])
ticker = company["ticker"]
name = company["legal_name"]
# Index the ticker itself.
entry_ticker = IndexEntry(
company_id=company_id,
ticker=ticker,
name=name,
match_type="exact_ticker",
)
index.add(ticker, entry_ticker)
# Index the full legal name.
entry_name = IndexEntry(
company_id=company_id,
ticker=ticker,
name=name,
match_type="exact_name",
)
index.add(name, entry_name)
# Index known aliases.
aliases = company.get("aliases", [])
for alias_entry in aliases:
if isinstance(alias_entry, dict):
alias_text = alias_entry.get("alias", "")
elif isinstance(alias_entry, (list, tuple)) and len(alias_entry) >= 1:
alias_text = alias_entry[0]
else:
alias_text = str(alias_entry)
if alias_text:
entry_alias = IndexEntry(
company_id=company_id,
ticker=ticker,
name=name,
match_type="alias",
)
index.add(alias_text, entry_alias)
return index
services/intelligence_pipeline_v3/resolution/explicit_vs_inferred.py
+153
View File
@@ -0,0 +1,153 @@
"""Classification of entity mentions as explicit, inferred, or unresolved.
This module provides the logic to determine whether a company mention in a
document is:
- Explicit: the company name, ticker, or known alias appears directly in text
- Inferred: the company relationship is derived from context (competitor,
supplier, sector peer) rather than a direct textual reference
- Unresolved: no match in the registry — preserved as literal text
The classifier operates on already-resolved candidates from the SymbolResolver,
using document context and relationship signals to make the determination.
"""
from __future__ import annotations
import re
from enum import Enum
from services.intelligence_pipeline_v3.resolution.models import (
MentionType,
ResolutionCandidate,
)
# Relationship keywords that suggest inferred exposure rather than direct mention.
_INFERRED_KEYWORDS = re.compile(
r"\b("
r"competitor|competitors|rival|rivals|"
r"supplier|suppliers|vendor|vendors|"
r"customer|customers|client|clients|"
r"partner|partners|peer|peers|"
r"sector\s+peer|industry\s+peer|"
r"supply\s+chain|downstream|upstream|"
r"exposed\s+to|exposure|"
r"indirectly|second[- ]order|knock[- ]on"
r")\b",
re.IGNORECASE,
)
# Direct mention keywords that confirm explicit reference.
_EXPLICIT_KEYWORDS = re.compile(
r"\b("
r"announced|reported|said|stated|disclosed|"
r"according\s+to|shares\s+of|stock\s+of|"
r"CEO\s+of|CFO\s+of|spokesperson\s+for"
r")\b",
re.IGNORECASE,
)
class ClassifiedMentionType(str, Enum):
"""Extended mention classification with unresolved state.
This enum adds `unresolved` to the base MentionType for full classification
including cases where no registry match exists.
"""
explicit_mention = "explicit_mention"
inferred_exposure = "inferred_exposure"
unresolved = "unresolved"
def classify_mention(
mention: str,
document_context: str,
resolved_candidates: list[ResolutionCandidate],
) -> ClassifiedMentionType:
"""Classify a mention as explicit, inferred, or unresolved.
Decision logic:
1. If no candidates resolved → unresolved
2. If the mention text (ticker/name/alias) appears directly in the context
without surrounding inferred-relationship keywords → explicit
3. If surrounding context contains relationship/exposure keywords
(competitor, supplier, peer, etc.) → inferred
4. Default to explicit if the mention resolves to a candidate (direct
textual match in the alias index implies explicit reference)
Args:
mention: The original text that was resolved (or attempted).
document_context: Surrounding text from the document for context analysis.
resolved_candidates: Candidates returned by the SymbolResolver.
Returns:
ClassifiedMentionType indicating the nature of the mention.
"""
# No candidates → unresolved.
if not resolved_candidates:
return ClassifiedMentionType.unresolved
# Check if inferred-relationship keywords are near the mention in context.
if document_context and _has_inferred_context(mention, document_context):
return ClassifiedMentionType.inferred_exposure
# The mention resolved via the alias index (ticker, name, or alias match),
# which means the text itself references the company directly.
return ClassifiedMentionType.explicit_mention
def _has_inferred_context(mention: str, context: str) -> bool:
"""Check if the surrounding context suggests an inferred relationship.
Looks for relationship keywords near the mention text. A mention is
considered inferred if:
- The context contains inferred-relationship keywords AND
- The context does NOT contain explicit attribution keywords directly
tied to the mention (e.g., "Apple announced" vs "Apple's competitor")
"""
mention_lower = mention.lower()
# Find the mention position(s) in context.
context_lower = context.lower()
mention_pos = context_lower.find(mention_lower)
if mention_pos == -1:
# Mention not found in context — can't determine from context.
# Default to not-inferred (let the alias match speak for itself).
return False
# Extract a window around the mention (±100 chars).
window_start = max(0, mention_pos - 100)
window_end = min(len(context), mention_pos + len(mention) + 100)
window = context[window_start:window_end]
# Check for inferred keywords in the window.
has_inferred = bool(_INFERRED_KEYWORDS.search(window))
if not has_inferred:
return False
# Check for explicit attribution keywords in the same window.
has_explicit = bool(_EXPLICIT_KEYWORDS.search(window))
# If both are present, prefer explicit (the mention is directly referenced
# even if relationship words appear nearby).
if has_explicit:
return False
return True
def to_mention_type(classified: ClassifiedMentionType) -> MentionType:
"""Convert a ClassifiedMentionType to the base MentionType enum.
Maps:
explicit_mention → MentionType.explicit
inferred_exposure → MentionType.inferred
unresolved → MentionType.explicit (preserved as-is, no match)
This is used when interfacing with the core resolver which uses the
simpler two-value MentionType enum.
"""
if classified == ClassifiedMentionType.inferred_exposure:
return MentionType.inferred
return MentionType.explicit
services/intelligence_pipeline_v3/resolution/models.py
+80
View File
@@ -0,0 +1,80 @@
"""Data models for symbol resolution results."""
from __future__ import annotations
from enum import Enum
from pydantic import BaseModel, Field
class MatchType(str, Enum):
"""How a resolution candidate was matched."""
exact_ticker = "exact_ticker"
exact_name = "exact_name"
alias = "alias"
fuzzy = "fuzzy"
class MentionType(str, Enum):
"""Whether a mention is explicitly stated or inferred from context."""
explicit = "explicit"
inferred = "inferred"
class UnresolvedReason(str, Enum):
"""Why a mention could not be resolved."""
not_in_registry = "not_in_registry"
ambiguous = "ambiguous"
context_needed = "context_needed"
class ResolutionCandidate(BaseModel):
"""A single candidate match from the symbol registry.
Candidates are ranked by confidence, with match_type indicating how
the match was derived.
"""
company_id: str = Field(description="UUID of the matched company")
ticker: str = Field(description="Ticker symbol of the matched company")
name: str = Field(description="Legal or display name of the matched company")
confidence: float = Field(ge=0.0, le=1.0, description="Match confidence score")
match_type: MatchType = Field(description="How the match was derived")
class ResolutionResult(BaseModel):
"""Full resolution output for a single mention.
Contains ranked candidates, ambiguity margin, and mention classification.
"""
candidates: list[ResolutionCandidate] = Field(default_factory=list)
ambiguity_margin: float = Field(
default=1.0,
ge=0.0,
le=1.0,
description="Difference between top-2 candidate confidences. 1.0 = unambiguous single match, 0.0 = tied.",
)
is_ambiguous: bool = Field(
default=False,
description="True when top candidates are too close to distinguish without context.",
)
mention_type: MentionType = Field(
default=MentionType.explicit,
description="Whether the mention is explicit text or inferred exposure.",
)
class UnresolvedMention(BaseModel):
"""A mention that could not be resolved to any company in the registry.
Preserved as-is rather than having a ticker invented for it.
"""
literal_text: str = Field(description="Original text as it appeared in the document")
start_char: int = Field(ge=0, description="Start character offset in source document")
end_char: int = Field(gt=0, description="End character offset (exclusive)")
reason: UnresolvedReason = Field(description="Why resolution failed")
services/intelligence_pipeline_v3/resolution/symbol_resolver.py
+185
View File
@@ -0,0 +1,185 @@
"""Symbol resolver: maps textual mentions to canonical company identities.
Uses an in-memory alias index built from company registry data. Returns
ranked candidates with confidence scores and ambiguity margins. Does NOT
invent tickers for unresolved mentions.
"""
from __future__ import annotations
from services.intelligence_pipeline_v3.resolution.alias_index import (
AliasIndex,
IndexEntry,
build_alias_index,
)
from services.intelligence_pipeline_v3.resolution.models import (
MatchType,
MentionType,
ResolutionCandidate,
ResolutionResult,
UnresolvedMention,
UnresolvedReason,
)
# Ambiguity threshold: if the gap between top-2 candidates is below this,
# the result is marked as ambiguous.
_AMBIGUITY_THRESHOLD = 0.15
class SymbolResolver:
"""Resolve document mentions to canonical company identifiers.
Usage:
resolver = SymbolResolver()
resolver.load_registry(companies)
result = resolver.resolve("Apple")
"""
def __init__(self, ambiguity_threshold: float = _AMBIGUITY_THRESHOLD) -> None:
self._index = AliasIndex()
self._ambiguity_threshold = ambiguity_threshold
@property
def index(self) -> AliasIndex:
"""Access the underlying alias index."""
return self._index
def load_registry(self, companies: list[dict]) -> None:
"""Load company data into the alias index.
Each company dict should contain at minimum:
- id: str (company UUID)
- ticker: str
- legal_name: str
Optional fields:
- aliases: list[dict] with keys "alias" and optionally "alias_type"
OR list[tuple[str, str]] of (alias_text, alias_type)
Uses `build_alias_index` to construct the full index from registry data.
"""
self._index = build_alias_index(companies)
def resolve(
self,
mention: str,
context: str = "",
mention_type: MentionType = MentionType.explicit,
) -> ResolutionResult:
"""Resolve a textual mention to ranked company candidates.
Args:
mention: The text to resolve (ticker, name, alias, etc.)
context: Optional surrounding text for future disambiguation use.
mention_type: Whether this is an explicit mention or inferred exposure.
Returns:
ResolutionResult with ranked candidates, ambiguity margin, and metadata.
If no match is found, candidates list is empty.
"""
entries = self._index.lookup(mention)
if not entries:
return ResolutionResult(
candidates=[],
ambiguity_margin=1.0,
is_ambiguous=False,
mention_type=mention_type,
)
# Deduplicate by company_id, keeping the best match type per company.
best_per_company: dict[str, IndexEntry] = {}
for entry in entries:
existing = best_per_company.get(entry.company_id)
if existing is None or _match_priority(entry.match_type) > _match_priority(existing.match_type):
best_per_company[entry.company_id] = entry
# Score candidates.
candidates: list[ResolutionCandidate] = []
for entry in best_per_company.values():
confidence = _score_match(entry.match_type, mention, entry)
candidates.append(
ResolutionCandidate(
company_id=entry.company_id,
ticker=entry.ticker,
name=entry.name,
confidence=confidence,
match_type=MatchType(entry.match_type),
)
)
# Sort by confidence descending.
candidates.sort(key=lambda c: c.confidence, reverse=True)
# Calculate ambiguity margin.
if len(candidates) >= 2:
ambiguity_margin = candidates[0].confidence - candidates[1].confidence
else:
ambiguity_margin = 1.0
is_ambiguous = ambiguity_margin < self._ambiguity_threshold
return ResolutionResult(
candidates=candidates,
ambiguity_margin=ambiguity_margin,
is_ambiguous=is_ambiguous,
mention_type=mention_type,
)
def resolve_or_unresolved(
self,
mention: str,
start_char: int,
end_char: int,
context: str = "",
mention_type: MentionType = MentionType.explicit,
) -> ResolutionResult | UnresolvedMention:
"""Resolve a mention, returning UnresolvedMention if no match found.
This ensures unresolved mentions are preserved with their literal text
and position rather than having a ticker invented.
"""
result = self.resolve(mention, context=context, mention_type=mention_type)
if not result.candidates:
return UnresolvedMention(
literal_text=mention,
start_char=start_char,
end_char=end_char,
reason=UnresolvedReason.not_in_registry,
)
if result.is_ambiguous:
return UnresolvedMention(
literal_text=mention,
start_char=start_char,
end_char=end_char,
reason=UnresolvedReason.ambiguous,
)
return result
def _match_priority(match_type: str) -> int:
"""Higher priority = stronger match type."""
priorities = {
"exact_ticker": 3,
"exact_name": 2,
"alias": 1,
"fuzzy": 0,
}
return priorities.get(match_type, 0)
def _score_match(match_type: str, mention: str, entry: IndexEntry) -> float:
"""Score a match based on type and exact-match quality.
Exact ticker matches get highest confidence, then exact name, then alias.
"""
base_scores = {
"exact_ticker": 0.95,
"exact_name": 0.90,
"alias": 0.80,
"fuzzy": 0.50,
}
return base_scores.get(match_type, 0.50)