feat: Intelligence Pipeline v3 — full implementation

Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
@@ -0,0 +1,153 @@
+"""Classification of entity mentions as explicit, inferred, or unresolved.
+
+This module provides the logic to determine whether a company mention in a
+document is:
+- Explicit: the company name, ticker, or known alias appears directly in text
+- Inferred: the company relationship is derived from context (competitor,
+  supplier, sector peer) rather than a direct textual reference
+- Unresolved: no match in the registry — preserved as literal text
+
+The classifier operates on already-resolved candidates from the SymbolResolver,
+using document context and relationship signals to make the determination.
+"""
+
+from __future__ import annotations
+
+import re
+from enum import Enum
+
+from services.intelligence_pipeline_v3.resolution.models import (
+    MentionType,
+    ResolutionCandidate,
+)
+
+# Relationship keywords that suggest inferred exposure rather than direct mention.
+_INFERRED_KEYWORDS = re.compile(
+    r"\b("
+    r"competitor|competitors|rival|rivals|"
+    r"supplier|suppliers|vendor|vendors|"
+    r"customer|customers|client|clients|"
+    r"partner|partners|peer|peers|"
+    r"sector\s+peer|industry\s+peer|"
+    r"supply\s+chain|downstream|upstream|"
+    r"exposed\s+to|exposure|"
+    r"indirectly|second[- ]order|knock[- ]on"
+    r")\b",
+    re.IGNORECASE,
+)
+
+# Direct mention keywords that confirm explicit reference.
+_EXPLICIT_KEYWORDS = re.compile(
+    r"\b("
+    r"announced|reported|said|stated|disclosed|"
+    r"according\s+to|shares\s+of|stock\s+of|"
+    r"CEO\s+of|CFO\s+of|spokesperson\s+for"
+    r")\b",
+    re.IGNORECASE,
+)
+
+
+class ClassifiedMentionType(str, Enum):
+    """Extended mention classification with unresolved state.
+
+    This enum adds `unresolved` to the base MentionType for full classification
+    including cases where no registry match exists.
+    """
+
+    explicit_mention = "explicit_mention"
+    inferred_exposure = "inferred_exposure"
+    unresolved = "unresolved"
+
+
+def classify_mention(
+    mention: str,
+    document_context: str,
+    resolved_candidates: list[ResolutionCandidate],
+) -> ClassifiedMentionType:
+    """Classify a mention as explicit, inferred, or unresolved.
+
+    Decision logic:
+    1. If no candidates resolved → unresolved
+    2. If the mention text (ticker/name/alias) appears directly in the context
+       without surrounding inferred-relationship keywords → explicit
+    3. If surrounding context contains relationship/exposure keywords
+       (competitor, supplier, peer, etc.) → inferred
+    4. Default to explicit if the mention resolves to a candidate (direct
+       textual match in the alias index implies explicit reference)
+
+    Args:
+        mention: The original text that was resolved (or attempted).
+        document_context: Surrounding text from the document for context analysis.
+        resolved_candidates: Candidates returned by the SymbolResolver.
+
+    Returns:
+        ClassifiedMentionType indicating the nature of the mention.
+    """
+    # No candidates → unresolved.
+    if not resolved_candidates:
+        return ClassifiedMentionType.unresolved
+
+    # Check if inferred-relationship keywords are near the mention in context.
+    if document_context and _has_inferred_context(mention, document_context):
+        return ClassifiedMentionType.inferred_exposure
+
+    # The mention resolved via the alias index (ticker, name, or alias match),
+    # which means the text itself references the company directly.
+    return ClassifiedMentionType.explicit_mention
+
+
+def _has_inferred_context(mention: str, context: str) -> bool:
+    """Check if the surrounding context suggests an inferred relationship.
+
+    Looks for relationship keywords near the mention text. A mention is
+    considered inferred if:
+    - The context contains inferred-relationship keywords AND
+    - The context does NOT contain explicit attribution keywords directly
+      tied to the mention (e.g., "Apple announced" vs "Apple's competitor")
+    """
+    mention_lower = mention.lower()
+
+    # Find the mention position(s) in context.
+    context_lower = context.lower()
+    mention_pos = context_lower.find(mention_lower)
+
+    if mention_pos == -1:
+        # Mention not found in context — can't determine from context.
+        # Default to not-inferred (let the alias match speak for itself).
+        return False
+
+    # Extract a window around the mention (±100 chars).
+    window_start = max(0, mention_pos - 100)
+    window_end = min(len(context), mention_pos + len(mention) + 100)
+    window = context[window_start:window_end]
+
+    # Check for inferred keywords in the window.
+    has_inferred = bool(_INFERRED_KEYWORDS.search(window))
+    if not has_inferred:
+        return False
+
+    # Check for explicit attribution keywords in the same window.
+    has_explicit = bool(_EXPLICIT_KEYWORDS.search(window))
+
+    # If both are present, prefer explicit (the mention is directly referenced
+    # even if relationship words appear nearby).
+    if has_explicit:
+        return False
+
+    return True
+
+
+def to_mention_type(classified: ClassifiedMentionType) -> MentionType:
+    """Convert a ClassifiedMentionType to the base MentionType enum.
+
+    Maps:
+        explicit_mention → MentionType.explicit
+        inferred_exposure → MentionType.inferred
+        unresolved → MentionType.explicit (preserved as-is, no match)
+
+    This is used when interfacing with the core resolver which uses the
+    simpler two-value MentionType enum.
+    """
+    if classified == ClassifiedMentionType.inferred_exposure:
+        return MentionType.inferred
+    return MentionType.explicit