Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
154 lines
5.4 KiB
Python
154 lines
5.4 KiB
Python
"""Classification of entity mentions as explicit, inferred, or unresolved.
|
|
|
|
This module provides the logic to determine whether a company mention in a
|
|
document is:
|
|
- Explicit: the company name, ticker, or known alias appears directly in text
|
|
- Inferred: the company relationship is derived from context (competitor,
|
|
supplier, sector peer) rather than a direct textual reference
|
|
- Unresolved: no match in the registry — preserved as literal text
|
|
|
|
The classifier operates on already-resolved candidates from the SymbolResolver,
|
|
using document context and relationship signals to make the determination.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from enum import Enum
|
|
|
|
from services.intelligence_pipeline_v3.resolution.models import (
|
|
MentionType,
|
|
ResolutionCandidate,
|
|
)
|
|
|
|
# Relationship keywords that suggest inferred exposure rather than direct mention.
|
|
_INFERRED_KEYWORDS = re.compile(
|
|
r"\b("
|
|
r"competitor|competitors|rival|rivals|"
|
|
r"supplier|suppliers|vendor|vendors|"
|
|
r"customer|customers|client|clients|"
|
|
r"partner|partners|peer|peers|"
|
|
r"sector\s+peer|industry\s+peer|"
|
|
r"supply\s+chain|downstream|upstream|"
|
|
r"exposed\s+to|exposure|"
|
|
r"indirectly|second[- ]order|knock[- ]on"
|
|
r")\b",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
# Direct mention keywords that confirm explicit reference.
|
|
_EXPLICIT_KEYWORDS = re.compile(
|
|
r"\b("
|
|
r"announced|reported|said|stated|disclosed|"
|
|
r"according\s+to|shares\s+of|stock\s+of|"
|
|
r"CEO\s+of|CFO\s+of|spokesperson\s+for"
|
|
r")\b",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
class ClassifiedMentionType(str, Enum):
|
|
"""Extended mention classification with unresolved state.
|
|
|
|
This enum adds `unresolved` to the base MentionType for full classification
|
|
including cases where no registry match exists.
|
|
"""
|
|
|
|
explicit_mention = "explicit_mention"
|
|
inferred_exposure = "inferred_exposure"
|
|
unresolved = "unresolved"
|
|
|
|
|
|
def classify_mention(
|
|
mention: str,
|
|
document_context: str,
|
|
resolved_candidates: list[ResolutionCandidate],
|
|
) -> ClassifiedMentionType:
|
|
"""Classify a mention as explicit, inferred, or unresolved.
|
|
|
|
Decision logic:
|
|
1. If no candidates resolved → unresolved
|
|
2. If the mention text (ticker/name/alias) appears directly in the context
|
|
without surrounding inferred-relationship keywords → explicit
|
|
3. If surrounding context contains relationship/exposure keywords
|
|
(competitor, supplier, peer, etc.) → inferred
|
|
4. Default to explicit if the mention resolves to a candidate (direct
|
|
textual match in the alias index implies explicit reference)
|
|
|
|
Args:
|
|
mention: The original text that was resolved (or attempted).
|
|
document_context: Surrounding text from the document for context analysis.
|
|
resolved_candidates: Candidates returned by the SymbolResolver.
|
|
|
|
Returns:
|
|
ClassifiedMentionType indicating the nature of the mention.
|
|
"""
|
|
# No candidates → unresolved.
|
|
if not resolved_candidates:
|
|
return ClassifiedMentionType.unresolved
|
|
|
|
# Check if inferred-relationship keywords are near the mention in context.
|
|
if document_context and _has_inferred_context(mention, document_context):
|
|
return ClassifiedMentionType.inferred_exposure
|
|
|
|
# The mention resolved via the alias index (ticker, name, or alias match),
|
|
# which means the text itself references the company directly.
|
|
return ClassifiedMentionType.explicit_mention
|
|
|
|
|
|
def _has_inferred_context(mention: str, context: str) -> bool:
|
|
"""Check if the surrounding context suggests an inferred relationship.
|
|
|
|
Looks for relationship keywords near the mention text. A mention is
|
|
considered inferred if:
|
|
- The context contains inferred-relationship keywords AND
|
|
- The context does NOT contain explicit attribution keywords directly
|
|
tied to the mention (e.g., "Apple announced" vs "Apple's competitor")
|
|
"""
|
|
mention_lower = mention.lower()
|
|
|
|
# Find the mention position(s) in context.
|
|
context_lower = context.lower()
|
|
mention_pos = context_lower.find(mention_lower)
|
|
|
|
if mention_pos == -1:
|
|
# Mention not found in context — can't determine from context.
|
|
# Default to not-inferred (let the alias match speak for itself).
|
|
return False
|
|
|
|
# Extract a window around the mention (±100 chars).
|
|
window_start = max(0, mention_pos - 100)
|
|
window_end = min(len(context), mention_pos + len(mention) + 100)
|
|
window = context[window_start:window_end]
|
|
|
|
# Check for inferred keywords in the window.
|
|
has_inferred = bool(_INFERRED_KEYWORDS.search(window))
|
|
if not has_inferred:
|
|
return False
|
|
|
|
# Check for explicit attribution keywords in the same window.
|
|
has_explicit = bool(_EXPLICIT_KEYWORDS.search(window))
|
|
|
|
# If both are present, prefer explicit (the mention is directly referenced
|
|
# even if relationship words appear nearby).
|
|
if has_explicit:
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def to_mention_type(classified: ClassifiedMentionType) -> MentionType:
|
|
"""Convert a ClassifiedMentionType to the base MentionType enum.
|
|
|
|
Maps:
|
|
explicit_mention → MentionType.explicit
|
|
inferred_exposure → MentionType.inferred
|
|
unresolved → MentionType.explicit (preserved as-is, no match)
|
|
|
|
This is used when interfacing with the core resolver which uses the
|
|
simpler two-value MentionType enum.
|
|
"""
|
|
if classified == ClassifiedMentionType.inferred_exposure:
|
|
return MentionType.inferred
|
|
return MentionType.explicit
|