feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,38 @@
|
||||
"""Company-specific financial sentiment analysis.
|
||||
|
||||
This package provides FinBERT-based sentiment classification
|
||||
on company-linked evidence groups, producing per-company probability
|
||||
distributions with full evidence provenance.
|
||||
"""
|
||||
|
||||
from services.intelligence_pipeline_v3.sentiment.aggregation import (
|
||||
aggregate_evidence_sentiments,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.sentiment.calibrator import SentimentCalibrator
|
||||
from services.intelligence_pipeline_v3.sentiment.evidence_groups import build_evidence_groups
|
||||
from services.intelligence_pipeline_v3.sentiment.finbert_adapter import FinBERTAdapter
|
||||
from services.intelligence_pipeline_v3.sentiment.mixed_sentiment import compute_mixed_sentiment
|
||||
from services.intelligence_pipeline_v3.sentiment.models import (
|
||||
CompanySentimentResult,
|
||||
EvidenceGroup,
|
||||
SentimentBatchResult,
|
||||
TextSentiment,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.sentiment.sentiment_scorer import (
|
||||
SentimentModel,
|
||||
SentimentScorer,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SentimentCalibrator",
|
||||
"SentimentModel",
|
||||
"SentimentScorer",
|
||||
"CompanySentimentResult",
|
||||
"EvidenceGroup",
|
||||
"FinBERTAdapter",
|
||||
"SentimentBatchResult",
|
||||
"TextSentiment",
|
||||
"aggregate_evidence_sentiments",
|
||||
"build_evidence_groups",
|
||||
"compute_mixed_sentiment",
|
||||
]
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Sentiment aggregation logic.
|
||||
|
||||
Aggregates per-text sentiment scores into a single company result,
|
||||
with mixed sentiment detection when evidence groups disagree.
|
||||
|
||||
Mixed sentiment is NOT an unconstrained fourth softmax label — it is
|
||||
computed from disagreement between evidence texts (some positive,
|
||||
some negative with margin > threshold).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from services.intelligence_pipeline_v3.sentiment.models import (
|
||||
CompanySentimentResult,
|
||||
TextSentiment,
|
||||
)
|
||||
|
||||
# Disagreement threshold: both positive and negative probabilities
|
||||
# must be >= this value across evidence texts to signal disagreement.
|
||||
# A text with positive_prob >= threshold AND another text with
|
||||
# negative_prob >= threshold indicates conflicting evidence.
|
||||
MIXED_DISAGREEMENT_THRESHOLD = 0.3
|
||||
|
||||
|
||||
def aggregate_evidence_sentiments(
|
||||
company_id: str,
|
||||
per_text_scores: list[TextSentiment],
|
||||
model_version: str,
|
||||
calibration_version: str = "uncalibrated",
|
||||
) -> CompanySentimentResult:
|
||||
"""Aggregate per-text sentiment scores into a company-level result.
|
||||
|
||||
Computes weighted average probabilities and detects mixed sentiment
|
||||
from evidence-group disagreement. Mixed is triggered when at least
|
||||
one text has positive_prob >= threshold AND at least one other text
|
||||
has negative_prob >= threshold.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
company_id
|
||||
Resolved company identifier.
|
||||
per_text_scores
|
||||
List of TextSentiment objects, one per evidence text.
|
||||
model_version
|
||||
Sentiment model version string for lineage.
|
||||
calibration_version
|
||||
Calibration artifact version.
|
||||
|
||||
Returns
|
||||
-------
|
||||
CompanySentimentResult
|
||||
Aggregated result with label, probabilities, per-text scores,
|
||||
is_mixed flag, and evidence IDs.
|
||||
"""
|
||||
if not per_text_scores:
|
||||
return CompanySentimentResult(
|
||||
company_id=company_id,
|
||||
label="neutral",
|
||||
positive_prob=0.0,
|
||||
negative_prob=0.0,
|
||||
neutral_prob=1.0,
|
||||
evidence_ids=[],
|
||||
is_mixed=False,
|
||||
per_text_scores=[],
|
||||
model_version=model_version,
|
||||
calibration_version=calibration_version,
|
||||
)
|
||||
|
||||
# Detect mixed sentiment from disagreement
|
||||
is_mixed = _detect_mixed_from_disagreement(per_text_scores)
|
||||
|
||||
# Compute average probabilities across texts
|
||||
n = len(per_text_scores)
|
||||
avg_pos = sum(s.positive_prob for s in per_text_scores) / n
|
||||
avg_neg = sum(s.negative_prob for s in per_text_scores) / n
|
||||
avg_neu = sum(s.neutral_prob for s in per_text_scores) / n
|
||||
|
||||
# Normalize to ensure probabilities sum to 1.0
|
||||
total = avg_pos + avg_neg + avg_neu
|
||||
if total > 0:
|
||||
avg_pos /= total
|
||||
avg_neg /= total
|
||||
avg_neu /= total
|
||||
else:
|
||||
avg_pos = 0.0
|
||||
avg_neg = 0.0
|
||||
avg_neu = 1.0
|
||||
|
||||
# Determine label
|
||||
if is_mixed:
|
||||
label = "mixed"
|
||||
else:
|
||||
label = _argmax_label(avg_pos, avg_neg, avg_neu)
|
||||
|
||||
evidence_ids = [s.evidence_id for s in per_text_scores]
|
||||
|
||||
return CompanySentimentResult(
|
||||
company_id=company_id,
|
||||
label=label,
|
||||
positive_prob=round(avg_pos, 6),
|
||||
negative_prob=round(avg_neg, 6),
|
||||
neutral_prob=round(avg_neu, 6),
|
||||
evidence_ids=evidence_ids,
|
||||
is_mixed=is_mixed,
|
||||
per_text_scores=per_text_scores,
|
||||
model_version=model_version,
|
||||
calibration_version=calibration_version,
|
||||
)
|
||||
|
||||
|
||||
def _detect_mixed_from_disagreement(per_text_scores: list[TextSentiment]) -> bool:
|
||||
"""Detect mixed sentiment from evidence-group disagreement.
|
||||
|
||||
Returns True when at least one text has positive_prob >= threshold
|
||||
AND at least one (different) text has negative_prob >= threshold.
|
||||
This indicates conflicting evidence directions.
|
||||
|
||||
A single text cannot trigger mixed on its own (we need disagreement
|
||||
between at least 2 texts).
|
||||
"""
|
||||
if len(per_text_scores) < 2:
|
||||
return False
|
||||
|
||||
max_pos = max(s.positive_prob for s in per_text_scores)
|
||||
max_neg = max(s.negative_prob for s in per_text_scores)
|
||||
|
||||
return max_pos >= MIXED_DISAGREEMENT_THRESHOLD and max_neg >= MIXED_DISAGREEMENT_THRESHOLD
|
||||
|
||||
|
||||
def _argmax_label(pos: float, neg: float, neu: float) -> str:
|
||||
"""Return the label with the highest probability."""
|
||||
if pos >= neg and pos >= neu:
|
||||
return "positive"
|
||||
elif neg >= pos and neg >= neu:
|
||||
return "negative"
|
||||
return "neutral"
|
||||
@@ -0,0 +1,207 @@
|
||||
"""Sentiment probability calibration.
|
||||
|
||||
Applies isotonic or Platt calibration to raw FinBERT probabilities
|
||||
to produce better-calibrated confidence estimates. The calibrator
|
||||
preserves probability ordering (monotonicity for isotonic) while
|
||||
improving expected calibration error (ECE).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Literal
|
||||
|
||||
import numpy as np
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default calibration version when no artifact is loaded
|
||||
DEFAULT_CALIBRATION_VERSION = "uncalibrated"
|
||||
|
||||
|
||||
class SentimentCalibrator:
|
||||
"""Calibrates raw sentiment probabilities using isotonic or Platt scaling.
|
||||
|
||||
The calibrator fits on a held-out calibration set from the Gold_Corpus
|
||||
and transforms raw model probabilities to better-calibrated values.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
method
|
||||
Calibration method: "isotonic" or "platt".
|
||||
"""
|
||||
|
||||
def __init__(self, method: Literal["isotonic", "platt"] = "isotonic") -> None:
|
||||
self._method = method
|
||||
self._calibration_version = DEFAULT_CALIBRATION_VERSION
|
||||
self._fitted = False
|
||||
self._calibrators: list | None = None # One per class
|
||||
|
||||
@property
|
||||
def calibration_version(self) -> str:
|
||||
"""Return the current calibration artifact version."""
|
||||
return self._calibration_version
|
||||
|
||||
@property
|
||||
def is_fitted(self) -> bool:
|
||||
"""Return whether the calibrator has been fitted."""
|
||||
return self._fitted
|
||||
|
||||
@property
|
||||
def method(self) -> str:
|
||||
"""Return the calibration method."""
|
||||
return self._method
|
||||
|
||||
def fit(
|
||||
self,
|
||||
raw_probs: list[list[float]],
|
||||
true_labels: list[int],
|
||||
version: str = "v1.0",
|
||||
) -> None:
|
||||
"""Fit the calibrator on a calibration dataset.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_probs
|
||||
List of [positive, negative, neutral] probability vectors.
|
||||
true_labels
|
||||
True class labels: 0=positive, 1=negative, 2=neutral.
|
||||
version
|
||||
Version string for this calibration artifact.
|
||||
"""
|
||||
if not raw_probs or not true_labels:
|
||||
raise ValueError("raw_probs and true_labels must not be empty")
|
||||
|
||||
if len(raw_probs) != len(true_labels):
|
||||
raise ValueError("raw_probs and true_labels must have the same length")
|
||||
|
||||
raw_array = np.array(raw_probs, dtype=np.float64)
|
||||
labels_array = np.array(true_labels, dtype=np.int32)
|
||||
|
||||
n_classes = raw_array.shape[1] if raw_array.ndim > 1 else 3
|
||||
|
||||
if self._method == "isotonic":
|
||||
self._fit_isotonic(raw_array, labels_array, n_classes)
|
||||
else:
|
||||
self._fit_platt(raw_array, labels_array, n_classes)
|
||||
|
||||
self._calibration_version = version
|
||||
self._fitted = True
|
||||
logger.info(
|
||||
"Calibrator fitted: method=%s, samples=%d, version=%s",
|
||||
self._method,
|
||||
len(true_labels),
|
||||
version,
|
||||
)
|
||||
|
||||
def calibrate(self, raw_probs: list[float]) -> list[float]:
|
||||
"""Calibrate a single probability vector.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_probs
|
||||
Raw [positive, negative, neutral] probabilities.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[float]
|
||||
Calibrated probabilities that sum to 1.0 and preserve
|
||||
relative ordering within each class.
|
||||
"""
|
||||
if not self._fitted:
|
||||
# Pass through uncalibrated
|
||||
return list(raw_probs)
|
||||
|
||||
calibrated = []
|
||||
for i, prob in enumerate(raw_probs):
|
||||
if self._calibrators and i < len(self._calibrators):
|
||||
cal = self._calibrators[i]
|
||||
cal_prob = float(cal.predict(np.array([[prob]]))[0])
|
||||
# Clamp to [0, 1]
|
||||
cal_prob = max(0.0, min(1.0, cal_prob))
|
||||
calibrated.append(cal_prob)
|
||||
else:
|
||||
calibrated.append(prob)
|
||||
|
||||
# Normalize to sum to 1.0
|
||||
total = sum(calibrated)
|
||||
if total > 0:
|
||||
calibrated = [p / total for p in calibrated]
|
||||
else:
|
||||
calibrated = [1.0 / len(calibrated)] * len(calibrated)
|
||||
|
||||
return calibrated
|
||||
|
||||
def calibrate_batch(self, raw_probs_batch: list[list[float]]) -> list[list[float]]:
|
||||
"""Calibrate a batch of probability vectors.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_probs_batch
|
||||
List of raw [positive, negative, neutral] probability vectors.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[list[float]]
|
||||
Calibrated probability vectors.
|
||||
"""
|
||||
return [self.calibrate(probs) for probs in raw_probs_batch]
|
||||
|
||||
def _fit_isotonic(
|
||||
self,
|
||||
raw_array: np.ndarray,
|
||||
labels_array: np.ndarray,
|
||||
n_classes: int,
|
||||
) -> None:
|
||||
"""Fit isotonic regression calibrators per class."""
|
||||
from sklearn.isotonic import IsotonicRegression
|
||||
|
||||
self._calibrators = []
|
||||
for cls_idx in range(n_classes):
|
||||
# Binary indicator: is this the true class?
|
||||
binary_labels = (labels_array == cls_idx).astype(np.float64)
|
||||
class_probs = raw_array[:, cls_idx]
|
||||
|
||||
iso = IsotonicRegression(y_min=0.0, y_max=1.0, out_of_bounds="clip")
|
||||
iso.fit(class_probs, binary_labels)
|
||||
self._calibrators.append(iso)
|
||||
|
||||
def _fit_platt(
|
||||
self,
|
||||
raw_array: np.ndarray,
|
||||
labels_array: np.ndarray,
|
||||
n_classes: int,
|
||||
) -> None:
|
||||
"""Fit Platt (logistic) scaling calibrators per class."""
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
|
||||
self._calibrators = []
|
||||
for cls_idx in range(n_classes):
|
||||
binary_labels = (labels_array == cls_idx).astype(np.int32)
|
||||
class_probs = raw_array[:, cls_idx].reshape(-1, 1)
|
||||
|
||||
lr = LogisticRegression(solver="lbfgs", max_iter=1000)
|
||||
# Need at least 2 classes in binary labels
|
||||
if len(np.unique(binary_labels)) < 2:
|
||||
# If only one class present, use identity
|
||||
self._calibrators.append(_IdentityCalibrator())
|
||||
else:
|
||||
lr.fit(class_probs, binary_labels)
|
||||
self._calibrators.append(_PlattWrapper(lr))
|
||||
|
||||
|
||||
class _IdentityCalibrator:
|
||||
"""Pass-through calibrator when insufficient data for fitting."""
|
||||
|
||||
def predict(self, x: np.ndarray) -> np.ndarray:
|
||||
return x.ravel()
|
||||
|
||||
|
||||
class _PlattWrapper:
|
||||
"""Wrapper that extracts probability of the positive class."""
|
||||
|
||||
def __init__(self, lr) -> None: # noqa: ANN001
|
||||
self._lr = lr
|
||||
|
||||
def predict(self, x: np.ndarray) -> np.ndarray:
|
||||
return self._lr.predict_proba(x)[:, 1]
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Build company-linked evidence groups from entities and evidence spans.
|
||||
|
||||
Groups evidence spans by the company they are associated with.
|
||||
A span can belong to multiple groups if it mentions multiple companies.
|
||||
Relations can add additional evidence linkage (e.g., inferred exposure).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
|
||||
from services.intelligence_pipeline_v3.sentiment.models import EvidenceGroup
|
||||
|
||||
|
||||
class EntityLike(Protocol):
|
||||
"""Protocol for entity objects that link to companies and evidence."""
|
||||
|
||||
@property
|
||||
def company_id(self) -> str | None: ...
|
||||
|
||||
@property
|
||||
def evidence_id(self) -> str: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EvidenceSpanInput:
|
||||
"""Minimal evidence span input for grouping."""
|
||||
|
||||
id: str
|
||||
text: str
|
||||
|
||||
|
||||
def build_evidence_groups(
|
||||
entities: list[dict[str, str | None]],
|
||||
evidence_spans: dict[str, str],
|
||||
relations: list[dict[str, str | None]] | None = None,
|
||||
) -> dict[str, EvidenceGroup]:
|
||||
"""Build company-linked evidence groups from entity-company associations.
|
||||
|
||||
Groups evidence spans by the company they relate to, using both
|
||||
direct entity associations and relation-based linkages. A span
|
||||
can appear in multiple groups when it mentions multiple companies.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
entities
|
||||
List of dicts with keys: company_id (str or None), evidence_id (str).
|
||||
Each entity associates an evidence span with a resolved company.
|
||||
Entities without a company_id are skipped.
|
||||
evidence_spans
|
||||
Mapping of evidence_id -> text content for each evidence span.
|
||||
relations
|
||||
Optional list of dicts with keys: company_id (str or None),
|
||||
evidence_id (str or None), relation_type (str or None).
|
||||
Relations link additional evidence to companies (e.g., via
|
||||
directly_affects or inferred_exposure edges).
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[str, EvidenceGroup]
|
||||
Mapping of company_id -> EvidenceGroup containing all evidence
|
||||
associated with that company.
|
||||
"""
|
||||
# Accumulate evidence IDs per company
|
||||
company_evidence: dict[str, list[str]] = {}
|
||||
|
||||
for entity in entities:
|
||||
company_id = entity.get("company_id")
|
||||
evidence_id = entity.get("evidence_id")
|
||||
|
||||
if company_id is None or evidence_id is None:
|
||||
continue
|
||||
|
||||
if company_id not in company_evidence:
|
||||
company_evidence[company_id] = []
|
||||
|
||||
# Avoid duplicate evidence IDs per company
|
||||
if evidence_id not in company_evidence[company_id]:
|
||||
company_evidence[company_id].append(evidence_id)
|
||||
|
||||
# Process relations for additional evidence linkage
|
||||
if relations:
|
||||
for relation in relations:
|
||||
company_id = relation.get("company_id")
|
||||
evidence_id = relation.get("evidence_id")
|
||||
|
||||
if company_id is None or evidence_id is None:
|
||||
continue
|
||||
|
||||
if company_id not in company_evidence:
|
||||
company_evidence[company_id] = []
|
||||
|
||||
if evidence_id not in company_evidence[company_id]:
|
||||
company_evidence[company_id].append(evidence_id)
|
||||
|
||||
# Build EvidenceGroup objects
|
||||
groups: dict[str, EvidenceGroup] = {}
|
||||
for company_id, evidence_ids in company_evidence.items():
|
||||
texts = []
|
||||
valid_ids = []
|
||||
for eid in evidence_ids:
|
||||
text = evidence_spans.get(eid)
|
||||
if text is not None:
|
||||
valid_ids.append(eid)
|
||||
texts.append(text)
|
||||
|
||||
if valid_ids:
|
||||
groups[company_id] = EvidenceGroup(
|
||||
company_id=company_id,
|
||||
evidence_ids=valid_ids,
|
||||
texts=texts,
|
||||
)
|
||||
|
||||
return groups
|
||||
@@ -0,0 +1,167 @@
|
||||
"""FinBERT adapter for financial sentiment classification.
|
||||
|
||||
Provides a unified interface for FinBERT inference with:
|
||||
- Production mode: loads ProsusAI/finbert and runs real inference
|
||||
- Test mode: deterministic keyword-based mock probabilities
|
||||
|
||||
Model version is pinned and exposed for lineage tracking.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Pinned model configuration
|
||||
FINBERT_MODEL_NAME = "ProsusAI/finbert"
|
||||
FINBERT_MODEL_VERSION = "ProsusAI/finbert@v1.0"
|
||||
|
||||
# Keywords for deterministic test mode
|
||||
_POSITIVE_KEYWORDS = frozenset({
|
||||
"growth", "profit", "beat", "raised", "upgrade", "strong",
|
||||
"surge", "gain", "bullish", "outperform", "exceeded", "record",
|
||||
"positive", "optimistic", "rally", "upside",
|
||||
})
|
||||
_NEGATIVE_KEYWORDS = frozenset({
|
||||
"loss", "decline", "miss", "cut", "downgrade", "weak",
|
||||
"plunge", "drop", "bearish", "underperform", "fell", "crash",
|
||||
"negative", "pessimistic", "risk", "downside", "slump",
|
||||
})
|
||||
|
||||
|
||||
class FinBERTAdapter:
|
||||
"""Adapter for FinBERT financial sentiment classification.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
test_mode
|
||||
When True, uses deterministic keyword-based classification
|
||||
instead of loading the actual FinBERT model. Useful for testing
|
||||
without GPU/large model dependencies.
|
||||
"""
|
||||
|
||||
def __init__(self, test_mode: bool = True) -> None:
|
||||
self._test_mode = test_mode
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
self._model_name = FINBERT_MODEL_NAME
|
||||
self._model_version = FINBERT_MODEL_VERSION
|
||||
|
||||
if not test_mode:
|
||||
self._load_model()
|
||||
|
||||
@property
|
||||
def model_version(self) -> str:
|
||||
"""Return the pinned model version string."""
|
||||
return self._model_version
|
||||
|
||||
@property
|
||||
def model_name(self) -> str:
|
||||
"""Return the model name."""
|
||||
return self._model_name
|
||||
|
||||
def _load_model(self) -> None:
|
||||
"""Load the FinBERT model and tokenizer for production inference."""
|
||||
try:
|
||||
from transformers import AutoModelForSequenceClassification, AutoTokenizer
|
||||
|
||||
logger.info("Loading FinBERT model: %s", self._model_name)
|
||||
self._tokenizer = AutoTokenizer.from_pretrained(self._model_name)
|
||||
self._model = AutoModelForSequenceClassification.from_pretrained(self._model_name)
|
||||
self._model.eval()
|
||||
logger.info("FinBERT model loaded successfully")
|
||||
except ImportError:
|
||||
raise RuntimeError(
|
||||
"transformers and torch are required for production FinBERT inference. "
|
||||
"Install with: pip install transformers torch"
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to load FinBERT model: {e}") from e
|
||||
|
||||
def classify(self, texts: list[str]) -> list[tuple[float, float, float]]:
|
||||
"""Classify texts and return probability distributions.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
texts
|
||||
List of text strings to classify.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[tuple[float, float, float]]
|
||||
List of (positive_prob, negative_prob, neutral_prob) tuples.
|
||||
Probabilities sum to 1.0 for each text.
|
||||
"""
|
||||
if not texts:
|
||||
return []
|
||||
|
||||
if self._test_mode:
|
||||
return self._classify_test_mode(texts)
|
||||
|
||||
return self._classify_production(texts)
|
||||
|
||||
def _classify_test_mode(self, texts: list[str]) -> list[tuple[float, float, float]]:
|
||||
"""Deterministic keyword-based classification for testing.
|
||||
|
||||
Returns consistent probabilities based on keyword presence:
|
||||
- Positive keywords dominant -> (0.75, 0.10, 0.15)
|
||||
- Negative keywords dominant -> (0.10, 0.75, 0.15)
|
||||
- Both present (mixed signals) -> (0.40, 0.40, 0.20)
|
||||
- Neither present -> (0.15, 0.15, 0.70)
|
||||
"""
|
||||
results: list[tuple[float, float, float]] = []
|
||||
|
||||
for text in texts:
|
||||
lower_text = text.lower()
|
||||
words = set(lower_text.split())
|
||||
|
||||
has_positive = bool(words & _POSITIVE_KEYWORDS)
|
||||
has_negative = bool(words & _NEGATIVE_KEYWORDS)
|
||||
|
||||
if has_positive and has_negative:
|
||||
# Mixed signals
|
||||
results.append((0.40, 0.40, 0.20))
|
||||
elif has_positive:
|
||||
results.append((0.75, 0.10, 0.15))
|
||||
elif has_negative:
|
||||
results.append((0.10, 0.75, 0.15))
|
||||
else:
|
||||
# Neutral — no sentiment keywords
|
||||
results.append((0.15, 0.15, 0.70))
|
||||
|
||||
return results
|
||||
|
||||
def _classify_production(self, texts: list[str]) -> list[tuple[float, float, float]]:
|
||||
"""Run FinBERT inference on texts using the loaded model."""
|
||||
import torch
|
||||
|
||||
if self._model is None or self._tokenizer is None:
|
||||
raise RuntimeError("Model not loaded. Initialize with test_mode=False.")
|
||||
|
||||
results: list[tuple[float, float, float]] = []
|
||||
|
||||
# Process in batches to manage memory
|
||||
batch_size = 16
|
||||
for i in range(0, len(texts), batch_size):
|
||||
batch = texts[i : i + batch_size]
|
||||
inputs = self._tokenizer(
|
||||
batch,
|
||||
padding=True,
|
||||
truncation=True,
|
||||
max_length=512,
|
||||
return_tensors="pt",
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
outputs = self._model(**inputs)
|
||||
# FinBERT output order: positive, negative, neutral
|
||||
probs = torch.softmax(outputs.logits, dim=-1)
|
||||
|
||||
for prob in probs:
|
||||
pos = float(prob[0])
|
||||
neg = float(prob[1])
|
||||
neu = float(prob[2])
|
||||
results.append((pos, neg, neu))
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Mixed sentiment detection from evidence-group disagreement.
|
||||
|
||||
When evidence groups for the same company disagree (some positive,
|
||||
some negative), the resulting label is "mixed" rather than an
|
||||
unconstrained model output. This follows the design requirement that
|
||||
mixed sentiment comes from conflicting supported evidence, not from
|
||||
a fourth softmax label.
|
||||
|
||||
NOTE: This module provides the legacy compute_mixed_sentiment function
|
||||
for backward compatibility. New code should prefer
|
||||
aggregation.aggregate_evidence_sentiments which accepts TextSentiment
|
||||
objects directly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from services.intelligence_pipeline_v3.sentiment.models import CompanySentimentResult, TextSentiment
|
||||
|
||||
# Disagreement threshold: both positive and negative probabilities
|
||||
# must be >= this value across evidence groups to signal disagreement
|
||||
DISAGREEMENT_THRESHOLD = 0.3
|
||||
|
||||
|
||||
def compute_mixed_sentiment(
|
||||
company_id: str,
|
||||
group_results: list[tuple[float, float, float]],
|
||||
evidence_ids: list[str],
|
||||
model_version: str,
|
||||
calibration_version: str = "uncalibrated",
|
||||
) -> CompanySentimentResult:
|
||||
"""Compute company sentiment from multiple evidence group results.
|
||||
|
||||
Applies disagreement detection: if evidence groups for the same
|
||||
company show both positive and negative signals above the threshold,
|
||||
the label is "mixed".
|
||||
|
||||
Parameters
|
||||
----------
|
||||
company_id
|
||||
Resolved company identifier.
|
||||
group_results
|
||||
List of (positive_prob, negative_prob, neutral_prob) tuples,
|
||||
one per evidence group for this company.
|
||||
evidence_ids
|
||||
All evidence span IDs contributing to this result.
|
||||
model_version
|
||||
Sentiment model version string.
|
||||
calibration_version
|
||||
Calibration artifact version.
|
||||
|
||||
Returns
|
||||
-------
|
||||
CompanySentimentResult
|
||||
Aggregated sentiment result with disagreement-based mixed detection.
|
||||
"""
|
||||
if not group_results:
|
||||
# No evidence — neutral by default
|
||||
return CompanySentimentResult(
|
||||
company_id=company_id,
|
||||
label="neutral",
|
||||
positive_prob=0.0,
|
||||
negative_prob=0.0,
|
||||
neutral_prob=1.0,
|
||||
evidence_ids=evidence_ids,
|
||||
is_mixed=False,
|
||||
per_text_scores=[],
|
||||
model_version=model_version,
|
||||
calibration_version=calibration_version,
|
||||
)
|
||||
|
||||
# Check for disagreement across groups
|
||||
is_mixed = _detect_disagreement(group_results)
|
||||
|
||||
# Compute weighted average of group probabilities
|
||||
n = len(group_results)
|
||||
avg_pos = sum(r[0] for r in group_results) / n
|
||||
avg_neg = sum(r[1] for r in group_results) / n
|
||||
avg_neu = sum(r[2] for r in group_results) / n
|
||||
|
||||
# Normalize to ensure probabilities sum to 1.0
|
||||
total = avg_pos + avg_neg + avg_neu
|
||||
if total > 0:
|
||||
avg_pos /= total
|
||||
avg_neg /= total
|
||||
avg_neu /= total
|
||||
else:
|
||||
avg_pos = 0.0
|
||||
avg_neg = 0.0
|
||||
avg_neu = 1.0
|
||||
|
||||
# Determine label
|
||||
if is_mixed:
|
||||
label = "mixed"
|
||||
else:
|
||||
label = _argmax_label(avg_pos, avg_neg, avg_neu)
|
||||
|
||||
# Build per-text scores from tuples for provenance
|
||||
per_text_scores: list[TextSentiment] = []
|
||||
for i, (pos, neg, neu) in enumerate(group_results):
|
||||
eid = evidence_ids[i] if i < len(evidence_ids) else f"unknown_{i}"
|
||||
per_text_scores.append(
|
||||
TextSentiment(
|
||||
evidence_id=eid,
|
||||
positive_prob=pos,
|
||||
negative_prob=neg,
|
||||
neutral_prob=neu,
|
||||
)
|
||||
)
|
||||
|
||||
return CompanySentimentResult(
|
||||
company_id=company_id,
|
||||
label=label,
|
||||
positive_prob=round(avg_pos, 6),
|
||||
negative_prob=round(avg_neg, 6),
|
||||
neutral_prob=round(avg_neu, 6),
|
||||
evidence_ids=evidence_ids,
|
||||
is_mixed=is_mixed,
|
||||
per_text_scores=per_text_scores,
|
||||
model_version=model_version,
|
||||
calibration_version=calibration_version,
|
||||
)
|
||||
|
||||
|
||||
def _detect_disagreement(group_results: list[tuple[float, float, float]]) -> bool:
|
||||
"""Detect if evidence groups disagree on sentiment direction.
|
||||
|
||||
Disagreement is detected when across all groups, the maximum
|
||||
positive probability is >= threshold AND the maximum negative
|
||||
probability is >= threshold. This means some evidence strongly
|
||||
suggests positive while other evidence strongly suggests negative.
|
||||
"""
|
||||
if len(group_results) < 2:
|
||||
return False
|
||||
|
||||
max_pos = max(r[0] for r in group_results)
|
||||
max_neg = max(r[1] for r in group_results)
|
||||
|
||||
return max_pos >= DISAGREEMENT_THRESHOLD and max_neg >= DISAGREEMENT_THRESHOLD
|
||||
|
||||
|
||||
def _argmax_label(pos: float, neg: float, neu: float) -> str:
|
||||
"""Return the label with the highest probability."""
|
||||
if pos >= neg and pos >= neu:
|
||||
return "positive"
|
||||
elif neg >= pos and neg >= neu:
|
||||
return "negative"
|
||||
else:
|
||||
return "neutral"
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Pydantic models for company-specific sentiment analysis."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class EvidenceGroup(BaseModel):
|
||||
"""A group of evidence spans associated with a specific company.
|
||||
|
||||
A single evidence span can appear in multiple groups when the
|
||||
underlying text mentions multiple companies.
|
||||
"""
|
||||
|
||||
company_id: str = Field(description="Resolved company identifier")
|
||||
evidence_ids: list[str] = Field(description="IDs of evidence spans in this group")
|
||||
texts: list[str] = Field(description="Text snippets from evidence spans")
|
||||
|
||||
@field_validator("evidence_ids")
|
||||
@classmethod
|
||||
def evidence_ids_non_empty(cls, v: list[str]) -> list[str]:
|
||||
if not v:
|
||||
raise ValueError("evidence_ids must not be empty")
|
||||
return v
|
||||
|
||||
@field_validator("texts")
|
||||
@classmethod
|
||||
def texts_non_empty(cls, v: list[str]) -> list[str]:
|
||||
if not v:
|
||||
raise ValueError("texts must not be empty")
|
||||
return v
|
||||
|
||||
|
||||
class TextSentiment(BaseModel):
|
||||
"""Per-text sentiment probability distribution with evidence linkage.
|
||||
|
||||
Stores the raw FinBERT output for a single evidence span text,
|
||||
enabling full provenance from probability to source evidence.
|
||||
"""
|
||||
|
||||
evidence_id: str = Field(description="Evidence span ID this score belongs to")
|
||||
positive_prob: float = Field(ge=0.0, le=1.0, description="Probability of positive sentiment")
|
||||
negative_prob: float = Field(ge=0.0, le=1.0, description="Probability of negative sentiment")
|
||||
neutral_prob: float = Field(ge=0.0, le=1.0, description="Probability of neutral sentiment")
|
||||
|
||||
@property
|
||||
def dominant_label(self) -> str:
|
||||
"""Return the label with highest probability."""
|
||||
if self.positive_prob >= self.negative_prob and self.positive_prob >= self.neutral_prob:
|
||||
return "positive"
|
||||
elif self.negative_prob >= self.positive_prob and self.negative_prob >= self.neutral_prob:
|
||||
return "negative"
|
||||
return "neutral"
|
||||
|
||||
|
||||
class CompanySentimentResult(BaseModel):
|
||||
"""Sentiment classification result for a single company.
|
||||
|
||||
Contains full probability distribution, supporting evidence IDs,
|
||||
per-text scores, and model/calibration versioning for lineage tracking.
|
||||
"""
|
||||
|
||||
company_id: str = Field(description="Resolved company identifier")
|
||||
label: str = Field(description="Derived label: positive, negative, neutral, or mixed")
|
||||
positive_prob: float = Field(ge=0.0, le=1.0, description="Probability of positive sentiment")
|
||||
negative_prob: float = Field(ge=0.0, le=1.0, description="Probability of negative sentiment")
|
||||
neutral_prob: float = Field(ge=0.0, le=1.0, description="Probability of neutral sentiment")
|
||||
evidence_ids: list[str] = Field(description="All evidence span IDs contributing to this result")
|
||||
is_mixed: bool = Field(default=False, description="Whether mixed sentiment was detected from disagreement")
|
||||
per_text_scores: list[TextSentiment] = Field(
|
||||
default_factory=list,
|
||||
description="Full probability distributions per evidence text",
|
||||
)
|
||||
model_version: str = Field(description="Sentiment model name and version")
|
||||
calibration_version: str = Field(default="uncalibrated", description="Calibration artifact version")
|
||||
|
||||
@field_validator("label")
|
||||
@classmethod
|
||||
def label_valid(cls, v: str) -> str:
|
||||
valid_labels = {"positive", "negative", "neutral", "mixed"}
|
||||
if v not in valid_labels:
|
||||
raise ValueError(f"label must be one of {valid_labels}, got '{v}'")
|
||||
return v
|
||||
|
||||
|
||||
class SentimentBatchResult(BaseModel):
|
||||
"""Result of sentiment classification for a batch of companies."""
|
||||
|
||||
results: list[CompanySentimentResult] = Field(description="Per-company sentiment results")
|
||||
model_version: str = Field(description="Sentiment model version used for batch")
|
||||
processing_time_ms: int = Field(ge=0, description="Total processing time in milliseconds")
|
||||
@@ -0,0 +1,155 @@
|
||||
"""SentimentScorer — abstracts sentiment model behind a protocol.
|
||||
|
||||
Supports both FinBERT production inference and deterministic test mode.
|
||||
Returns per-text probability distributions and aggregates across evidence
|
||||
texts for a company, integrating with the calibration pipeline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from services.intelligence_pipeline_v3.sentiment.aggregation import (
|
||||
aggregate_evidence_sentiments,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.sentiment.calibrator import SentimentCalibrator
|
||||
from services.intelligence_pipeline_v3.sentiment.finbert_adapter import FinBERTAdapter
|
||||
from services.intelligence_pipeline_v3.sentiment.models import (
|
||||
CompanySentimentResult,
|
||||
EvidenceGroup,
|
||||
SentimentBatchResult,
|
||||
TextSentiment,
|
||||
)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SentimentModel(Protocol):
|
||||
"""Protocol for sentiment classification models.
|
||||
|
||||
Any model implementing this interface can be used by SentimentScorer,
|
||||
enabling easy swapping between FinBERT, mock models, or future
|
||||
alternatives without changing the scoring logic.
|
||||
"""
|
||||
|
||||
@property
|
||||
def model_version(self) -> str:
|
||||
"""Return the model version string for lineage tracking."""
|
||||
...
|
||||
|
||||
def classify(self, texts: list[str]) -> list[tuple[float, float, float]]:
|
||||
"""Classify texts and return (positive, negative, neutral) tuples."""
|
||||
...
|
||||
|
||||
|
||||
class SentimentScorer:
|
||||
"""Scores evidence groups for company-specific sentiment.
|
||||
|
||||
Orchestrates the full sentiment pipeline:
|
||||
1. Classifies each evidence text via the underlying model
|
||||
2. Stores per-text probability distributions
|
||||
3. Optionally calibrates raw scores
|
||||
4. Aggregates and detects mixed sentiment from disagreement
|
||||
|
||||
Parameters
|
||||
----------
|
||||
model
|
||||
Sentiment classification model implementing the SentimentModel protocol.
|
||||
Defaults to FinBERTAdapter in test mode.
|
||||
calibrator
|
||||
Optional calibration wrapper for raw probabilities.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: SentimentModel | None = None,
|
||||
calibrator: SentimentCalibrator | None = None,
|
||||
) -> None:
|
||||
self._model: SentimentModel = model or FinBERTAdapter(test_mode=True)
|
||||
self._calibrator = calibrator
|
||||
|
||||
@property
|
||||
def model_version(self) -> str:
|
||||
"""Return the underlying model version."""
|
||||
return self._model.model_version
|
||||
|
||||
@property
|
||||
def calibration_version(self) -> str:
|
||||
"""Return the calibration version (or 'uncalibrated')."""
|
||||
if self._calibrator and self._calibrator.is_fitted:
|
||||
return self._calibrator.calibration_version
|
||||
return "uncalibrated"
|
||||
|
||||
async def score(self, evidence_group: EvidenceGroup) -> CompanySentimentResult:
|
||||
"""Score a single evidence group and return aggregated company sentiment.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
evidence_group
|
||||
Company-linked evidence group with texts to classify.
|
||||
|
||||
Returns
|
||||
-------
|
||||
CompanySentimentResult
|
||||
Aggregated sentiment with per-text scores, mixed detection,
|
||||
and full probability distributions.
|
||||
"""
|
||||
# Classify all texts in the group
|
||||
raw_probs = self._model.classify(evidence_group.texts)
|
||||
|
||||
# Build per-text sentiment scores
|
||||
per_text_scores: list[TextSentiment] = []
|
||||
for i, (pos, neg, neu) in enumerate(raw_probs):
|
||||
evidence_id = evidence_group.evidence_ids[i]
|
||||
|
||||
# Apply calibration if available
|
||||
if self._calibrator and self._calibrator.is_fitted:
|
||||
calibrated = self._calibrator.calibrate([pos, neg, neu])
|
||||
pos, neg, neu = calibrated[0], calibrated[1], calibrated[2]
|
||||
|
||||
per_text_scores.append(
|
||||
TextSentiment(
|
||||
evidence_id=evidence_id,
|
||||
positive_prob=pos,
|
||||
negative_prob=neg,
|
||||
neutral_prob=neu,
|
||||
)
|
||||
)
|
||||
|
||||
# Aggregate across evidence texts with mixed detection
|
||||
return aggregate_evidence_sentiments(
|
||||
company_id=evidence_group.company_id,
|
||||
per_text_scores=per_text_scores,
|
||||
model_version=self._model.model_version,
|
||||
calibration_version=self.calibration_version,
|
||||
)
|
||||
|
||||
async def score_batch(
|
||||
self, evidence_groups: dict[str, EvidenceGroup]
|
||||
) -> SentimentBatchResult:
|
||||
"""Score multiple evidence groups and return batch results.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
evidence_groups
|
||||
Mapping of company_id -> EvidenceGroup.
|
||||
|
||||
Returns
|
||||
-------
|
||||
SentimentBatchResult
|
||||
Batch result with per-company sentiment and timing.
|
||||
"""
|
||||
start_ms = int(time.time() * 1000)
|
||||
|
||||
results: list[CompanySentimentResult] = []
|
||||
for _company_id, group in evidence_groups.items():
|
||||
result = await self.score(group)
|
||||
results.append(result)
|
||||
|
||||
elapsed_ms = int(time.time() * 1000) - start_ms
|
||||
|
||||
return SentimentBatchResult(
|
||||
results=results,
|
||||
model_version=self._model.model_version,
|
||||
processing_time_ms=elapsed_ms,
|
||||
)
|
||||
Reference in New Issue
Block a user