feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,32 @@
|
||||
"""Confidence feature pipeline for Intelligence Pipeline v3.
|
||||
|
||||
Provides calibrated extraction confidence from specialist scores,
|
||||
symbol resolution, evidence validation, schema completeness,
|
||||
model agreement, and historical calibration data. Replaces
|
||||
generative model self-reported confidence with empirically
|
||||
calibrated probabilities.
|
||||
"""
|
||||
|
||||
from services.intelligence_pipeline_v3.confidence.artifacts import (
|
||||
load_artifact,
|
||||
save_artifact,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.confidence.calibrator import ConfidenceCalibrator
|
||||
from services.intelligence_pipeline_v3.confidence.defaults import get_default_confidence
|
||||
from services.intelligence_pipeline_v3.confidence.features import ConfidenceFeatureExtractor
|
||||
from services.intelligence_pipeline_v3.confidence.models import (
|
||||
CalibrationArtifactMetadata,
|
||||
ConfidenceFeatures,
|
||||
ConfidenceResult,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CalibrationArtifactMetadata",
|
||||
"ConfidenceCalibrator",
|
||||
"ConfidenceFeatureExtractor",
|
||||
"ConfidenceFeatures",
|
||||
"ConfidenceResult",
|
||||
"get_default_confidence",
|
||||
"load_artifact",
|
||||
"save_artifact",
|
||||
]
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Calibration artifact persistence.
|
||||
|
||||
Handles versioned save/load of fitted calibrator objects alongside
|
||||
metadata including training provenance, quality metrics, and version.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
|
||||
from services.intelligence_pipeline_v3.confidence.calibrator import ConfidenceCalibrator
|
||||
from services.intelligence_pipeline_v3.confidence.models import CalibrationArtifactMetadata
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ARTIFACT_FILE = "calibrator.pkl"
|
||||
METADATA_FILE = "metadata.json"
|
||||
|
||||
|
||||
def save_artifact(
|
||||
calibrator: ConfidenceCalibrator,
|
||||
version: str,
|
||||
path: str | Path,
|
||||
) -> Path:
|
||||
"""Save a fitted calibrator and metadata to a versioned directory.
|
||||
|
||||
Creates the directory structure:
|
||||
<path>/<version>/calibrator.pkl
|
||||
<path>/<version>/metadata.json
|
||||
|
||||
Parameters
|
||||
----------
|
||||
calibrator
|
||||
A fitted ConfidenceCalibrator instance.
|
||||
version
|
||||
Version string for this artifact (e.g., "v1.0.0").
|
||||
path
|
||||
Base directory for artifact storage.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Path
|
||||
Path to the versioned artifact directory.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If the calibrator has not been fitted.
|
||||
"""
|
||||
if not calibrator.is_fitted:
|
||||
raise ValueError("Cannot save an unfitted calibrator")
|
||||
|
||||
artifact_dir = Path(path) / version
|
||||
artifact_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Save the calibrator model
|
||||
calibrator_path = artifact_dir / ARTIFACT_FILE
|
||||
with open(calibrator_path, "wb") as f:
|
||||
pickle.dump(calibrator, f, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
|
||||
# Save metadata
|
||||
metadata = calibrator.metadata
|
||||
if metadata is None:
|
||||
metadata = CalibrationArtifactMetadata(
|
||||
version=version,
|
||||
method=calibrator.method, # type: ignore[arg-type]
|
||||
training_count=0,
|
||||
training_range="unknown",
|
||||
ece=0.0,
|
||||
brier_score=0.0,
|
||||
)
|
||||
|
||||
metadata_path = artifact_dir / METADATA_FILE
|
||||
with open(metadata_path, "w") as f:
|
||||
json.dump(metadata.model_dump(mode="json"), f, indent=2, default=str)
|
||||
|
||||
logger.info(
|
||||
"Saved calibration artifact: version=%s, method=%s, path=%s",
|
||||
version,
|
||||
calibrator.method,
|
||||
artifact_dir,
|
||||
)
|
||||
return artifact_dir
|
||||
|
||||
|
||||
def load_artifact(path: str | Path) -> ConfidenceCalibrator:
|
||||
"""Load a calibrator from a versioned artifact directory.
|
||||
|
||||
Expects the directory to contain calibrator.pkl and metadata.json.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path
|
||||
Path to the versioned artifact directory (e.g., <base>/v1.0.0/).
|
||||
|
||||
Returns
|
||||
-------
|
||||
ConfidenceCalibrator
|
||||
The loaded and ready-to-use calibrator.
|
||||
|
||||
Raises
|
||||
------
|
||||
FileNotFoundError
|
||||
If the artifact directory or files don't exist.
|
||||
ValueError
|
||||
If the loaded object is not a ConfidenceCalibrator.
|
||||
"""
|
||||
artifact_dir = Path(path)
|
||||
|
||||
calibrator_path = artifact_dir / ARTIFACT_FILE
|
||||
if not calibrator_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Calibrator artifact not found at {calibrator_path}"
|
||||
)
|
||||
|
||||
with open(calibrator_path, "rb") as f:
|
||||
calibrator = pickle.load(f) # noqa: S301
|
||||
|
||||
if not isinstance(calibrator, ConfidenceCalibrator):
|
||||
raise ValueError(
|
||||
f"Loaded object is not a ConfidenceCalibrator: {type(calibrator)}"
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Loaded calibration artifact: version=%s, method=%s, path=%s",
|
||||
calibrator.version,
|
||||
calibrator.method,
|
||||
artifact_dir,
|
||||
)
|
||||
return calibrator
|
||||
|
||||
|
||||
def load_metadata(path: str | Path) -> CalibrationArtifactMetadata:
|
||||
"""Load only the metadata for an artifact without loading the full model.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path
|
||||
Path to the versioned artifact directory.
|
||||
|
||||
Returns
|
||||
-------
|
||||
CalibrationArtifactMetadata
|
||||
The artifact metadata.
|
||||
|
||||
Raises
|
||||
------
|
||||
FileNotFoundError
|
||||
If the metadata file doesn't exist.
|
||||
"""
|
||||
metadata_path = Path(path) / METADATA_FILE
|
||||
if not metadata_path.exists():
|
||||
raise FileNotFoundError(f"Metadata not found at {metadata_path}")
|
||||
|
||||
with open(metadata_path) as f:
|
||||
data = json.load(f)
|
||||
|
||||
return CalibrationArtifactMetadata(**data)
|
||||
|
||||
|
||||
def list_versions(base_path: str | Path) -> list[str]:
|
||||
"""List all available artifact versions in a base directory.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
base_path
|
||||
Base directory containing versioned subdirectories.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[str]
|
||||
Sorted list of version strings.
|
||||
"""
|
||||
base = Path(base_path)
|
||||
if not base.exists():
|
||||
return []
|
||||
|
||||
versions = []
|
||||
for item in base.iterdir():
|
||||
if item.is_dir() and (item / ARTIFACT_FILE).exists():
|
||||
versions.append(item.name)
|
||||
|
||||
return sorted(versions)
|
||||
@@ -0,0 +1,406 @@
|
||||
"""Confidence calibrator using isotonic or Platt scaling.
|
||||
|
||||
Maps confidence feature vectors to calibrated correctness probabilities.
|
||||
Supports training on held-out Gold_Corpus data, cross-validation for
|
||||
method comparison, and versioned artifact tracking.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Literal
|
||||
|
||||
import numpy as np
|
||||
|
||||
from services.intelligence_pipeline_v3.confidence.models import (
|
||||
CalibrationArtifactMetadata,
|
||||
ConfidenceFeatures,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_VERSION = "uncalibrated"
|
||||
|
||||
|
||||
class ConfidenceCalibrator:
|
||||
"""Calibrates confidence features to correctness probabilities.
|
||||
|
||||
Supports isotonic regression and Platt (logistic) scaling.
|
||||
The calibrator is fitted on labeled Gold_Corpus data where labels
|
||||
indicate whether the extraction was correct (True) or not (False).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
method
|
||||
Calibration method: "isotonic" for non-parametric monotone fit,
|
||||
"platt" for logistic regression scaling.
|
||||
"""
|
||||
|
||||
def __init__(self, method: Literal["isotonic", "platt"] = "isotonic") -> None:
|
||||
self._method: Literal["isotonic", "platt"] = method
|
||||
self._version: str = DEFAULT_VERSION
|
||||
self._fitted: bool = False
|
||||
self._model: object | None = None
|
||||
self._metadata: CalibrationArtifactMetadata | None = None
|
||||
self._training_count: int = 0
|
||||
|
||||
@property
|
||||
def method(self) -> str:
|
||||
"""Return the calibration method."""
|
||||
return self._method
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
"""Return the calibration artifact version."""
|
||||
return self._version
|
||||
|
||||
@property
|
||||
def is_fitted(self) -> bool:
|
||||
"""Return whether the calibrator has been fitted."""
|
||||
return self._fitted
|
||||
|
||||
@property
|
||||
def metadata(self) -> CalibrationArtifactMetadata | None:
|
||||
"""Return the artifact metadata if fitted."""
|
||||
return self._metadata
|
||||
|
||||
def fit(
|
||||
self,
|
||||
features: list[ConfidenceFeatures],
|
||||
labels: list[bool],
|
||||
method: str | None = None,
|
||||
version: str = "v1.0.0",
|
||||
training_range: str = "unknown",
|
||||
) -> None:
|
||||
"""Train the calibrator on labeled feature/correctness pairs.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
features
|
||||
List of confidence feature vectors from training data.
|
||||
labels
|
||||
True if the extraction was correct, False otherwise.
|
||||
method
|
||||
Override method for this fit (isotonic or platt).
|
||||
If None, uses the instance default.
|
||||
version
|
||||
Version string for the resulting artifact.
|
||||
training_range
|
||||
Description of the training data date range.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If features and labels have different lengths or are empty.
|
||||
"""
|
||||
if not features or not labels:
|
||||
raise ValueError("features and labels must not be empty")
|
||||
if len(features) != len(labels):
|
||||
raise ValueError(
|
||||
f"features ({len(features)}) and labels ({len(labels)}) must have the same length"
|
||||
)
|
||||
|
||||
if method is not None:
|
||||
if method not in ("isotonic", "platt"):
|
||||
raise ValueError(f"method must be 'isotonic' or 'platt', got '{method}'")
|
||||
self._method = method # type: ignore[assignment]
|
||||
|
||||
# Convert features to matrix
|
||||
X = np.array([f.to_vector() for f in features], dtype=np.float64)
|
||||
y = np.array(labels, dtype=np.float64)
|
||||
|
||||
if self._method == "isotonic":
|
||||
self._fit_isotonic(X, y)
|
||||
else:
|
||||
self._fit_platt(X, y)
|
||||
|
||||
self._version = version
|
||||
self._training_count = len(features)
|
||||
self._fitted = True
|
||||
|
||||
# Compute calibration quality on training data (for metadata)
|
||||
predictions = self._predict_batch(X)
|
||||
ece = _compute_ece(predictions, y)
|
||||
brier = _compute_brier(predictions, y)
|
||||
|
||||
self._metadata = CalibrationArtifactMetadata(
|
||||
version=version,
|
||||
method=self._method,
|
||||
training_count=len(features),
|
||||
training_range=training_range,
|
||||
ece=ece,
|
||||
brier_score=brier,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"ConfidenceCalibrator fitted: method=%s, n=%d, version=%s, ECE=%.4f, Brier=%.4f",
|
||||
self._method,
|
||||
len(features),
|
||||
version,
|
||||
ece,
|
||||
brier,
|
||||
)
|
||||
|
||||
def predict(self, features: ConfidenceFeatures) -> float:
|
||||
"""Return calibrated probability of extraction correctness.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
features
|
||||
Confidence feature vector for a single extraction.
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
Calibrated probability in [0, 1].
|
||||
"""
|
||||
if not self._fitted:
|
||||
# Return a neutral default when uncalibrated
|
||||
return 0.5
|
||||
|
||||
X = np.array([features.to_vector()], dtype=np.float64)
|
||||
predictions = self._predict_batch(X)
|
||||
return float(np.clip(predictions[0], 0.0, 1.0))
|
||||
|
||||
def predict_batch(self, features_list: list[ConfidenceFeatures]) -> list[float]:
|
||||
"""Return calibrated probabilities for a batch of feature vectors.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
features_list
|
||||
List of confidence feature vectors.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[float]
|
||||
Calibrated probabilities in [0, 1].
|
||||
"""
|
||||
if not self._fitted:
|
||||
return [0.5] * len(features_list)
|
||||
|
||||
X = np.array([f.to_vector() for f in features_list], dtype=np.float64)
|
||||
predictions = self._predict_batch(X)
|
||||
return [float(np.clip(p, 0.0, 1.0)) for p in predictions]
|
||||
|
||||
def evaluate(
|
||||
self,
|
||||
features: list[ConfidenceFeatures],
|
||||
labels: list[bool],
|
||||
) -> tuple[float, float]:
|
||||
"""Evaluate ECE and Brier score on held-out data.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
features
|
||||
Held-out feature vectors.
|
||||
labels
|
||||
True correctness labels.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple[float, float]
|
||||
(ECE, Brier_score) on the held-out set.
|
||||
"""
|
||||
if not features or not labels:
|
||||
raise ValueError("features and labels must not be empty")
|
||||
if len(features) != len(labels):
|
||||
raise ValueError("features and labels must have the same length")
|
||||
|
||||
X = np.array([f.to_vector() for f in features], dtype=np.float64)
|
||||
y = np.array(labels, dtype=np.float64)
|
||||
|
||||
if self._fitted:
|
||||
predictions = self._predict_batch(X)
|
||||
else:
|
||||
predictions = np.full(len(y), 0.5)
|
||||
|
||||
ece = _compute_ece(predictions, y)
|
||||
brier = _compute_brier(predictions, y)
|
||||
return ece, brier
|
||||
|
||||
def _fit_isotonic(self, X: np.ndarray, y: np.ndarray) -> None:
|
||||
"""Fit isotonic regression on aggregated feature scores."""
|
||||
from sklearn.isotonic import IsotonicRegression
|
||||
|
||||
# Aggregate features into a single score for isotonic monotone fit
|
||||
aggregated = X.mean(axis=1)
|
||||
iso = IsotonicRegression(y_min=0.0, y_max=1.0, out_of_bounds="clip")
|
||||
iso.fit(aggregated, y)
|
||||
self._model = iso
|
||||
|
||||
def _fit_platt(self, X: np.ndarray, y: np.ndarray) -> None:
|
||||
"""Fit logistic regression (Platt scaling) on the full feature vector."""
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
|
||||
y_int = y.astype(np.int32)
|
||||
if len(np.unique(y_int)) < 2:
|
||||
# Not enough class diversity — store a dummy model
|
||||
self._model = _ConstantPredictor(float(y.mean()))
|
||||
return
|
||||
|
||||
lr = LogisticRegression(solver="lbfgs", max_iter=1000, C=1.0)
|
||||
lr.fit(X, y_int)
|
||||
self._model = lr
|
||||
|
||||
def _predict_batch(self, X: np.ndarray) -> np.ndarray:
|
||||
"""Internal prediction dispatch."""
|
||||
if self._model is None:
|
||||
return np.full(X.shape[0], 0.5)
|
||||
|
||||
if self._method == "isotonic":
|
||||
# Isotonic uses aggregated score
|
||||
aggregated = X.mean(axis=1)
|
||||
return self._model.predict(aggregated) # type: ignore[union-attr]
|
||||
else:
|
||||
# Platt uses full feature vector
|
||||
if isinstance(self._model, _ConstantPredictor):
|
||||
return self._model.predict(X)
|
||||
return self._model.predict_proba(X)[:, 1] # type: ignore[union-attr]
|
||||
|
||||
|
||||
class _ConstantPredictor:
|
||||
"""Fallback predictor when training data has only one class."""
|
||||
|
||||
def __init__(self, value: float) -> None:
|
||||
self._value = value
|
||||
|
||||
def predict(self, X: np.ndarray) -> np.ndarray:
|
||||
return np.full(X.shape[0], self._value)
|
||||
|
||||
|
||||
def _compute_ece(
|
||||
predictions: np.ndarray,
|
||||
labels: np.ndarray,
|
||||
n_bins: int = 10,
|
||||
) -> float:
|
||||
"""Compute Expected Calibration Error.
|
||||
|
||||
Partitions predictions into equal-width bins and computes the
|
||||
weighted average of |avg_predicted - avg_actual| per bin.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
predictions
|
||||
Predicted probabilities.
|
||||
labels
|
||||
True binary labels (0 or 1).
|
||||
n_bins
|
||||
Number of equal-width bins.
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
ECE value in [0, 1].
|
||||
"""
|
||||
if len(predictions) == 0:
|
||||
return 0.0
|
||||
|
||||
bin_boundaries = np.linspace(0.0, 1.0, n_bins + 1)
|
||||
ece = 0.0
|
||||
n = len(predictions)
|
||||
|
||||
for i in range(n_bins):
|
||||
lower = bin_boundaries[i]
|
||||
upper = bin_boundaries[i + 1]
|
||||
|
||||
if i == n_bins - 1:
|
||||
# Include right boundary in last bin
|
||||
mask = (predictions >= lower) & (predictions <= upper)
|
||||
else:
|
||||
mask = (predictions >= lower) & (predictions < upper)
|
||||
|
||||
bin_count = mask.sum()
|
||||
if bin_count == 0:
|
||||
continue
|
||||
|
||||
avg_predicted = predictions[mask].mean()
|
||||
avg_actual = labels[mask].mean()
|
||||
ece += (bin_count / n) * abs(avg_predicted - avg_actual)
|
||||
|
||||
return float(ece)
|
||||
|
||||
|
||||
def _compute_brier(predictions: np.ndarray, labels: np.ndarray) -> float:
|
||||
"""Compute Brier score (mean squared error of probability predictions).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
predictions
|
||||
Predicted probabilities.
|
||||
labels
|
||||
True binary labels (0 or 1).
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
Brier score in [0, 1].
|
||||
"""
|
||||
if len(predictions) == 0:
|
||||
return 0.0
|
||||
return float(np.mean((predictions - labels) ** 2))
|
||||
|
||||
|
||||
def compare_methods(
|
||||
features: list[ConfidenceFeatures],
|
||||
labels: list[bool],
|
||||
n_folds: int = 5,
|
||||
) -> dict[str, dict[str, float]]:
|
||||
"""Compare isotonic and Platt methods using k-fold cross-validation.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
features
|
||||
Full set of training features.
|
||||
labels
|
||||
Full set of correctness labels.
|
||||
n_folds
|
||||
Number of cross-validation folds.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict
|
||||
Mapping of method name to {"ece": float, "brier": float} averages.
|
||||
"""
|
||||
if len(features) < n_folds * 2:
|
||||
raise ValueError(
|
||||
f"Need at least {n_folds * 2} samples for {n_folds}-fold CV, got {len(features)}"
|
||||
)
|
||||
|
||||
results: dict[str, list[tuple[float, float]]] = {
|
||||
"isotonic": [],
|
||||
"platt": [],
|
||||
}
|
||||
|
||||
indices = np.arange(len(features))
|
||||
fold_size = len(features) // n_folds
|
||||
|
||||
for fold in range(n_folds):
|
||||
val_start = fold * fold_size
|
||||
val_end = val_start + fold_size if fold < n_folds - 1 else len(features)
|
||||
|
||||
val_indices = indices[val_start:val_end]
|
||||
train_indices = np.concatenate([indices[:val_start], indices[val_end:]])
|
||||
|
||||
train_features = [features[i] for i in train_indices]
|
||||
train_labels = [labels[i] for i in train_indices]
|
||||
val_features = [features[i] for i in val_indices]
|
||||
val_labels = [labels[i] for i in val_indices]
|
||||
|
||||
for method_name in ("isotonic", "platt"):
|
||||
cal = ConfidenceCalibrator(method=method_name) # type: ignore[arg-type]
|
||||
cal.fit(
|
||||
train_features,
|
||||
train_labels,
|
||||
version=f"cv-fold-{fold}",
|
||||
training_range="cross-validation",
|
||||
)
|
||||
ece, brier = cal.evaluate(val_features, val_labels)
|
||||
results[method_name].append((ece, brier))
|
||||
|
||||
return {
|
||||
method: {
|
||||
"ece": float(np.mean([r[0] for r in scores])),
|
||||
"brier": float(np.mean([r[1] for r in scores])),
|
||||
}
|
||||
for method, scores in results.items()
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Conservative confidence defaults for underrepresented classes.
|
||||
|
||||
When calibration data is insufficient for a specific document type or
|
||||
event class, returns conservative values (0.3-0.5) and marks the result
|
||||
as under-calibrated per Requirement 10.7.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from services.intelligence_pipeline_v3.confidence.models import ConfidenceResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Conservative default probabilities by document type.
|
||||
# These are intentionally low (0.3-0.5) to avoid overconfidence
|
||||
# when insufficient calibration data exists.
|
||||
_DOCUMENT_TYPE_DEFAULTS: dict[str, float] = {
|
||||
"news": 0.45,
|
||||
"filing": 0.40,
|
||||
"transcript": 0.40,
|
||||
"press_release": 0.45,
|
||||
"macro_event": 0.35,
|
||||
"unknown": 0.30,
|
||||
}
|
||||
|
||||
# Conservative default probabilities by event class.
|
||||
# More complex or rare event types get lower defaults.
|
||||
_EVENT_CLASS_DEFAULTS: dict[str, float] = {
|
||||
"earnings_beat": 0.50,
|
||||
"earnings_miss": 0.50,
|
||||
"guidance_raise": 0.45,
|
||||
"guidance_cut": 0.45,
|
||||
"merger_acquisition": 0.40,
|
||||
"product_launch": 0.45,
|
||||
"regulatory_action": 0.40,
|
||||
"management_change": 0.45,
|
||||
"legal_proceeding": 0.40,
|
||||
"supply_chain": 0.35,
|
||||
"rating_change": 0.45,
|
||||
"dividend_change": 0.45,
|
||||
"buyback": 0.45,
|
||||
"macro_policy": 0.35,
|
||||
"geopolitical": 0.30,
|
||||
"sector_rotation": 0.35,
|
||||
"unknown": 0.30,
|
||||
}
|
||||
|
||||
# Features used when returning conservative defaults
|
||||
_DEFAULT_FEATURES_USED = [
|
||||
"document_type_prior",
|
||||
"event_class_prior",
|
||||
]
|
||||
|
||||
|
||||
def get_default_confidence(
|
||||
document_type: str,
|
||||
event_class: str,
|
||||
) -> ConfidenceResult:
|
||||
"""Return a conservative confidence result for underrepresented classes.
|
||||
|
||||
Used when calibration data is insufficient for the given document type
|
||||
and event class combination. Returns conservative probabilities (0.3-0.5)
|
||||
and marks the result as under-calibrated.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
document_type
|
||||
The document type (news, filing, transcript, etc.).
|
||||
event_class
|
||||
The classified event type (earnings_beat, merger_acquisition, etc.).
|
||||
|
||||
Returns
|
||||
-------
|
||||
ConfidenceResult
|
||||
A conservative confidence result with under_calibrated=True.
|
||||
"""
|
||||
doc_default = _DOCUMENT_TYPE_DEFAULTS.get(
|
||||
document_type, _DOCUMENT_TYPE_DEFAULTS["unknown"]
|
||||
)
|
||||
event_default = _EVENT_CLASS_DEFAULTS.get(
|
||||
event_class, _EVENT_CLASS_DEFAULTS["unknown"]
|
||||
)
|
||||
|
||||
# Take the minimum of document and event defaults for extra conservatism
|
||||
probability = min(doc_default, event_default)
|
||||
|
||||
logger.debug(
|
||||
"Using conservative default confidence: doc_type=%s (%.2f), event=%s (%.2f) -> %.2f",
|
||||
document_type,
|
||||
doc_default,
|
||||
event_class,
|
||||
event_default,
|
||||
probability,
|
||||
)
|
||||
|
||||
return ConfidenceResult(
|
||||
probability=probability,
|
||||
features_used=_DEFAULT_FEATURES_USED,
|
||||
is_calibrated=False,
|
||||
under_calibrated=True,
|
||||
calibration_version="conservative-default-v1",
|
||||
)
|
||||
|
||||
|
||||
def is_underrepresented(
|
||||
document_type: str,
|
||||
event_class: str,
|
||||
min_samples: int = 30,
|
||||
known_counts: dict[tuple[str, str], int] | None = None,
|
||||
) -> bool:
|
||||
"""Check if a document_type + event_class combination is underrepresented.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
document_type
|
||||
The document type.
|
||||
event_class
|
||||
The event class.
|
||||
min_samples
|
||||
Minimum number of calibration samples to consider a class well-represented.
|
||||
known_counts
|
||||
Optional mapping of (doc_type, event_class) -> sample count.
|
||||
If None, treats any unknown combination as underrepresented.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
True if the class has insufficient calibration data.
|
||||
"""
|
||||
if known_counts is None:
|
||||
# Without explicit counts, use heuristic: unknown types are underrepresented
|
||||
if document_type not in _DOCUMENT_TYPE_DEFAULTS:
|
||||
return True
|
||||
if event_class not in _EVENT_CLASS_DEFAULTS:
|
||||
return True
|
||||
return False
|
||||
|
||||
key = (document_type, event_class)
|
||||
count = known_counts.get(key, 0)
|
||||
return count < min_samples
|
||||
@@ -0,0 +1,221 @@
|
||||
"""Confidence feature extraction from upstream pipeline stages.
|
||||
|
||||
Computes field-level features from extraction, resolution, evidence,
|
||||
sentiment, and cross-stage agreement to produce a ConfidenceFeatures
|
||||
vector for calibration or conservative defaults.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
from services.intelligence_pipeline_v3.confidence.models import ConfidenceFeatures
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtractionStageResult:
|
||||
"""Subset of extraction results relevant to confidence features.
|
||||
|
||||
This is an adapter interface — callers populate it from
|
||||
the full extraction/specialist output.
|
||||
"""
|
||||
|
||||
entity_scores: list[float]
|
||||
"""Per-entity confidence scores from specialist extractor."""
|
||||
|
||||
relation_scores: list[float]
|
||||
"""Per-relation confidence scores."""
|
||||
|
||||
total_facts: int
|
||||
"""Total facts extracted."""
|
||||
|
||||
valid_numeric_facts: int
|
||||
"""Facts that passed deterministic parser validation."""
|
||||
|
||||
populated_fields: int
|
||||
"""Schema fields that have values."""
|
||||
|
||||
expected_fields: int
|
||||
"""Total expected schema fields for this document type."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResolutionStageResult:
|
||||
"""Subset of resolution results relevant to confidence features."""
|
||||
|
||||
ambiguity_margins: list[float]
|
||||
"""Per-mention ambiguity margins (gap between top-2 candidates)."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvidenceStageResult:
|
||||
"""Subset of evidence verification results relevant to confidence features."""
|
||||
|
||||
total_claims: int
|
||||
"""Total extracted claims/facts."""
|
||||
|
||||
supported_claims: int
|
||||
"""Claims backed by valid evidence spans."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SentimentStageResult:
|
||||
"""Subset of sentiment results relevant to confidence features."""
|
||||
|
||||
max_class_probabilities: list[float]
|
||||
"""Per-company maximum class probability after calibration."""
|
||||
|
||||
calibration_version: str
|
||||
"""Version of sentiment calibration artifact used."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgreementStageResult:
|
||||
"""Cross-stage agreement analysis results."""
|
||||
|
||||
agreement_ratio: float
|
||||
"""Fraction of facts that agree across independent extraction paths."""
|
||||
|
||||
novelty_certainty: float
|
||||
"""Certainty of the novelty/duplicate classification (0-1)."""
|
||||
|
||||
hard_case_score: float
|
||||
"""Score indicating presence of known difficult patterns."""
|
||||
|
||||
|
||||
class ConfidenceFeatureExtractor:
|
||||
"""Extracts confidence features from upstream pipeline stage results.
|
||||
|
||||
Produces a normalized ConfidenceFeatures vector that can be passed
|
||||
to the calibrator or used to determine conservative defaults.
|
||||
"""
|
||||
|
||||
def extract_features(
|
||||
self,
|
||||
extraction_result: ExtractionStageResult,
|
||||
resolution_result: ResolutionStageResult,
|
||||
evidence_result: EvidenceStageResult,
|
||||
sentiment_result: SentimentStageResult,
|
||||
agreement_result: AgreementStageResult | None = None,
|
||||
document_type: str = "unknown",
|
||||
) -> ConfidenceFeatures:
|
||||
"""Compute confidence features from all upstream stage results.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
extraction_result
|
||||
Entity/relation/fact extraction outputs with scores.
|
||||
resolution_result
|
||||
Symbol resolution outputs with ambiguity margins.
|
||||
evidence_result
|
||||
Evidence verification outputs with coverage stats.
|
||||
sentiment_result
|
||||
Sentiment classification outputs with calibrated probabilities.
|
||||
agreement_result
|
||||
Optional cross-stage agreement analysis. Defaults used if None.
|
||||
document_type
|
||||
Document type string for type-specific calibration.
|
||||
|
||||
Returns
|
||||
-------
|
||||
ConfidenceFeatures
|
||||
Normalized feature vector ready for calibration.
|
||||
"""
|
||||
# Entity span score: average of entity scores, or 0 if none
|
||||
entity_span_score = (
|
||||
sum(extraction_result.entity_scores) / len(extraction_result.entity_scores)
|
||||
if extraction_result.entity_scores
|
||||
else 0.0
|
||||
)
|
||||
|
||||
# Alias resolution margin: average of per-mention margins
|
||||
alias_resolution_margin = (
|
||||
sum(resolution_result.ambiguity_margins)
|
||||
/ len(resolution_result.ambiguity_margins)
|
||||
if resolution_result.ambiguity_margins
|
||||
else 1.0 # No ambiguity if no mentions to resolve
|
||||
)
|
||||
|
||||
# Numeric parser validity: fraction of valid numeric facts
|
||||
numeric_parser_validity = (
|
||||
extraction_result.valid_numeric_facts / extraction_result.total_facts
|
||||
if extraction_result.total_facts > 0
|
||||
else 1.0 # No numeric facts = no parser failures
|
||||
)
|
||||
|
||||
# Evidence coverage: fraction of claims with valid evidence
|
||||
evidence_coverage = (
|
||||
evidence_result.supported_claims / evidence_result.total_claims
|
||||
if evidence_result.total_claims > 0
|
||||
else 0.0
|
||||
)
|
||||
|
||||
# Relation score: average relation confidence
|
||||
relation_score = (
|
||||
sum(extraction_result.relation_scores)
|
||||
/ len(extraction_result.relation_scores)
|
||||
if extraction_result.relation_scores
|
||||
else 0.0
|
||||
)
|
||||
|
||||
# Sentiment calibration confidence: average max class probability
|
||||
sentiment_calibration_confidence = (
|
||||
sum(sentiment_result.max_class_probabilities)
|
||||
/ len(sentiment_result.max_class_probabilities)
|
||||
if sentiment_result.max_class_probabilities
|
||||
else 0.5 # Neutral default when no sentiment data
|
||||
)
|
||||
|
||||
# Document completeness: fraction of expected fields populated
|
||||
document_completeness = (
|
||||
extraction_result.populated_fields / extraction_result.expected_fields
|
||||
if extraction_result.expected_fields > 0
|
||||
else 0.0
|
||||
)
|
||||
|
||||
# Cross-stage agreement features (use defaults if not provided)
|
||||
if agreement_result is not None:
|
||||
cross_stage_agreement = agreement_result.agreement_ratio
|
||||
duplicate_novelty_certainty = agreement_result.novelty_certainty
|
||||
known_hard_case_patterns = agreement_result.hard_case_score
|
||||
else:
|
||||
cross_stage_agreement = 0.5 # Neutral default
|
||||
duplicate_novelty_certainty = 0.5
|
||||
known_hard_case_patterns = 0.0
|
||||
|
||||
# Validate document type
|
||||
valid_types = {
|
||||
"news",
|
||||
"filing",
|
||||
"transcript",
|
||||
"press_release",
|
||||
"macro_event",
|
||||
"unknown",
|
||||
}
|
||||
if document_type not in valid_types:
|
||||
logger.warning(
|
||||
"Unknown document_type '%s', defaulting to 'unknown'", document_type
|
||||
)
|
||||
document_type = "unknown"
|
||||
|
||||
return ConfidenceFeatures(
|
||||
entity_span_score=_clamp(entity_span_score),
|
||||
alias_resolution_margin=_clamp(alias_resolution_margin),
|
||||
numeric_parser_validity=_clamp(numeric_parser_validity),
|
||||
evidence_coverage=_clamp(evidence_coverage),
|
||||
relation_score=_clamp(relation_score),
|
||||
sentiment_calibration_confidence=_clamp(sentiment_calibration_confidence),
|
||||
cross_stage_agreement=_clamp(cross_stage_agreement),
|
||||
duplicate_novelty_certainty=_clamp(duplicate_novelty_certainty),
|
||||
document_completeness=_clamp(document_completeness),
|
||||
document_type=document_type,
|
||||
known_hard_case_patterns=_clamp(known_hard_case_patterns),
|
||||
)
|
||||
|
||||
|
||||
def _clamp(value: float, low: float = 0.0, high: float = 1.0) -> float:
|
||||
"""Clamp value to [low, high]."""
|
||||
return max(low, min(high, value))
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Pydantic models for confidence calibration pipeline.
|
||||
|
||||
Defines feature vectors, calibration artifact metadata, and
|
||||
confidence results used throughout the confidence pipeline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class ConfidenceFeatures(BaseModel):
|
||||
"""Feature vector for confidence estimation.
|
||||
|
||||
Each feature is a normalized float derived from upstream pipeline
|
||||
stages: extraction, resolution, evidence verification, sentiment,
|
||||
and cross-stage agreement analysis.
|
||||
"""
|
||||
|
||||
entity_span_score: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Best entity span confidence from specialist extractor.",
|
||||
)
|
||||
alias_resolution_margin: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Gap between top-2 alias candidates. 1.0 = unambiguous.",
|
||||
)
|
||||
numeric_parser_validity: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Fraction of numeric facts that passed parser validation.",
|
||||
)
|
||||
evidence_coverage: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Fraction of extracted facts backed by valid evidence spans.",
|
||||
)
|
||||
relation_score: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Average confidence of extracted relations.",
|
||||
)
|
||||
sentiment_calibration_confidence: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Calibrated sentiment model confidence (max class probability).",
|
||||
)
|
||||
cross_stage_agreement: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Agreement ratio between independently derived facts across stages.",
|
||||
)
|
||||
duplicate_novelty_certainty: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Certainty of the novelty/duplicate classification.",
|
||||
)
|
||||
document_completeness: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Fraction of expected schema fields that were populated.",
|
||||
)
|
||||
document_type: str = Field(
|
||||
description="Document type (news, filing, transcript, press_release, macro_event).",
|
||||
)
|
||||
known_hard_case_patterns: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Score indicating presence of known hard patterns (multi-company, contradictions).",
|
||||
)
|
||||
|
||||
@field_validator("document_type")
|
||||
@classmethod
|
||||
def document_type_valid(cls, v: str) -> str:
|
||||
valid_types = {
|
||||
"news",
|
||||
"filing",
|
||||
"transcript",
|
||||
"press_release",
|
||||
"macro_event",
|
||||
"unknown",
|
||||
}
|
||||
if v not in valid_types:
|
||||
raise ValueError(f"document_type must be one of {valid_types}, got '{v}'")
|
||||
return v
|
||||
|
||||
def to_vector(self) -> list[float]:
|
||||
"""Convert features to a flat numeric vector for calibration models.
|
||||
|
||||
document_type is encoded as a categorical index.
|
||||
"""
|
||||
type_map = {
|
||||
"news": 0.0,
|
||||
"filing": 0.2,
|
||||
"transcript": 0.4,
|
||||
"press_release": 0.6,
|
||||
"macro_event": 0.8,
|
||||
"unknown": 1.0,
|
||||
}
|
||||
return [
|
||||
self.entity_span_score,
|
||||
self.alias_resolution_margin,
|
||||
self.numeric_parser_validity,
|
||||
self.evidence_coverage,
|
||||
self.relation_score,
|
||||
self.sentiment_calibration_confidence,
|
||||
self.cross_stage_agreement,
|
||||
self.duplicate_novelty_certainty,
|
||||
self.document_completeness,
|
||||
type_map.get(self.document_type, 1.0),
|
||||
self.known_hard_case_patterns,
|
||||
]
|
||||
|
||||
|
||||
class CalibrationArtifactMetadata(BaseModel):
|
||||
"""Metadata for a versioned calibration artifact.
|
||||
|
||||
Stored alongside the serialized calibrator to track provenance,
|
||||
training conditions, and quality metrics.
|
||||
"""
|
||||
|
||||
version: str = Field(description="Artifact version string (e.g., 'v1.0.0').")
|
||||
method: Literal["isotonic", "platt"] = Field(
|
||||
description="Calibration method used."
|
||||
)
|
||||
training_count: int = Field(
|
||||
ge=0, description="Number of samples used for training."
|
||||
)
|
||||
training_range: str = Field(
|
||||
description="Date range of training data (e.g., '2024-01-01 to 2024-06-30')."
|
||||
)
|
||||
ece: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Expected Calibration Error on held-out data.",
|
||||
)
|
||||
brier_score: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Brier score on held-out data.",
|
||||
)
|
||||
created_at: datetime = Field(
|
||||
default_factory=lambda: datetime.now(tz=timezone.utc),
|
||||
description="When the artifact was created.",
|
||||
)
|
||||
|
||||
|
||||
class ConfidenceResult(BaseModel):
|
||||
"""Final confidence output for an extraction record.
|
||||
|
||||
Contains the calibrated probability, feature breakdown, and
|
||||
metadata about whether calibration was applied or defaults used.
|
||||
"""
|
||||
|
||||
probability: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Calibrated probability of extraction correctness.",
|
||||
)
|
||||
features_used: list[str] = Field(
|
||||
description="Names of features that contributed to this confidence score.",
|
||||
)
|
||||
is_calibrated: bool = Field(
|
||||
default=True,
|
||||
description="Whether a trained calibrator was used (vs conservative default).",
|
||||
)
|
||||
under_calibrated: bool = Field(
|
||||
default=False,
|
||||
description="True if class has insufficient calibration data and conservative default was applied.",
|
||||
)
|
||||
calibration_version: str = Field(
|
||||
default="uncalibrated",
|
||||
description="Version of the calibration artifact used.",
|
||||
)
|
||||
Reference in New Issue
Block a user