feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,406 @@
|
||||
"""Confidence calibrator using isotonic or Platt scaling.
|
||||
|
||||
Maps confidence feature vectors to calibrated correctness probabilities.
|
||||
Supports training on held-out Gold_Corpus data, cross-validation for
|
||||
method comparison, and versioned artifact tracking.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Literal
|
||||
|
||||
import numpy as np
|
||||
|
||||
from services.intelligence_pipeline_v3.confidence.models import (
|
||||
CalibrationArtifactMetadata,
|
||||
ConfidenceFeatures,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_VERSION = "uncalibrated"
|
||||
|
||||
|
||||
class ConfidenceCalibrator:
|
||||
"""Calibrates confidence features to correctness probabilities.
|
||||
|
||||
Supports isotonic regression and Platt (logistic) scaling.
|
||||
The calibrator is fitted on labeled Gold_Corpus data where labels
|
||||
indicate whether the extraction was correct (True) or not (False).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
method
|
||||
Calibration method: "isotonic" for non-parametric monotone fit,
|
||||
"platt" for logistic regression scaling.
|
||||
"""
|
||||
|
||||
def __init__(self, method: Literal["isotonic", "platt"] = "isotonic") -> None:
|
||||
self._method: Literal["isotonic", "platt"] = method
|
||||
self._version: str = DEFAULT_VERSION
|
||||
self._fitted: bool = False
|
||||
self._model: object | None = None
|
||||
self._metadata: CalibrationArtifactMetadata | None = None
|
||||
self._training_count: int = 0
|
||||
|
||||
@property
|
||||
def method(self) -> str:
|
||||
"""Return the calibration method."""
|
||||
return self._method
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
"""Return the calibration artifact version."""
|
||||
return self._version
|
||||
|
||||
@property
|
||||
def is_fitted(self) -> bool:
|
||||
"""Return whether the calibrator has been fitted."""
|
||||
return self._fitted
|
||||
|
||||
@property
|
||||
def metadata(self) -> CalibrationArtifactMetadata | None:
|
||||
"""Return the artifact metadata if fitted."""
|
||||
return self._metadata
|
||||
|
||||
def fit(
|
||||
self,
|
||||
features: list[ConfidenceFeatures],
|
||||
labels: list[bool],
|
||||
method: str | None = None,
|
||||
version: str = "v1.0.0",
|
||||
training_range: str = "unknown",
|
||||
) -> None:
|
||||
"""Train the calibrator on labeled feature/correctness pairs.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
features
|
||||
List of confidence feature vectors from training data.
|
||||
labels
|
||||
True if the extraction was correct, False otherwise.
|
||||
method
|
||||
Override method for this fit (isotonic or platt).
|
||||
If None, uses the instance default.
|
||||
version
|
||||
Version string for the resulting artifact.
|
||||
training_range
|
||||
Description of the training data date range.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If features and labels have different lengths or are empty.
|
||||
"""
|
||||
if not features or not labels:
|
||||
raise ValueError("features and labels must not be empty")
|
||||
if len(features) != len(labels):
|
||||
raise ValueError(
|
||||
f"features ({len(features)}) and labels ({len(labels)}) must have the same length"
|
||||
)
|
||||
|
||||
if method is not None:
|
||||
if method not in ("isotonic", "platt"):
|
||||
raise ValueError(f"method must be 'isotonic' or 'platt', got '{method}'")
|
||||
self._method = method # type: ignore[assignment]
|
||||
|
||||
# Convert features to matrix
|
||||
X = np.array([f.to_vector() for f in features], dtype=np.float64)
|
||||
y = np.array(labels, dtype=np.float64)
|
||||
|
||||
if self._method == "isotonic":
|
||||
self._fit_isotonic(X, y)
|
||||
else:
|
||||
self._fit_platt(X, y)
|
||||
|
||||
self._version = version
|
||||
self._training_count = len(features)
|
||||
self._fitted = True
|
||||
|
||||
# Compute calibration quality on training data (for metadata)
|
||||
predictions = self._predict_batch(X)
|
||||
ece = _compute_ece(predictions, y)
|
||||
brier = _compute_brier(predictions, y)
|
||||
|
||||
self._metadata = CalibrationArtifactMetadata(
|
||||
version=version,
|
||||
method=self._method,
|
||||
training_count=len(features),
|
||||
training_range=training_range,
|
||||
ece=ece,
|
||||
brier_score=brier,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"ConfidenceCalibrator fitted: method=%s, n=%d, version=%s, ECE=%.4f, Brier=%.4f",
|
||||
self._method,
|
||||
len(features),
|
||||
version,
|
||||
ece,
|
||||
brier,
|
||||
)
|
||||
|
||||
def predict(self, features: ConfidenceFeatures) -> float:
|
||||
"""Return calibrated probability of extraction correctness.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
features
|
||||
Confidence feature vector for a single extraction.
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
Calibrated probability in [0, 1].
|
||||
"""
|
||||
if not self._fitted:
|
||||
# Return a neutral default when uncalibrated
|
||||
return 0.5
|
||||
|
||||
X = np.array([features.to_vector()], dtype=np.float64)
|
||||
predictions = self._predict_batch(X)
|
||||
return float(np.clip(predictions[0], 0.0, 1.0))
|
||||
|
||||
def predict_batch(self, features_list: list[ConfidenceFeatures]) -> list[float]:
|
||||
"""Return calibrated probabilities for a batch of feature vectors.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
features_list
|
||||
List of confidence feature vectors.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[float]
|
||||
Calibrated probabilities in [0, 1].
|
||||
"""
|
||||
if not self._fitted:
|
||||
return [0.5] * len(features_list)
|
||||
|
||||
X = np.array([f.to_vector() for f in features_list], dtype=np.float64)
|
||||
predictions = self._predict_batch(X)
|
||||
return [float(np.clip(p, 0.0, 1.0)) for p in predictions]
|
||||
|
||||
def evaluate(
|
||||
self,
|
||||
features: list[ConfidenceFeatures],
|
||||
labels: list[bool],
|
||||
) -> tuple[float, float]:
|
||||
"""Evaluate ECE and Brier score on held-out data.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
features
|
||||
Held-out feature vectors.
|
||||
labels
|
||||
True correctness labels.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple[float, float]
|
||||
(ECE, Brier_score) on the held-out set.
|
||||
"""
|
||||
if not features or not labels:
|
||||
raise ValueError("features and labels must not be empty")
|
||||
if len(features) != len(labels):
|
||||
raise ValueError("features and labels must have the same length")
|
||||
|
||||
X = np.array([f.to_vector() for f in features], dtype=np.float64)
|
||||
y = np.array(labels, dtype=np.float64)
|
||||
|
||||
if self._fitted:
|
||||
predictions = self._predict_batch(X)
|
||||
else:
|
||||
predictions = np.full(len(y), 0.5)
|
||||
|
||||
ece = _compute_ece(predictions, y)
|
||||
brier = _compute_brier(predictions, y)
|
||||
return ece, brier
|
||||
|
||||
def _fit_isotonic(self, X: np.ndarray, y: np.ndarray) -> None:
|
||||
"""Fit isotonic regression on aggregated feature scores."""
|
||||
from sklearn.isotonic import IsotonicRegression
|
||||
|
||||
# Aggregate features into a single score for isotonic monotone fit
|
||||
aggregated = X.mean(axis=1)
|
||||
iso = IsotonicRegression(y_min=0.0, y_max=1.0, out_of_bounds="clip")
|
||||
iso.fit(aggregated, y)
|
||||
self._model = iso
|
||||
|
||||
def _fit_platt(self, X: np.ndarray, y: np.ndarray) -> None:
|
||||
"""Fit logistic regression (Platt scaling) on the full feature vector."""
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
|
||||
y_int = y.astype(np.int32)
|
||||
if len(np.unique(y_int)) < 2:
|
||||
# Not enough class diversity — store a dummy model
|
||||
self._model = _ConstantPredictor(float(y.mean()))
|
||||
return
|
||||
|
||||
lr = LogisticRegression(solver="lbfgs", max_iter=1000, C=1.0)
|
||||
lr.fit(X, y_int)
|
||||
self._model = lr
|
||||
|
||||
def _predict_batch(self, X: np.ndarray) -> np.ndarray:
|
||||
"""Internal prediction dispatch."""
|
||||
if self._model is None:
|
||||
return np.full(X.shape[0], 0.5)
|
||||
|
||||
if self._method == "isotonic":
|
||||
# Isotonic uses aggregated score
|
||||
aggregated = X.mean(axis=1)
|
||||
return self._model.predict(aggregated) # type: ignore[union-attr]
|
||||
else:
|
||||
# Platt uses full feature vector
|
||||
if isinstance(self._model, _ConstantPredictor):
|
||||
return self._model.predict(X)
|
||||
return self._model.predict_proba(X)[:, 1] # type: ignore[union-attr]
|
||||
|
||||
|
||||
class _ConstantPredictor:
|
||||
"""Fallback predictor when training data has only one class."""
|
||||
|
||||
def __init__(self, value: float) -> None:
|
||||
self._value = value
|
||||
|
||||
def predict(self, X: np.ndarray) -> np.ndarray:
|
||||
return np.full(X.shape[0], self._value)
|
||||
|
||||
|
||||
def _compute_ece(
|
||||
predictions: np.ndarray,
|
||||
labels: np.ndarray,
|
||||
n_bins: int = 10,
|
||||
) -> float:
|
||||
"""Compute Expected Calibration Error.
|
||||
|
||||
Partitions predictions into equal-width bins and computes the
|
||||
weighted average of |avg_predicted - avg_actual| per bin.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
predictions
|
||||
Predicted probabilities.
|
||||
labels
|
||||
True binary labels (0 or 1).
|
||||
n_bins
|
||||
Number of equal-width bins.
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
ECE value in [0, 1].
|
||||
"""
|
||||
if len(predictions) == 0:
|
||||
return 0.0
|
||||
|
||||
bin_boundaries = np.linspace(0.0, 1.0, n_bins + 1)
|
||||
ece = 0.0
|
||||
n = len(predictions)
|
||||
|
||||
for i in range(n_bins):
|
||||
lower = bin_boundaries[i]
|
||||
upper = bin_boundaries[i + 1]
|
||||
|
||||
if i == n_bins - 1:
|
||||
# Include right boundary in last bin
|
||||
mask = (predictions >= lower) & (predictions <= upper)
|
||||
else:
|
||||
mask = (predictions >= lower) & (predictions < upper)
|
||||
|
||||
bin_count = mask.sum()
|
||||
if bin_count == 0:
|
||||
continue
|
||||
|
||||
avg_predicted = predictions[mask].mean()
|
||||
avg_actual = labels[mask].mean()
|
||||
ece += (bin_count / n) * abs(avg_predicted - avg_actual)
|
||||
|
||||
return float(ece)
|
||||
|
||||
|
||||
def _compute_brier(predictions: np.ndarray, labels: np.ndarray) -> float:
|
||||
"""Compute Brier score (mean squared error of probability predictions).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
predictions
|
||||
Predicted probabilities.
|
||||
labels
|
||||
True binary labels (0 or 1).
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
Brier score in [0, 1].
|
||||
"""
|
||||
if len(predictions) == 0:
|
||||
return 0.0
|
||||
return float(np.mean((predictions - labels) ** 2))
|
||||
|
||||
|
||||
def compare_methods(
|
||||
features: list[ConfidenceFeatures],
|
||||
labels: list[bool],
|
||||
n_folds: int = 5,
|
||||
) -> dict[str, dict[str, float]]:
|
||||
"""Compare isotonic and Platt methods using k-fold cross-validation.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
features
|
||||
Full set of training features.
|
||||
labels
|
||||
Full set of correctness labels.
|
||||
n_folds
|
||||
Number of cross-validation folds.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict
|
||||
Mapping of method name to {"ece": float, "brier": float} averages.
|
||||
"""
|
||||
if len(features) < n_folds * 2:
|
||||
raise ValueError(
|
||||
f"Need at least {n_folds * 2} samples for {n_folds}-fold CV, got {len(features)}"
|
||||
)
|
||||
|
||||
results: dict[str, list[tuple[float, float]]] = {
|
||||
"isotonic": [],
|
||||
"platt": [],
|
||||
}
|
||||
|
||||
indices = np.arange(len(features))
|
||||
fold_size = len(features) // n_folds
|
||||
|
||||
for fold in range(n_folds):
|
||||
val_start = fold * fold_size
|
||||
val_end = val_start + fold_size if fold < n_folds - 1 else len(features)
|
||||
|
||||
val_indices = indices[val_start:val_end]
|
||||
train_indices = np.concatenate([indices[:val_start], indices[val_end:]])
|
||||
|
||||
train_features = [features[i] for i in train_indices]
|
||||
train_labels = [labels[i] for i in train_indices]
|
||||
val_features = [features[i] for i in val_indices]
|
||||
val_labels = [labels[i] for i in val_indices]
|
||||
|
||||
for method_name in ("isotonic", "platt"):
|
||||
cal = ConfidenceCalibrator(method=method_name) # type: ignore[arg-type]
|
||||
cal.fit(
|
||||
train_features,
|
||||
train_labels,
|
||||
version=f"cv-fold-{fold}",
|
||||
training_range="cross-validation",
|
||||
)
|
||||
ece, brier = cal.evaluate(val_features, val_labels)
|
||||
results[method_name].append((ece, brier))
|
||||
|
||||
return {
|
||||
method: {
|
||||
"ece": float(np.mean([r[0] for r in scores])),
|
||||
"brier": float(np.mean([r[1] for r in scores])),
|
||||
}
|
||||
for method, scores in results.items()
|
||||
}
|
||||
Reference in New Issue
Block a user