feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,459 @@
|
||||
"""Numeric exact/tolerance-aware matching metrics for extracted financial facts.
|
||||
|
||||
Implements evaluation metrics for numeric extraction quality against a gold
|
||||
standard corpus. Supports exact match, default 5% tolerance, and configurable
|
||||
tolerance matching. Provides per-fact-type breakdowns, unit consistency
|
||||
checks, and period matching.
|
||||
|
||||
Input model fields: fact_type, predicate, literal_value, normalized_value,
|
||||
unit, period, evidence_ids.
|
||||
|
||||
Validates: Requirements 16.3, 16.4
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Domain Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FactType(str, Enum):
|
||||
"""Known financial fact types for per-type breakdown."""
|
||||
|
||||
eps = "eps"
|
||||
revenue = "revenue"
|
||||
percentage_change = "percentage_change"
|
||||
price_target = "price_target"
|
||||
guidance = "guidance"
|
||||
dividend = "dividend"
|
||||
margin = "margin"
|
||||
growth_rate = "growth_rate"
|
||||
other = "other"
|
||||
|
||||
|
||||
class NumericFact(BaseModel):
|
||||
"""A single extracted numeric fact with normalization and context."""
|
||||
|
||||
fact_type: str
|
||||
predicate: str
|
||||
literal_value: str
|
||||
normalized_value: float | None = None
|
||||
unit: str | None = None
|
||||
period: str | None = None
|
||||
evidence_ids: list[str] = Field(default_factory=list)
|
||||
document_id: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class NumericMatchResult(BaseModel):
|
||||
"""Result of matching a single predicted fact against gold."""
|
||||
|
||||
exact_match: bool = False
|
||||
within_tolerance: bool = False
|
||||
tolerance_pct: float = 0.0
|
||||
unit_consistent: bool = True
|
||||
period_match: bool = True
|
||||
absolute_error: float | None = None
|
||||
relative_error_pct: float | None = None
|
||||
|
||||
|
||||
class AccuracyMetric(BaseModel):
|
||||
"""Simple accuracy metric with support count."""
|
||||
|
||||
accuracy: float = Field(ge=0.0, le=1.0)
|
||||
matches: int = Field(ge=0)
|
||||
total: int = Field(ge=0)
|
||||
|
||||
|
||||
class ToleranceDistribution(BaseModel):
|
||||
"""Distribution of relative errors across tolerance buckets."""
|
||||
|
||||
exact: int = Field(ge=0, default=0)
|
||||
within_1pct: int = Field(ge=0, default=0)
|
||||
within_5pct: int = Field(ge=0, default=0)
|
||||
within_10pct: int = Field(ge=0, default=0)
|
||||
beyond_10pct: int = Field(ge=0, default=0)
|
||||
not_comparable: int = Field(ge=0, default=0)
|
||||
|
||||
|
||||
class ErrorCategory(str, Enum):
|
||||
"""Common numeric extraction error categories."""
|
||||
|
||||
unit_mismatch = "unit_mismatch"
|
||||
period_mismatch = "period_mismatch"
|
||||
magnitude_error = "magnitude_error"
|
||||
sign_error = "sign_error"
|
||||
parsing_failure = "parsing_failure"
|
||||
missing_value = "missing_value"
|
||||
|
||||
|
||||
class ErrorBreakdown(BaseModel):
|
||||
"""Counts of errors by category."""
|
||||
|
||||
counts: dict[str, int] = Field(default_factory=dict)
|
||||
total_errors: int = Field(ge=0, default=0)
|
||||
|
||||
|
||||
class NumericEvaluationReport(BaseModel):
|
||||
"""Complete numeric extraction evaluation report."""
|
||||
|
||||
exact_match_accuracy: AccuracyMetric
|
||||
tolerance_accuracy: AccuracyMetric
|
||||
tolerance_pct_used: float = Field(ge=0.0)
|
||||
per_type_exact: dict[str, AccuracyMetric] = Field(default_factory=dict)
|
||||
per_type_tolerance: dict[str, AccuracyMetric] = Field(default_factory=dict)
|
||||
unit_consistency: AccuracyMetric
|
||||
period_match: AccuracyMetric
|
||||
tolerance_distribution: ToleranceDistribution
|
||||
error_breakdown: ErrorBreakdown
|
||||
document_count: int = Field(ge=0, default=0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Matching Logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DEFAULT_TOLERANCE_PCT = 5.0
|
||||
|
||||
|
||||
def _is_exact_match(pred_value: float, gold_value: float) -> bool:
|
||||
"""Check if predicted value exactly equals gold value (within float epsilon)."""
|
||||
return abs(pred_value - gold_value) < 1e-9
|
||||
|
||||
|
||||
def _is_within_tolerance(
|
||||
pred_value: float, gold_value: float, tolerance_pct: float
|
||||
) -> bool:
|
||||
"""Check if predicted value is within ±tolerance_pct of gold value.
|
||||
|
||||
For zero gold values, uses absolute comparison with a small epsilon
|
||||
derived from the tolerance percentage.
|
||||
"""
|
||||
if abs(gold_value) < 1e-12:
|
||||
# For zero gold, allow small absolute tolerance
|
||||
return abs(pred_value) < tolerance_pct / 100.0
|
||||
threshold = abs(gold_value) * (tolerance_pct / 100.0)
|
||||
return abs(pred_value - gold_value) <= threshold
|
||||
|
||||
|
||||
def _compute_relative_error_pct(pred_value: float, gold_value: float) -> float | None:
|
||||
"""Compute relative error as a percentage of gold value.
|
||||
|
||||
Returns None if gold value is zero (relative error undefined).
|
||||
"""
|
||||
if abs(gold_value) < 1e-12:
|
||||
return None
|
||||
return abs(pred_value - gold_value) / abs(gold_value) * 100.0
|
||||
|
||||
|
||||
def _classify_error(
|
||||
pred: NumericFact, gold: NumericFact, pred_value: float | None, gold_value: float
|
||||
) -> str | None:
|
||||
"""Classify the type of error for a mismatched prediction."""
|
||||
if pred_value is None:
|
||||
if pred.normalized_value is None:
|
||||
return ErrorCategory.parsing_failure.value
|
||||
return ErrorCategory.missing_value.value
|
||||
|
||||
# Check sign error (opposite signs, both non-zero)
|
||||
if pred_value * gold_value < 0 and abs(pred_value) > 1e-9 and abs(gold_value) > 1e-9:
|
||||
return ErrorCategory.sign_error.value
|
||||
|
||||
# Check magnitude error (off by factor of 10+)
|
||||
if abs(gold_value) > 1e-9:
|
||||
ratio = abs(pred_value / gold_value)
|
||||
if ratio >= 10.0 or ratio <= 0.1:
|
||||
return ErrorCategory.magnitude_error.value
|
||||
|
||||
# Unit mismatch (if units don't match)
|
||||
if pred.unit and gold.unit and pred.unit != gold.unit:
|
||||
return ErrorCategory.unit_mismatch.value
|
||||
|
||||
# Period mismatch
|
||||
if pred.period and gold.period and pred.period != gold.period:
|
||||
return ErrorCategory.period_mismatch.value
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _bucket_relative_error(relative_error_pct: float | None) -> str:
|
||||
"""Assign a relative error to a tolerance bucket name."""
|
||||
if relative_error_pct is None:
|
||||
return "not_comparable"
|
||||
if relative_error_pct < 1e-7:
|
||||
return "exact"
|
||||
if relative_error_pct <= 1.0:
|
||||
return "within_1pct"
|
||||
if relative_error_pct <= 5.0:
|
||||
return "within_5pct"
|
||||
if relative_error_pct <= 10.0:
|
||||
return "within_10pct"
|
||||
return "beyond_10pct"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Single Fact Matching
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def match_numeric_fact(
|
||||
pred: NumericFact,
|
||||
gold: NumericFact,
|
||||
tolerance_pct: float = DEFAULT_TOLERANCE_PCT,
|
||||
) -> NumericMatchResult:
|
||||
"""Match a predicted numeric fact against a gold standard fact.
|
||||
|
||||
Compares normalized values, checks unit consistency and period match.
|
||||
|
||||
Args:
|
||||
pred: Predicted numeric fact.
|
||||
gold: Gold standard numeric fact.
|
||||
tolerance_pct: Tolerance percentage for approximate matching.
|
||||
|
||||
Returns:
|
||||
NumericMatchResult with match details.
|
||||
"""
|
||||
# Unit consistency check
|
||||
unit_consistent = True
|
||||
if pred.unit is not None and gold.unit is not None:
|
||||
unit_consistent = pred.unit == gold.unit
|
||||
elif pred.unit is None and gold.unit is not None:
|
||||
unit_consistent = False
|
||||
# If gold has no unit, we consider it consistent regardless
|
||||
|
||||
# Period match check
|
||||
period_match = True
|
||||
if pred.period is not None and gold.period is not None:
|
||||
period_match = pred.period == gold.period
|
||||
elif pred.period is None and gold.period is not None:
|
||||
period_match = False
|
||||
|
||||
# Value comparison
|
||||
pred_value = pred.normalized_value
|
||||
gold_value = gold.normalized_value
|
||||
|
||||
if pred_value is None or gold_value is None:
|
||||
return NumericMatchResult(
|
||||
exact_match=False,
|
||||
within_tolerance=False,
|
||||
tolerance_pct=tolerance_pct,
|
||||
unit_consistent=unit_consistent,
|
||||
period_match=period_match,
|
||||
absolute_error=None,
|
||||
relative_error_pct=None,
|
||||
)
|
||||
|
||||
absolute_error = abs(pred_value - gold_value)
|
||||
relative_error_pct = _compute_relative_error_pct(pred_value, gold_value)
|
||||
exact = _is_exact_match(pred_value, gold_value)
|
||||
within_tol = _is_within_tolerance(pred_value, gold_value, tolerance_pct)
|
||||
|
||||
return NumericMatchResult(
|
||||
exact_match=exact,
|
||||
within_tolerance=within_tol,
|
||||
tolerance_pct=tolerance_pct,
|
||||
unit_consistent=unit_consistent,
|
||||
period_match=period_match,
|
||||
absolute_error=absolute_error,
|
||||
relative_error_pct=relative_error_pct,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Batch Evaluation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _align_facts(
|
||||
predicted: list[NumericFact],
|
||||
gold: list[NumericFact],
|
||||
) -> list[tuple[NumericFact, NumericFact]]:
|
||||
"""Align predicted facts to gold facts using greedy matching.
|
||||
|
||||
Matches on fact_type and predicate. Each gold fact can match at most
|
||||
one predicted fact.
|
||||
"""
|
||||
pairs: list[tuple[NumericFact, NumericFact]] = []
|
||||
matched_gold: set[int] = set()
|
||||
|
||||
for pred in predicted:
|
||||
for g_idx, g in enumerate(gold):
|
||||
if g_idx in matched_gold:
|
||||
continue
|
||||
if pred.fact_type == g.fact_type and pred.predicate == g.predicate:
|
||||
pairs.append((pred, g))
|
||||
matched_gold.add(g_idx)
|
||||
break
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def evaluate_numeric_facts(
|
||||
predicted: list[NumericFact],
|
||||
gold: list[NumericFact],
|
||||
tolerance_pct: float = DEFAULT_TOLERANCE_PCT,
|
||||
document_count: int = 1,
|
||||
) -> NumericEvaluationReport:
|
||||
"""Run full numeric extraction evaluation.
|
||||
|
||||
Aligns predicted facts to gold facts by fact_type and predicate,
|
||||
then computes exact match accuracy, tolerance-based accuracy,
|
||||
per-type breakdowns, unit consistency, period match accuracy,
|
||||
tolerance distribution histogram, and error categories.
|
||||
|
||||
Args:
|
||||
predicted: All predicted numeric facts.
|
||||
gold: All gold standard numeric facts.
|
||||
tolerance_pct: Tolerance percentage for approximate matching.
|
||||
document_count: Number of documents evaluated.
|
||||
|
||||
Returns:
|
||||
NumericEvaluationReport with complete evaluation results.
|
||||
"""
|
||||
pairs = _align_facts(predicted, gold)
|
||||
total_aligned = len(pairs)
|
||||
|
||||
# Track results
|
||||
exact_matches = 0
|
||||
tolerance_matches = 0
|
||||
unit_matches = 0
|
||||
period_matches = 0
|
||||
comparable_count = 0
|
||||
|
||||
# Per-type tracking
|
||||
per_type_exact_counts: dict[str, tuple[int, int]] = {} # type -> (matches, total)
|
||||
per_type_tol_counts: dict[str, tuple[int, int]] = {}
|
||||
|
||||
# Tolerance distribution
|
||||
dist = ToleranceDistribution()
|
||||
|
||||
# Error tracking
|
||||
error_counts: dict[str, int] = {}
|
||||
|
||||
for pred, g in pairs:
|
||||
result = match_numeric_fact(pred, g, tolerance_pct)
|
||||
|
||||
# Unit consistency
|
||||
if result.unit_consistent:
|
||||
unit_matches += 1
|
||||
|
||||
# Period match
|
||||
if result.period_match:
|
||||
period_matches += 1
|
||||
|
||||
# Only count value comparisons when both values exist
|
||||
if pred.normalized_value is not None and g.normalized_value is not None:
|
||||
comparable_count += 1
|
||||
|
||||
if result.exact_match:
|
||||
exact_matches += 1
|
||||
if result.within_tolerance:
|
||||
tolerance_matches += 1
|
||||
|
||||
# Per-type tracking
|
||||
ft = pred.fact_type
|
||||
ex_m, ex_t = per_type_exact_counts.get(ft, (0, 0))
|
||||
tol_m, tol_t = per_type_tol_counts.get(ft, (0, 0))
|
||||
per_type_exact_counts[ft] = (
|
||||
ex_m + (1 if result.exact_match else 0),
|
||||
ex_t + 1,
|
||||
)
|
||||
per_type_tol_counts[ft] = (
|
||||
tol_m + (1 if result.within_tolerance else 0),
|
||||
tol_t + 1,
|
||||
)
|
||||
|
||||
# Tolerance distribution
|
||||
bucket = _bucket_relative_error(result.relative_error_pct)
|
||||
if bucket == "exact":
|
||||
dist.exact += 1
|
||||
elif bucket == "within_1pct":
|
||||
dist.within_1pct += 1
|
||||
elif bucket == "within_5pct":
|
||||
dist.within_5pct += 1
|
||||
elif bucket == "within_10pct":
|
||||
dist.within_10pct += 1
|
||||
elif bucket == "beyond_10pct":
|
||||
dist.beyond_10pct += 1
|
||||
else:
|
||||
dist.not_comparable += 1
|
||||
|
||||
# Error classification for non-exact matches
|
||||
if not result.exact_match:
|
||||
error_cat = _classify_error(pred, g, pred.normalized_value, g.normalized_value)
|
||||
if error_cat:
|
||||
error_counts[error_cat] = error_counts.get(error_cat, 0) + 1
|
||||
else:
|
||||
dist.not_comparable += 1
|
||||
# Classify missing value error
|
||||
if pred.normalized_value is None:
|
||||
error_cat = ErrorCategory.parsing_failure.value
|
||||
elif g.normalized_value is None:
|
||||
error_cat = ErrorCategory.missing_value.value
|
||||
else:
|
||||
error_cat = None
|
||||
if error_cat:
|
||||
error_counts[error_cat] = error_counts.get(error_cat, 0) + 1
|
||||
|
||||
# Build accuracy metrics
|
||||
exact_accuracy = AccuracyMetric(
|
||||
accuracy=exact_matches / comparable_count if comparable_count > 0 else 1.0,
|
||||
matches=exact_matches,
|
||||
total=comparable_count,
|
||||
)
|
||||
tolerance_accuracy = AccuracyMetric(
|
||||
accuracy=tolerance_matches / comparable_count if comparable_count > 0 else 1.0,
|
||||
matches=tolerance_matches,
|
||||
total=comparable_count,
|
||||
)
|
||||
unit_consistency = AccuracyMetric(
|
||||
accuracy=unit_matches / total_aligned if total_aligned > 0 else 1.0,
|
||||
matches=unit_matches,
|
||||
total=total_aligned,
|
||||
)
|
||||
period_match_metric = AccuracyMetric(
|
||||
accuracy=period_matches / total_aligned if total_aligned > 0 else 1.0,
|
||||
matches=period_matches,
|
||||
total=total_aligned,
|
||||
)
|
||||
|
||||
# Per-type exact accuracy
|
||||
per_type_exact: dict[str, AccuracyMetric] = {}
|
||||
for ft, (m, t) in sorted(per_type_exact_counts.items()):
|
||||
per_type_exact[ft] = AccuracyMetric(
|
||||
accuracy=m / t if t > 0 else 1.0,
|
||||
matches=m,
|
||||
total=t,
|
||||
)
|
||||
|
||||
# Per-type tolerance accuracy
|
||||
per_type_tolerance: dict[str, AccuracyMetric] = {}
|
||||
for ft, (m, t) in sorted(per_type_tol_counts.items()):
|
||||
per_type_tolerance[ft] = AccuracyMetric(
|
||||
accuracy=m / t if t > 0 else 1.0,
|
||||
matches=m,
|
||||
total=t,
|
||||
)
|
||||
|
||||
total_errors = sum(error_counts.values())
|
||||
|
||||
return NumericEvaluationReport(
|
||||
exact_match_accuracy=exact_accuracy,
|
||||
tolerance_accuracy=tolerance_accuracy,
|
||||
tolerance_pct_used=tolerance_pct,
|
||||
per_type_exact=per_type_exact,
|
||||
per_type_tolerance=per_type_tolerance,
|
||||
unit_consistency=unit_consistency,
|
||||
period_match=period_match_metric,
|
||||
tolerance_distribution=dist,
|
||||
error_breakdown=ErrorBreakdown(counts=error_counts, total_errors=total_errors),
|
||||
document_count=document_count,
|
||||
)
|
||||
Reference in New Issue
Block a user