Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
460 lines
15 KiB
Python
460 lines
15 KiB
Python
"""Numeric exact/tolerance-aware matching metrics for extracted financial facts.
|
|
|
|
Implements evaluation metrics for numeric extraction quality against a gold
|
|
standard corpus. Supports exact match, default 5% tolerance, and configurable
|
|
tolerance matching. Provides per-fact-type breakdowns, unit consistency
|
|
checks, and period matching.
|
|
|
|
Input model fields: fact_type, predicate, literal_value, normalized_value,
|
|
unit, period, evidence_ids.
|
|
|
|
Validates: Requirements 16.3, 16.4
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from enum import Enum
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Domain Models
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class FactType(str, Enum):
|
|
"""Known financial fact types for per-type breakdown."""
|
|
|
|
eps = "eps"
|
|
revenue = "revenue"
|
|
percentage_change = "percentage_change"
|
|
price_target = "price_target"
|
|
guidance = "guidance"
|
|
dividend = "dividend"
|
|
margin = "margin"
|
|
growth_rate = "growth_rate"
|
|
other = "other"
|
|
|
|
|
|
class NumericFact(BaseModel):
|
|
"""A single extracted numeric fact with normalization and context."""
|
|
|
|
fact_type: str
|
|
predicate: str
|
|
literal_value: str
|
|
normalized_value: float | None = None
|
|
unit: str | None = None
|
|
period: str | None = None
|
|
evidence_ids: list[str] = Field(default_factory=list)
|
|
document_id: str = ""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Result Models
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class NumericMatchResult(BaseModel):
|
|
"""Result of matching a single predicted fact against gold."""
|
|
|
|
exact_match: bool = False
|
|
within_tolerance: bool = False
|
|
tolerance_pct: float = 0.0
|
|
unit_consistent: bool = True
|
|
period_match: bool = True
|
|
absolute_error: float | None = None
|
|
relative_error_pct: float | None = None
|
|
|
|
|
|
class AccuracyMetric(BaseModel):
|
|
"""Simple accuracy metric with support count."""
|
|
|
|
accuracy: float = Field(ge=0.0, le=1.0)
|
|
matches: int = Field(ge=0)
|
|
total: int = Field(ge=0)
|
|
|
|
|
|
class ToleranceDistribution(BaseModel):
|
|
"""Distribution of relative errors across tolerance buckets."""
|
|
|
|
exact: int = Field(ge=0, default=0)
|
|
within_1pct: int = Field(ge=0, default=0)
|
|
within_5pct: int = Field(ge=0, default=0)
|
|
within_10pct: int = Field(ge=0, default=0)
|
|
beyond_10pct: int = Field(ge=0, default=0)
|
|
not_comparable: int = Field(ge=0, default=0)
|
|
|
|
|
|
class ErrorCategory(str, Enum):
|
|
"""Common numeric extraction error categories."""
|
|
|
|
unit_mismatch = "unit_mismatch"
|
|
period_mismatch = "period_mismatch"
|
|
magnitude_error = "magnitude_error"
|
|
sign_error = "sign_error"
|
|
parsing_failure = "parsing_failure"
|
|
missing_value = "missing_value"
|
|
|
|
|
|
class ErrorBreakdown(BaseModel):
|
|
"""Counts of errors by category."""
|
|
|
|
counts: dict[str, int] = Field(default_factory=dict)
|
|
total_errors: int = Field(ge=0, default=0)
|
|
|
|
|
|
class NumericEvaluationReport(BaseModel):
|
|
"""Complete numeric extraction evaluation report."""
|
|
|
|
exact_match_accuracy: AccuracyMetric
|
|
tolerance_accuracy: AccuracyMetric
|
|
tolerance_pct_used: float = Field(ge=0.0)
|
|
per_type_exact: dict[str, AccuracyMetric] = Field(default_factory=dict)
|
|
per_type_tolerance: dict[str, AccuracyMetric] = Field(default_factory=dict)
|
|
unit_consistency: AccuracyMetric
|
|
period_match: AccuracyMetric
|
|
tolerance_distribution: ToleranceDistribution
|
|
error_breakdown: ErrorBreakdown
|
|
document_count: int = Field(ge=0, default=0)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Matching Logic
|
|
# ---------------------------------------------------------------------------
|
|
|
|
DEFAULT_TOLERANCE_PCT = 5.0
|
|
|
|
|
|
def _is_exact_match(pred_value: float, gold_value: float) -> bool:
|
|
"""Check if predicted value exactly equals gold value (within float epsilon)."""
|
|
return abs(pred_value - gold_value) < 1e-9
|
|
|
|
|
|
def _is_within_tolerance(
|
|
pred_value: float, gold_value: float, tolerance_pct: float
|
|
) -> bool:
|
|
"""Check if predicted value is within ±tolerance_pct of gold value.
|
|
|
|
For zero gold values, uses absolute comparison with a small epsilon
|
|
derived from the tolerance percentage.
|
|
"""
|
|
if abs(gold_value) < 1e-12:
|
|
# For zero gold, allow small absolute tolerance
|
|
return abs(pred_value) < tolerance_pct / 100.0
|
|
threshold = abs(gold_value) * (tolerance_pct / 100.0)
|
|
return abs(pred_value - gold_value) <= threshold
|
|
|
|
|
|
def _compute_relative_error_pct(pred_value: float, gold_value: float) -> float | None:
|
|
"""Compute relative error as a percentage of gold value.
|
|
|
|
Returns None if gold value is zero (relative error undefined).
|
|
"""
|
|
if abs(gold_value) < 1e-12:
|
|
return None
|
|
return abs(pred_value - gold_value) / abs(gold_value) * 100.0
|
|
|
|
|
|
def _classify_error(
|
|
pred: NumericFact, gold: NumericFact, pred_value: float | None, gold_value: float
|
|
) -> str | None:
|
|
"""Classify the type of error for a mismatched prediction."""
|
|
if pred_value is None:
|
|
if pred.normalized_value is None:
|
|
return ErrorCategory.parsing_failure.value
|
|
return ErrorCategory.missing_value.value
|
|
|
|
# Check sign error (opposite signs, both non-zero)
|
|
if pred_value * gold_value < 0 and abs(pred_value) > 1e-9 and abs(gold_value) > 1e-9:
|
|
return ErrorCategory.sign_error.value
|
|
|
|
# Check magnitude error (off by factor of 10+)
|
|
if abs(gold_value) > 1e-9:
|
|
ratio = abs(pred_value / gold_value)
|
|
if ratio >= 10.0 or ratio <= 0.1:
|
|
return ErrorCategory.magnitude_error.value
|
|
|
|
# Unit mismatch (if units don't match)
|
|
if pred.unit and gold.unit and pred.unit != gold.unit:
|
|
return ErrorCategory.unit_mismatch.value
|
|
|
|
# Period mismatch
|
|
if pred.period and gold.period and pred.period != gold.period:
|
|
return ErrorCategory.period_mismatch.value
|
|
|
|
return None
|
|
|
|
|
|
def _bucket_relative_error(relative_error_pct: float | None) -> str:
|
|
"""Assign a relative error to a tolerance bucket name."""
|
|
if relative_error_pct is None:
|
|
return "not_comparable"
|
|
if relative_error_pct < 1e-7:
|
|
return "exact"
|
|
if relative_error_pct <= 1.0:
|
|
return "within_1pct"
|
|
if relative_error_pct <= 5.0:
|
|
return "within_5pct"
|
|
if relative_error_pct <= 10.0:
|
|
return "within_10pct"
|
|
return "beyond_10pct"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Single Fact Matching
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def match_numeric_fact(
|
|
pred: NumericFact,
|
|
gold: NumericFact,
|
|
tolerance_pct: float = DEFAULT_TOLERANCE_PCT,
|
|
) -> NumericMatchResult:
|
|
"""Match a predicted numeric fact against a gold standard fact.
|
|
|
|
Compares normalized values, checks unit consistency and period match.
|
|
|
|
Args:
|
|
pred: Predicted numeric fact.
|
|
gold: Gold standard numeric fact.
|
|
tolerance_pct: Tolerance percentage for approximate matching.
|
|
|
|
Returns:
|
|
NumericMatchResult with match details.
|
|
"""
|
|
# Unit consistency check
|
|
unit_consistent = True
|
|
if pred.unit is not None and gold.unit is not None:
|
|
unit_consistent = pred.unit == gold.unit
|
|
elif pred.unit is None and gold.unit is not None:
|
|
unit_consistent = False
|
|
# If gold has no unit, we consider it consistent regardless
|
|
|
|
# Period match check
|
|
period_match = True
|
|
if pred.period is not None and gold.period is not None:
|
|
period_match = pred.period == gold.period
|
|
elif pred.period is None and gold.period is not None:
|
|
period_match = False
|
|
|
|
# Value comparison
|
|
pred_value = pred.normalized_value
|
|
gold_value = gold.normalized_value
|
|
|
|
if pred_value is None or gold_value is None:
|
|
return NumericMatchResult(
|
|
exact_match=False,
|
|
within_tolerance=False,
|
|
tolerance_pct=tolerance_pct,
|
|
unit_consistent=unit_consistent,
|
|
period_match=period_match,
|
|
absolute_error=None,
|
|
relative_error_pct=None,
|
|
)
|
|
|
|
absolute_error = abs(pred_value - gold_value)
|
|
relative_error_pct = _compute_relative_error_pct(pred_value, gold_value)
|
|
exact = _is_exact_match(pred_value, gold_value)
|
|
within_tol = _is_within_tolerance(pred_value, gold_value, tolerance_pct)
|
|
|
|
return NumericMatchResult(
|
|
exact_match=exact,
|
|
within_tolerance=within_tol,
|
|
tolerance_pct=tolerance_pct,
|
|
unit_consistent=unit_consistent,
|
|
period_match=period_match,
|
|
absolute_error=absolute_error,
|
|
relative_error_pct=relative_error_pct,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Batch Evaluation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _align_facts(
|
|
predicted: list[NumericFact],
|
|
gold: list[NumericFact],
|
|
) -> list[tuple[NumericFact, NumericFact]]:
|
|
"""Align predicted facts to gold facts using greedy matching.
|
|
|
|
Matches on fact_type and predicate. Each gold fact can match at most
|
|
one predicted fact.
|
|
"""
|
|
pairs: list[tuple[NumericFact, NumericFact]] = []
|
|
matched_gold: set[int] = set()
|
|
|
|
for pred in predicted:
|
|
for g_idx, g in enumerate(gold):
|
|
if g_idx in matched_gold:
|
|
continue
|
|
if pred.fact_type == g.fact_type and pred.predicate == g.predicate:
|
|
pairs.append((pred, g))
|
|
matched_gold.add(g_idx)
|
|
break
|
|
|
|
return pairs
|
|
|
|
|
|
def evaluate_numeric_facts(
|
|
predicted: list[NumericFact],
|
|
gold: list[NumericFact],
|
|
tolerance_pct: float = DEFAULT_TOLERANCE_PCT,
|
|
document_count: int = 1,
|
|
) -> NumericEvaluationReport:
|
|
"""Run full numeric extraction evaluation.
|
|
|
|
Aligns predicted facts to gold facts by fact_type and predicate,
|
|
then computes exact match accuracy, tolerance-based accuracy,
|
|
per-type breakdowns, unit consistency, period match accuracy,
|
|
tolerance distribution histogram, and error categories.
|
|
|
|
Args:
|
|
predicted: All predicted numeric facts.
|
|
gold: All gold standard numeric facts.
|
|
tolerance_pct: Tolerance percentage for approximate matching.
|
|
document_count: Number of documents evaluated.
|
|
|
|
Returns:
|
|
NumericEvaluationReport with complete evaluation results.
|
|
"""
|
|
pairs = _align_facts(predicted, gold)
|
|
total_aligned = len(pairs)
|
|
|
|
# Track results
|
|
exact_matches = 0
|
|
tolerance_matches = 0
|
|
unit_matches = 0
|
|
period_matches = 0
|
|
comparable_count = 0
|
|
|
|
# Per-type tracking
|
|
per_type_exact_counts: dict[str, tuple[int, int]] = {} # type -> (matches, total)
|
|
per_type_tol_counts: dict[str, tuple[int, int]] = {}
|
|
|
|
# Tolerance distribution
|
|
dist = ToleranceDistribution()
|
|
|
|
# Error tracking
|
|
error_counts: dict[str, int] = {}
|
|
|
|
for pred, g in pairs:
|
|
result = match_numeric_fact(pred, g, tolerance_pct)
|
|
|
|
# Unit consistency
|
|
if result.unit_consistent:
|
|
unit_matches += 1
|
|
|
|
# Period match
|
|
if result.period_match:
|
|
period_matches += 1
|
|
|
|
# Only count value comparisons when both values exist
|
|
if pred.normalized_value is not None and g.normalized_value is not None:
|
|
comparable_count += 1
|
|
|
|
if result.exact_match:
|
|
exact_matches += 1
|
|
if result.within_tolerance:
|
|
tolerance_matches += 1
|
|
|
|
# Per-type tracking
|
|
ft = pred.fact_type
|
|
ex_m, ex_t = per_type_exact_counts.get(ft, (0, 0))
|
|
tol_m, tol_t = per_type_tol_counts.get(ft, (0, 0))
|
|
per_type_exact_counts[ft] = (
|
|
ex_m + (1 if result.exact_match else 0),
|
|
ex_t + 1,
|
|
)
|
|
per_type_tol_counts[ft] = (
|
|
tol_m + (1 if result.within_tolerance else 0),
|
|
tol_t + 1,
|
|
)
|
|
|
|
# Tolerance distribution
|
|
bucket = _bucket_relative_error(result.relative_error_pct)
|
|
if bucket == "exact":
|
|
dist.exact += 1
|
|
elif bucket == "within_1pct":
|
|
dist.within_1pct += 1
|
|
elif bucket == "within_5pct":
|
|
dist.within_5pct += 1
|
|
elif bucket == "within_10pct":
|
|
dist.within_10pct += 1
|
|
elif bucket == "beyond_10pct":
|
|
dist.beyond_10pct += 1
|
|
else:
|
|
dist.not_comparable += 1
|
|
|
|
# Error classification for non-exact matches
|
|
if not result.exact_match:
|
|
error_cat = _classify_error(pred, g, pred.normalized_value, g.normalized_value)
|
|
if error_cat:
|
|
error_counts[error_cat] = error_counts.get(error_cat, 0) + 1
|
|
else:
|
|
dist.not_comparable += 1
|
|
# Classify missing value error
|
|
if pred.normalized_value is None:
|
|
error_cat = ErrorCategory.parsing_failure.value
|
|
elif g.normalized_value is None:
|
|
error_cat = ErrorCategory.missing_value.value
|
|
else:
|
|
error_cat = None
|
|
if error_cat:
|
|
error_counts[error_cat] = error_counts.get(error_cat, 0) + 1
|
|
|
|
# Build accuracy metrics
|
|
exact_accuracy = AccuracyMetric(
|
|
accuracy=exact_matches / comparable_count if comparable_count > 0 else 1.0,
|
|
matches=exact_matches,
|
|
total=comparable_count,
|
|
)
|
|
tolerance_accuracy = AccuracyMetric(
|
|
accuracy=tolerance_matches / comparable_count if comparable_count > 0 else 1.0,
|
|
matches=tolerance_matches,
|
|
total=comparable_count,
|
|
)
|
|
unit_consistency = AccuracyMetric(
|
|
accuracy=unit_matches / total_aligned if total_aligned > 0 else 1.0,
|
|
matches=unit_matches,
|
|
total=total_aligned,
|
|
)
|
|
period_match_metric = AccuracyMetric(
|
|
accuracy=period_matches / total_aligned if total_aligned > 0 else 1.0,
|
|
matches=period_matches,
|
|
total=total_aligned,
|
|
)
|
|
|
|
# Per-type exact accuracy
|
|
per_type_exact: dict[str, AccuracyMetric] = {}
|
|
for ft, (m, t) in sorted(per_type_exact_counts.items()):
|
|
per_type_exact[ft] = AccuracyMetric(
|
|
accuracy=m / t if t > 0 else 1.0,
|
|
matches=m,
|
|
total=t,
|
|
)
|
|
|
|
# Per-type tolerance accuracy
|
|
per_type_tolerance: dict[str, AccuracyMetric] = {}
|
|
for ft, (m, t) in sorted(per_type_tol_counts.items()):
|
|
per_type_tolerance[ft] = AccuracyMetric(
|
|
accuracy=m / t if t > 0 else 1.0,
|
|
matches=m,
|
|
total=t,
|
|
)
|
|
|
|
total_errors = sum(error_counts.values())
|
|
|
|
return NumericEvaluationReport(
|
|
exact_match_accuracy=exact_accuracy,
|
|
tolerance_accuracy=tolerance_accuracy,
|
|
tolerance_pct_used=tolerance_pct,
|
|
per_type_exact=per_type_exact,
|
|
per_type_tolerance=per_type_tolerance,
|
|
unit_consistency=unit_consistency,
|
|
period_match=period_match_metric,
|
|
tolerance_distribution=dist,
|
|
error_breakdown=ErrorBreakdown(counts=error_counts, total_errors=total_errors),
|
|
document_count=document_count,
|
|
)
|