Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
520 lines
20 KiB
Python
520 lines
20 KiB
Python
"""Unit tests for numeric exact/tolerance-aware matching metrics.
|
|
|
|
Validates: Requirements 16.3, 16.4
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from services.intelligence_pipeline_v3.evaluation.numeric_metrics import (
|
|
DEFAULT_TOLERANCE_PCT,
|
|
AccuracyMetric,
|
|
ErrorCategory,
|
|
NumericEvaluationReport,
|
|
NumericFact,
|
|
ToleranceDistribution,
|
|
evaluate_numeric_facts,
|
|
match_numeric_fact,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _fact(
|
|
fact_type: str = "eps",
|
|
predicate: str = "actual",
|
|
literal_value: str = "$1.25",
|
|
normalized_value: float | None = 1.25,
|
|
unit: str | None = "USD",
|
|
period: str | None = "Q1 2024",
|
|
) -> NumericFact:
|
|
return NumericFact(
|
|
fact_type=fact_type,
|
|
predicate=predicate,
|
|
literal_value=literal_value,
|
|
normalized_value=normalized_value,
|
|
unit=unit,
|
|
period=period,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Single Fact Matching - Exact Match
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestExactMatch:
|
|
def test_identical_values(self) -> None:
|
|
pred = _fact(normalized_value=1.25)
|
|
gold = _fact(normalized_value=1.25)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.exact_match is True
|
|
assert result.within_tolerance is True
|
|
|
|
def test_different_values(self) -> None:
|
|
pred = _fact(normalized_value=1.30)
|
|
gold = _fact(normalized_value=1.25)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.exact_match is False
|
|
|
|
def test_zero_values(self) -> None:
|
|
pred = _fact(normalized_value=0.0)
|
|
gold = _fact(normalized_value=0.0)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.exact_match is True
|
|
|
|
def test_negative_values(self) -> None:
|
|
pred = _fact(normalized_value=-0.50)
|
|
gold = _fact(normalized_value=-0.50)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.exact_match is True
|
|
|
|
def test_float_precision(self) -> None:
|
|
"""Values that differ only by float rounding should be exact."""
|
|
pred = _fact(normalized_value=0.1 + 0.2)
|
|
gold = _fact(normalized_value=0.3)
|
|
result = match_numeric_fact(pred, gold)
|
|
# 0.1 + 0.2 is ~0.30000000000000004, within 1e-9 of 0.3
|
|
assert result.exact_match is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Single Fact Matching - Tolerance
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestToleranceMatch:
|
|
def test_within_5pct_default(self) -> None:
|
|
# 5% of 100 = 5, so 104 is within tolerance
|
|
pred = _fact(normalized_value=104.0)
|
|
gold = _fact(normalized_value=100.0)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.within_tolerance is True
|
|
assert result.exact_match is False
|
|
|
|
def test_exactly_at_5pct_boundary(self) -> None:
|
|
# 5% of 100 = 5, so 105 is exactly at the boundary (inclusive)
|
|
pred = _fact(normalized_value=105.0)
|
|
gold = _fact(normalized_value=100.0)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.within_tolerance is True
|
|
|
|
def test_beyond_5pct(self) -> None:
|
|
# 5% of 100 = 5, so 105.01 is beyond
|
|
pred = _fact(normalized_value=105.01)
|
|
gold = _fact(normalized_value=100.0)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.within_tolerance is False
|
|
|
|
def test_negative_tolerance(self) -> None:
|
|
# 5% of 100 = 5, so 95 is within tolerance (below)
|
|
pred = _fact(normalized_value=95.0)
|
|
gold = _fact(normalized_value=100.0)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.within_tolerance is True
|
|
|
|
def test_custom_tolerance_1pct(self) -> None:
|
|
pred = _fact(normalized_value=101.0)
|
|
gold = _fact(normalized_value=100.0)
|
|
result = match_numeric_fact(pred, gold, tolerance_pct=1.0)
|
|
assert result.within_tolerance is True
|
|
assert result.tolerance_pct == 1.0
|
|
|
|
def test_custom_tolerance_10pct(self) -> None:
|
|
pred = _fact(normalized_value=109.0)
|
|
gold = _fact(normalized_value=100.0)
|
|
result = match_numeric_fact(pred, gold, tolerance_pct=10.0)
|
|
assert result.within_tolerance is True
|
|
|
|
def test_zero_gold_value_tolerance(self) -> None:
|
|
"""When gold is zero, tolerance uses absolute comparison."""
|
|
pred = _fact(normalized_value=0.01)
|
|
gold = _fact(normalized_value=0.0)
|
|
result = match_numeric_fact(pred, gold, tolerance_pct=5.0)
|
|
# 0.01 < 5/100 = 0.05
|
|
assert result.within_tolerance is True
|
|
|
|
def test_zero_gold_value_beyond_tolerance(self) -> None:
|
|
pred = _fact(normalized_value=0.1)
|
|
gold = _fact(normalized_value=0.0)
|
|
result = match_numeric_fact(pred, gold, tolerance_pct=5.0)
|
|
# 0.1 >= 5/100 = 0.05
|
|
assert result.within_tolerance is False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Unit Consistency
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestUnitConsistency:
|
|
def test_same_units(self) -> None:
|
|
pred = _fact(unit="USD")
|
|
gold = _fact(unit="USD")
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.unit_consistent is True
|
|
|
|
def test_different_units(self) -> None:
|
|
pred = _fact(unit="EUR")
|
|
gold = _fact(unit="USD")
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.unit_consistent is False
|
|
|
|
def test_pred_missing_unit_gold_has_unit(self) -> None:
|
|
pred = _fact(unit=None)
|
|
gold = _fact(unit="USD")
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.unit_consistent is False
|
|
|
|
def test_gold_missing_unit(self) -> None:
|
|
"""If gold has no unit, consistency is assumed."""
|
|
pred = _fact(unit="USD")
|
|
gold = _fact(unit=None)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.unit_consistent is True
|
|
|
|
def test_both_none_units(self) -> None:
|
|
pred = _fact(unit=None)
|
|
gold = _fact(unit=None)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.unit_consistent is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Period Match
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestPeriodMatch:
|
|
def test_same_period(self) -> None:
|
|
pred = _fact(period="Q1 2024")
|
|
gold = _fact(period="Q1 2024")
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.period_match is True
|
|
|
|
def test_different_period(self) -> None:
|
|
pred = _fact(period="Q2 2024")
|
|
gold = _fact(period="Q1 2024")
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.period_match is False
|
|
|
|
def test_pred_missing_period_gold_has_period(self) -> None:
|
|
pred = _fact(period=None)
|
|
gold = _fact(period="Q1 2024")
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.period_match is False
|
|
|
|
def test_gold_missing_period(self) -> None:
|
|
"""If gold has no period, match is assumed."""
|
|
pred = _fact(period="Q1 2024")
|
|
gold = _fact(period=None)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.period_match is True
|
|
|
|
def test_both_none_periods(self) -> None:
|
|
pred = _fact(period=None)
|
|
gold = _fact(period=None)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.period_match is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Error Metrics
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestErrorMetrics:
|
|
def test_absolute_error(self) -> None:
|
|
pred = _fact(normalized_value=1.30)
|
|
gold = _fact(normalized_value=1.25)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.absolute_error is not None
|
|
assert abs(result.absolute_error - 0.05) < 1e-9
|
|
|
|
def test_relative_error(self) -> None:
|
|
pred = _fact(normalized_value=105.0)
|
|
gold = _fact(normalized_value=100.0)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.relative_error_pct is not None
|
|
assert abs(result.relative_error_pct - 5.0) < 1e-9
|
|
|
|
def test_relative_error_zero_gold(self) -> None:
|
|
pred = _fact(normalized_value=1.0)
|
|
gold = _fact(normalized_value=0.0)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.relative_error_pct is None
|
|
|
|
def test_none_pred_value(self) -> None:
|
|
pred = _fact(normalized_value=None)
|
|
gold = _fact(normalized_value=1.25)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.exact_match is False
|
|
assert result.within_tolerance is False
|
|
assert result.absolute_error is None
|
|
assert result.relative_error_pct is None
|
|
|
|
def test_none_gold_value(self) -> None:
|
|
pred = _fact(normalized_value=1.25)
|
|
gold = _fact(normalized_value=None)
|
|
result = match_numeric_fact(pred, gold)
|
|
assert result.exact_match is False
|
|
assert result.within_tolerance is False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Batch Evaluation - Overall Accuracy
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestBatchEvaluation:
|
|
def test_perfect_match(self) -> None:
|
|
gold = [
|
|
_fact(fact_type="eps", predicate="actual", normalized_value=1.25),
|
|
_fact(fact_type="revenue", predicate="actual", normalized_value=50.0e9),
|
|
]
|
|
pred = [
|
|
_fact(fact_type="eps", predicate="actual", normalized_value=1.25),
|
|
_fact(fact_type="revenue", predicate="actual", normalized_value=50.0e9),
|
|
]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.exact_match_accuracy.accuracy == 1.0
|
|
assert report.tolerance_accuracy.accuracy == 1.0
|
|
|
|
def test_empty_inputs(self) -> None:
|
|
report = evaluate_numeric_facts([], [])
|
|
assert report.exact_match_accuracy.accuracy == 1.0
|
|
assert report.exact_match_accuracy.total == 0
|
|
assert report.tolerance_accuracy.accuracy == 1.0
|
|
|
|
def test_no_matches(self) -> None:
|
|
gold = [_fact(fact_type="eps", predicate="actual", normalized_value=1.25)]
|
|
pred = [_fact(fact_type="eps", predicate="actual", normalized_value=2.00)]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.exact_match_accuracy.accuracy == 0.0
|
|
assert report.tolerance_accuracy.accuracy == 0.0
|
|
|
|
def test_tolerance_only_match(self) -> None:
|
|
gold = [_fact(fact_type="eps", predicate="actual", normalized_value=100.0)]
|
|
pred = [_fact(fact_type="eps", predicate="actual", normalized_value=103.0)]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.exact_match_accuracy.accuracy == 0.0
|
|
assert report.tolerance_accuracy.accuracy == 1.0
|
|
|
|
def test_unmatched_facts_not_aligned(self) -> None:
|
|
"""Facts with different predicates don't align."""
|
|
gold = [_fact(fact_type="eps", predicate="actual", normalized_value=1.25)]
|
|
pred = [_fact(fact_type="eps", predicate="estimate", normalized_value=1.25)]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
# No pairs aligned
|
|
assert report.exact_match_accuracy.total == 0
|
|
|
|
def test_multiple_same_type_predicate(self) -> None:
|
|
"""Multiple facts with same type and predicate align one-to-one."""
|
|
gold = [
|
|
_fact(fact_type="eps", predicate="actual", normalized_value=1.25),
|
|
_fact(fact_type="eps", predicate="actual", normalized_value=2.50),
|
|
]
|
|
pred = [
|
|
_fact(fact_type="eps", predicate="actual", normalized_value=1.25),
|
|
_fact(fact_type="eps", predicate="actual", normalized_value=2.50),
|
|
]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.exact_match_accuracy.matches == 2
|
|
assert report.exact_match_accuracy.total == 2
|
|
|
|
def test_document_count(self) -> None:
|
|
report = evaluate_numeric_facts([], [], document_count=5)
|
|
assert report.document_count == 5
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-Type Breakdown
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestPerTypeBreakdown:
|
|
def test_single_type(self) -> None:
|
|
gold = [_fact(fact_type="eps", predicate="actual", normalized_value=1.25)]
|
|
pred = [_fact(fact_type="eps", predicate="actual", normalized_value=1.25)]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert "eps" in report.per_type_exact
|
|
assert report.per_type_exact["eps"].accuracy == 1.0
|
|
|
|
def test_multiple_types(self) -> None:
|
|
gold = [
|
|
_fact(fact_type="eps", predicate="actual", normalized_value=1.25),
|
|
_fact(fact_type="revenue", predicate="actual", normalized_value=50.0e9),
|
|
_fact(fact_type="price_target", predicate="consensus", normalized_value=180.0),
|
|
]
|
|
pred = [
|
|
_fact(fact_type="eps", predicate="actual", normalized_value=1.25),
|
|
_fact(fact_type="revenue", predicate="actual", normalized_value=51.0e9),
|
|
_fact(fact_type="price_target", predicate="consensus", normalized_value=200.0),
|
|
]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.per_type_exact["eps"].accuracy == 1.0
|
|
assert report.per_type_exact["revenue"].accuracy == 0.0
|
|
# Revenue: 51e9 vs 50e9 = 2% off, within 5% tolerance
|
|
assert report.per_type_tolerance["revenue"].accuracy == 1.0
|
|
# Price target: 200 vs 180 = 11.1% off, beyond 5%
|
|
assert report.per_type_tolerance["price_target"].accuracy == 0.0
|
|
|
|
def test_custom_tolerance_per_type(self) -> None:
|
|
gold = [
|
|
_fact(fact_type="guidance", predicate="low", normalized_value=5.0),
|
|
]
|
|
pred = [
|
|
_fact(fact_type="guidance", predicate="low", normalized_value=5.4),
|
|
]
|
|
# 5.4 vs 5.0 = 8%, within 10% but not 5%
|
|
report_5 = evaluate_numeric_facts(pred, gold, tolerance_pct=5.0)
|
|
report_10 = evaluate_numeric_facts(pred, gold, tolerance_pct=10.0)
|
|
assert report_5.per_type_tolerance["guidance"].accuracy == 0.0
|
|
assert report_10.per_type_tolerance["guidance"].accuracy == 1.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Unit Consistency Report
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestUnitConsistencyReport:
|
|
def test_all_consistent(self) -> None:
|
|
gold = [
|
|
_fact(unit="USD", normalized_value=1.0),
|
|
_fact(fact_type="revenue", predicate="actual", unit="USD", normalized_value=50.0),
|
|
]
|
|
pred = [
|
|
_fact(unit="USD", normalized_value=1.0),
|
|
_fact(fact_type="revenue", predicate="actual", unit="USD", normalized_value=50.0),
|
|
]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.unit_consistency.accuracy == 1.0
|
|
|
|
def test_mixed_consistency(self) -> None:
|
|
gold = [
|
|
_fact(unit="USD", normalized_value=1.0),
|
|
_fact(fact_type="revenue", predicate="actual", unit="USD", normalized_value=50.0),
|
|
]
|
|
pred = [
|
|
_fact(unit="USD", normalized_value=1.0),
|
|
_fact(fact_type="revenue", predicate="actual", unit="EUR", normalized_value=50.0),
|
|
]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.unit_consistency.accuracy == 0.5
|
|
assert report.unit_consistency.matches == 1
|
|
assert report.unit_consistency.total == 2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Period Match Report
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestPeriodMatchReport:
|
|
def test_all_periods_match(self) -> None:
|
|
gold = [_fact(period="Q1 2024", normalized_value=1.0)]
|
|
pred = [_fact(period="Q1 2024", normalized_value=1.0)]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.period_match.accuracy == 1.0
|
|
|
|
def test_period_mismatch(self) -> None:
|
|
gold = [_fact(period="Q1 2024", normalized_value=1.0)]
|
|
pred = [_fact(period="FY 2024", normalized_value=1.0)]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.period_match.accuracy == 0.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tolerance Distribution
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestToleranceDistribution:
|
|
def test_exact_bucket(self) -> None:
|
|
gold = [_fact(normalized_value=1.0)]
|
|
pred = [_fact(normalized_value=1.0)]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.tolerance_distribution.exact == 1
|
|
|
|
def test_within_1pct_bucket(self) -> None:
|
|
gold = [_fact(normalized_value=100.0)]
|
|
pred = [_fact(normalized_value=100.5)] # 0.5% off
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.tolerance_distribution.within_1pct == 1
|
|
|
|
def test_within_5pct_bucket(self) -> None:
|
|
gold = [_fact(normalized_value=100.0)]
|
|
pred = [_fact(normalized_value=103.0)] # 3% off
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.tolerance_distribution.within_5pct == 1
|
|
|
|
def test_within_10pct_bucket(self) -> None:
|
|
gold = [_fact(normalized_value=100.0)]
|
|
pred = [_fact(normalized_value=108.0)] # 8% off
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.tolerance_distribution.within_10pct == 1
|
|
|
|
def test_beyond_10pct_bucket(self) -> None:
|
|
gold = [_fact(normalized_value=100.0)]
|
|
pred = [_fact(normalized_value=115.0)] # 15% off
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.tolerance_distribution.beyond_10pct == 1
|
|
|
|
def test_not_comparable(self) -> None:
|
|
gold = [_fact(normalized_value=None)]
|
|
pred = [_fact(normalized_value=1.0)]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.tolerance_distribution.not_comparable == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Error Breakdown
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestErrorBreakdown:
|
|
def test_sign_error(self) -> None:
|
|
gold = [_fact(normalized_value=1.0)]
|
|
pred = [_fact(normalized_value=-1.0)]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert ErrorCategory.sign_error.value in report.error_breakdown.counts
|
|
assert report.error_breakdown.total_errors >= 1
|
|
|
|
def test_magnitude_error(self) -> None:
|
|
gold = [_fact(normalized_value=1.0)]
|
|
pred = [_fact(normalized_value=100.0)] # 100x off
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert ErrorCategory.magnitude_error.value in report.error_breakdown.counts
|
|
|
|
def test_parsing_failure(self) -> None:
|
|
gold = [_fact(normalized_value=1.0)]
|
|
pred = [_fact(normalized_value=None)]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert ErrorCategory.parsing_failure.value in report.error_breakdown.counts
|
|
|
|
def test_no_errors_on_exact_match(self) -> None:
|
|
gold = [_fact(normalized_value=1.0)]
|
|
pred = [_fact(normalized_value=1.0)]
|
|
report = evaluate_numeric_facts(pred, gold)
|
|
assert report.error_breakdown.total_errors == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Report Model Validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestReportModel:
|
|
def test_report_fields(self) -> None:
|
|
report = evaluate_numeric_facts([], [], tolerance_pct=7.5, document_count=3)
|
|
assert isinstance(report, NumericEvaluationReport)
|
|
assert report.tolerance_pct_used == 7.5
|
|
assert report.document_count == 3
|
|
assert isinstance(report.tolerance_distribution, ToleranceDistribution)
|
|
assert isinstance(report.exact_match_accuracy, AccuracyMetric)
|
|
|
|
def test_default_tolerance(self) -> None:
|
|
report = evaluate_numeric_facts([], [])
|
|
assert report.tolerance_pct_used == DEFAULT_TOLERANCE_PCT
|