Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
544 lines
18 KiB
Python
544 lines
18 KiB
Python
"""Unit tests for evidence offset validity, support rate, and related metrics.
|
|
|
|
Validates: Requirements 16.3, 16.4
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from services.intelligence_pipeline_v3.evaluation.evidence_metrics import (
|
|
EvidenceMetricsResult,
|
|
EvidenceSpan,
|
|
ExtractionResult,
|
|
FieldType,
|
|
compute_coverage_score,
|
|
compute_offset_validity,
|
|
compute_orphan_rate,
|
|
compute_per_field_support,
|
|
compute_support_rate,
|
|
compute_unsupported_claim_rate,
|
|
evaluate_evidence,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SOURCE_TEXT = "Apple reported revenue of $94.8 billion for Q3 2024. Tim Cook said growth was strong."
|
|
|
|
|
|
def _span(span_id: str, text: str, start: int, end: int) -> EvidenceSpan:
|
|
return EvidenceSpan(span_id=span_id, text=text, start_char=start, end_char=end)
|
|
|
|
|
|
def _item(
|
|
item_id: str,
|
|
field_type: FieldType,
|
|
evidence_ids: list[str] | None = None,
|
|
required_fields: list[str] | None = None,
|
|
supported_fields: list[str] | None = None,
|
|
) -> ExtractionResult:
|
|
return ExtractionResult(
|
|
item_id=item_id,
|
|
field_type=field_type,
|
|
evidence_ids=evidence_ids or [],
|
|
required_fields=required_fields or [],
|
|
supported_fields=supported_fields or [],
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Offset Validity
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestOffsetValidity:
|
|
def test_all_valid(self) -> None:
|
|
spans = [
|
|
_span("s1", "Apple", 0, 5),
|
|
_span("s2", "revenue", 15, 22),
|
|
]
|
|
rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
|
|
assert rate == 1.0
|
|
assert valid == 2
|
|
assert total == 2
|
|
|
|
def test_one_invalid_text_mismatch(self) -> None:
|
|
spans = [
|
|
_span("s1", "Apple", 0, 5),
|
|
_span("s2", "WRONG", 15, 22), # text doesn't match source
|
|
]
|
|
rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
|
|
assert rate == 0.5
|
|
assert valid == 1
|
|
assert total == 2
|
|
|
|
def test_offset_out_of_bounds(self) -> None:
|
|
spans = [
|
|
_span("s1", "Apple", 0, 5),
|
|
_span("s2", "text", 1000, 1004), # beyond source length
|
|
]
|
|
rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
|
|
assert rate == 0.5
|
|
assert valid == 1
|
|
assert total == 2
|
|
|
|
def test_negative_offsets(self) -> None:
|
|
spans = [
|
|
_span("s1", "Apple", -1, 5),
|
|
]
|
|
rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
|
|
assert rate == 0.0
|
|
assert valid == 0
|
|
assert total == 1
|
|
|
|
def test_start_greater_than_end(self) -> None:
|
|
spans = [
|
|
_span("s1", "Apple", 5, 0),
|
|
]
|
|
rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
|
|
assert rate == 0.0
|
|
assert valid == 0
|
|
assert total == 1
|
|
|
|
def test_empty_spans(self) -> None:
|
|
rate, valid, total = compute_offset_validity([], SOURCE_TEXT)
|
|
assert rate == 1.0
|
|
assert valid == 0
|
|
assert total == 0
|
|
|
|
def test_empty_text_span_at_boundary(self) -> None:
|
|
# An empty span (start == end) should match empty string
|
|
spans = [_span("s1", "", 5, 5)]
|
|
rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
|
|
assert rate == 1.0
|
|
assert valid == 1
|
|
|
|
def test_all_invalid(self) -> None:
|
|
spans = [
|
|
_span("s1", "WRONG", 0, 5),
|
|
_span("s2", "ALSO_WRONG", 10, 20),
|
|
]
|
|
rate, valid, total = compute_offset_validity(spans, SOURCE_TEXT)
|
|
assert rate == 0.0
|
|
assert valid == 0
|
|
assert total == 2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Support Rate
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestSupportRate:
|
|
def test_all_supported(self) -> None:
|
|
valid_ids = {"s1", "s2"}
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1"]),
|
|
_item("i2", FieldType.fact, evidence_ids=["s2"]),
|
|
]
|
|
rate, supported, total = compute_support_rate(items, valid_ids)
|
|
assert rate == 1.0
|
|
assert supported == 2
|
|
assert total == 2
|
|
|
|
def test_none_supported(self) -> None:
|
|
valid_ids = {"s1"}
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s99"]),
|
|
_item("i2", FieldType.fact, evidence_ids=["s100"]),
|
|
]
|
|
rate, supported, total = compute_support_rate(items, valid_ids)
|
|
assert rate == 0.0
|
|
assert supported == 0
|
|
assert total == 2
|
|
|
|
def test_partial_support(self) -> None:
|
|
valid_ids = {"s1"}
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1"]),
|
|
_item("i2", FieldType.fact, evidence_ids=["s99"]),
|
|
]
|
|
rate, supported, total = compute_support_rate(items, valid_ids)
|
|
assert rate == 0.5
|
|
assert supported == 1
|
|
assert total == 2
|
|
|
|
def test_item_with_multiple_evidence_one_valid(self) -> None:
|
|
valid_ids = {"s2"}
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1", "s2"]),
|
|
]
|
|
rate, supported, total = compute_support_rate(items, valid_ids)
|
|
assert rate == 1.0
|
|
assert supported == 1
|
|
|
|
def test_empty_items(self) -> None:
|
|
rate, supported, total = compute_support_rate([], {"s1"})
|
|
assert rate == 1.0
|
|
assert supported == 0
|
|
assert total == 0
|
|
|
|
def test_item_with_no_evidence_ids(self) -> None:
|
|
valid_ids = {"s1"}
|
|
items = [_item("i1", FieldType.entity, evidence_ids=[])]
|
|
rate, supported, total = compute_support_rate(items, valid_ids)
|
|
assert rate == 0.0
|
|
assert supported == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Coverage Score
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCoverageScore:
|
|
def test_full_coverage(self) -> None:
|
|
items = [
|
|
_item(
|
|
"i1", FieldType.entity,
|
|
required_fields=["name", "type"],
|
|
supported_fields=["name", "type"],
|
|
),
|
|
]
|
|
score = compute_coverage_score(items)
|
|
assert score == 1.0
|
|
|
|
def test_partial_coverage(self) -> None:
|
|
items = [
|
|
_item(
|
|
"i1", FieldType.entity,
|
|
required_fields=["name", "type", "value"],
|
|
supported_fields=["name"],
|
|
),
|
|
]
|
|
score = compute_coverage_score(items)
|
|
assert abs(score - 1 / 3) < 1e-9
|
|
|
|
def test_no_coverage(self) -> None:
|
|
items = [
|
|
_item(
|
|
"i1", FieldType.entity,
|
|
required_fields=["name", "type"],
|
|
supported_fields=[],
|
|
),
|
|
]
|
|
score = compute_coverage_score(items)
|
|
assert score == 0.0
|
|
|
|
def test_no_required_fields_full_coverage(self) -> None:
|
|
items = [
|
|
_item("i1", FieldType.entity, required_fields=[], supported_fields=[]),
|
|
]
|
|
score = compute_coverage_score(items)
|
|
assert score == 1.0
|
|
|
|
def test_average_across_items(self) -> None:
|
|
items = [
|
|
_item(
|
|
"i1", FieldType.entity,
|
|
required_fields=["name", "type"],
|
|
supported_fields=["name", "type"],
|
|
), # 1.0
|
|
_item(
|
|
"i2", FieldType.fact,
|
|
required_fields=["value", "unit"],
|
|
supported_fields=["value"],
|
|
), # 0.5
|
|
]
|
|
score = compute_coverage_score(items)
|
|
assert abs(score - 0.75) < 1e-9
|
|
|
|
def test_empty_items(self) -> None:
|
|
score = compute_coverage_score([])
|
|
assert score == 1.0
|
|
|
|
def test_supported_field_not_in_required(self) -> None:
|
|
# Extra supported fields beyond required don't inflate the score
|
|
items = [
|
|
_item(
|
|
"i1", FieldType.entity,
|
|
required_fields=["name"],
|
|
supported_fields=["name", "extra_field"],
|
|
),
|
|
]
|
|
score = compute_coverage_score(items)
|
|
assert score == 1.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Orphan Rate
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestOrphanRate:
|
|
def test_no_orphans(self) -> None:
|
|
spans = [_span("s1", "Apple", 0, 5), _span("s2", "revenue", 15, 22)]
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1"]),
|
|
_item("i2", FieldType.fact, evidence_ids=["s2"]),
|
|
]
|
|
rate, count = compute_orphan_rate(spans, items)
|
|
assert rate == 0.0
|
|
assert count == 0
|
|
|
|
def test_all_orphans(self) -> None:
|
|
spans = [_span("s1", "Apple", 0, 5), _span("s2", "revenue", 15, 22)]
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s99"]),
|
|
]
|
|
rate, count = compute_orphan_rate(spans, items)
|
|
assert rate == 1.0
|
|
assert count == 2
|
|
|
|
def test_partial_orphans(self) -> None:
|
|
spans = [
|
|
_span("s1", "Apple", 0, 5),
|
|
_span("s2", "revenue", 15, 22),
|
|
_span("s3", "Q3 2024", 43, 50),
|
|
]
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1"]),
|
|
]
|
|
rate, count = compute_orphan_rate(spans, items)
|
|
assert abs(rate - 2 / 3) < 1e-9
|
|
assert count == 2
|
|
|
|
def test_empty_spans(self) -> None:
|
|
items = [_item("i1", FieldType.entity, evidence_ids=["s1"])]
|
|
rate, count = compute_orphan_rate([], items)
|
|
assert rate == 0.0
|
|
assert count == 0
|
|
|
|
def test_empty_items_all_orphans(self) -> None:
|
|
spans = [_span("s1", "Apple", 0, 5)]
|
|
rate, count = compute_orphan_rate(spans, [])
|
|
assert rate == 1.0
|
|
assert count == 1
|
|
|
|
def test_shared_evidence(self) -> None:
|
|
# Multiple items referencing the same span - span is not orphan
|
|
spans = [_span("s1", "Apple", 0, 5)]
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1"]),
|
|
_item("i2", FieldType.sentiment, evidence_ids=["s1"]),
|
|
]
|
|
rate, count = compute_orphan_rate(spans, items)
|
|
assert rate == 0.0
|
|
assert count == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-Field Support
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestPerFieldSupport:
|
|
def test_all_types_supported(self) -> None:
|
|
valid_ids = {"s1", "s2", "s3", "s4"}
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1"]),
|
|
_item("i2", FieldType.event, evidence_ids=["s2"]),
|
|
_item("i3", FieldType.fact, evidence_ids=["s3"]),
|
|
_item("i4", FieldType.sentiment, evidence_ids=["s4"]),
|
|
]
|
|
result = compute_per_field_support(items, valid_ids)
|
|
assert result["entity"] == 1.0
|
|
assert result["event"] == 1.0
|
|
assert result["fact"] == 1.0
|
|
assert result["sentiment"] == 1.0
|
|
|
|
def test_mixed_support(self) -> None:
|
|
valid_ids = {"s1"}
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1"]),
|
|
_item("i2", FieldType.entity, evidence_ids=["s99"]),
|
|
_item("i3", FieldType.fact, evidence_ids=["s1"]),
|
|
]
|
|
result = compute_per_field_support(items, valid_ids)
|
|
assert result["entity"] == 0.5
|
|
assert result["fact"] == 1.0
|
|
|
|
def test_empty_items(self) -> None:
|
|
result = compute_per_field_support([], {"s1"})
|
|
assert result == {}
|
|
|
|
def test_single_type(self) -> None:
|
|
valid_ids = {"s1"}
|
|
items = [
|
|
_item("i1", FieldType.sentiment, evidence_ids=["s1"]),
|
|
_item("i2", FieldType.sentiment, evidence_ids=["s1"]),
|
|
]
|
|
result = compute_per_field_support(items, valid_ids)
|
|
assert len(result) == 1
|
|
assert result["sentiment"] == 1.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Unsupported Claim Rate
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestUnsupportedClaimRate:
|
|
def test_all_supported(self) -> None:
|
|
valid_ids = {"s1", "s2"}
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1"]),
|
|
_item("i2", FieldType.fact, evidence_ids=["s2"]),
|
|
]
|
|
rate = compute_unsupported_claim_rate(items, valid_ids)
|
|
assert rate == 0.0
|
|
|
|
def test_all_unsupported_no_evidence(self) -> None:
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=[]),
|
|
_item("i2", FieldType.fact, evidence_ids=[]),
|
|
]
|
|
rate = compute_unsupported_claim_rate(items, {"s1"})
|
|
assert rate == 1.0
|
|
|
|
def test_all_unsupported_invalid_evidence(self) -> None:
|
|
valid_ids = {"s1"}
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s99"]),
|
|
_item("i2", FieldType.fact, evidence_ids=["s100"]),
|
|
]
|
|
rate = compute_unsupported_claim_rate(items, valid_ids)
|
|
assert rate == 1.0
|
|
|
|
def test_partial_unsupported(self) -> None:
|
|
valid_ids = {"s1"}
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1"]),
|
|
_item("i2", FieldType.fact, evidence_ids=[]),
|
|
_item("i3", FieldType.event, evidence_ids=["s99"]),
|
|
]
|
|
rate = compute_unsupported_claim_rate(items, valid_ids)
|
|
assert abs(rate - 2 / 3) < 1e-9
|
|
|
|
def test_empty_items(self) -> None:
|
|
rate = compute_unsupported_claim_rate([], {"s1"})
|
|
assert rate == 0.0
|
|
|
|
def test_mixed_evidence_one_valid(self) -> None:
|
|
valid_ids = {"s2"}
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1", "s2"]),
|
|
]
|
|
rate = compute_unsupported_claim_rate(items, valid_ids)
|
|
assert rate == 0.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Full Evaluation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestEvaluateEvidence:
|
|
def test_perfect_evaluation(self) -> None:
|
|
source = "Apple reported revenue of $94.8 billion"
|
|
spans = [
|
|
_span("s1", "Apple", 0, 5),
|
|
_span("s2", "$94.8 billion", 26, 39),
|
|
]
|
|
items = [
|
|
_item(
|
|
"i1", FieldType.entity, evidence_ids=["s1"],
|
|
required_fields=["name"], supported_fields=["name"],
|
|
),
|
|
_item(
|
|
"i2", FieldType.fact, evidence_ids=["s2"],
|
|
required_fields=["value", "unit"], supported_fields=["value", "unit"],
|
|
),
|
|
]
|
|
result = evaluate_evidence(spans, source, items)
|
|
|
|
assert isinstance(result, EvidenceMetricsResult)
|
|
assert result.validity_rate == 1.0
|
|
assert result.support_rate == 1.0
|
|
assert result.coverage_score == 1.0
|
|
assert result.orphan_rate == 0.0
|
|
assert result.unsupported_claim_rate == 0.0
|
|
assert result.total_spans == 2
|
|
assert result.valid_spans == 2
|
|
assert result.total_items == 2
|
|
assert result.supported_items == 2
|
|
assert result.orphan_spans == 0
|
|
|
|
def test_evaluation_with_invalid_spans(self) -> None:
|
|
source = "Apple reported revenue"
|
|
spans = [
|
|
_span("s1", "Apple", 0, 5), # valid
|
|
_span("s2", "WRONG", 6, 14), # invalid text
|
|
]
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1"]),
|
|
_item("i2", FieldType.fact, evidence_ids=["s2"]),
|
|
]
|
|
result = evaluate_evidence(spans, source, items)
|
|
|
|
assert result.validity_rate == 0.5
|
|
assert result.support_rate == 0.5 # only i1 has valid evidence
|
|
assert result.unsupported_claim_rate == 0.5
|
|
|
|
def test_evaluation_with_orphans(self) -> None:
|
|
source = "Apple reported revenue of $94.8 billion"
|
|
spans = [
|
|
_span("s1", "Apple", 0, 5),
|
|
_span("s2", "revenue", 15, 22),
|
|
_span("s3", "$94.8 billion", 26, 39),
|
|
]
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1"]),
|
|
]
|
|
result = evaluate_evidence(spans, source, items)
|
|
|
|
assert result.validity_rate == 1.0
|
|
assert result.support_rate == 1.0
|
|
assert abs(result.orphan_rate - 2 / 3) < 1e-9
|
|
assert result.orphan_spans == 2
|
|
|
|
def test_evaluation_empty_inputs(self) -> None:
|
|
result = evaluate_evidence([], "", [])
|
|
|
|
assert result.validity_rate == 1.0
|
|
assert result.support_rate == 1.0
|
|
assert result.coverage_score == 1.0
|
|
assert result.orphan_rate == 0.0
|
|
assert result.unsupported_claim_rate == 0.0
|
|
assert result.total_spans == 0
|
|
assert result.total_items == 0
|
|
|
|
def test_per_field_support_in_report(self) -> None:
|
|
source = "Apple reported strong growth in Q3"
|
|
spans = [
|
|
_span("s1", "Apple", 0, 5),
|
|
_span("s2", "strong growth", 15, 28),
|
|
]
|
|
items = [
|
|
_item("i1", FieldType.entity, evidence_ids=["s1"]),
|
|
_item("i2", FieldType.sentiment, evidence_ids=["s2"]),
|
|
_item("i3", FieldType.fact, evidence_ids=["s99"]), # unsupported
|
|
]
|
|
result = evaluate_evidence(spans, source, items)
|
|
|
|
assert result.per_field_support["entity"] == 1.0
|
|
assert result.per_field_support["sentiment"] == 1.0
|
|
assert result.per_field_support["fact"] == 0.0
|
|
|
|
def test_result_model_fields(self) -> None:
|
|
result = EvidenceMetricsResult(
|
|
validity_rate=0.9,
|
|
support_rate=0.8,
|
|
coverage_score=0.85,
|
|
orphan_rate=0.1,
|
|
per_field_support={"entity": 0.9, "fact": 0.7},
|
|
unsupported_claim_rate=0.2,
|
|
total_spans=10,
|
|
valid_spans=9,
|
|
total_items=5,
|
|
supported_items=4,
|
|
orphan_spans=1,
|
|
)
|
|
assert result.validity_rate == 0.9
|
|
assert result.per_field_support["entity"] == 0.9
|
|
assert result.orphan_spans == 1
|