feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1 @@
|
||||
"""Evaluation metrics for Intelligence Pipeline v3."""
|
||||
@@ -0,0 +1,396 @@
|
||||
"""Entity and ticker precision, recall, F1, and ambiguity accuracy metrics.
|
||||
|
||||
Implements evaluation metrics for entity extraction quality against a gold
|
||||
standard corpus. Supports both strict matching (exact span) and relaxed
|
||||
matching (overlapping span with same type), with per-type breakdowns.
|
||||
|
||||
Validates: Requirements 16.3, 16.4
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Domain Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class MatchMode(str, Enum):
|
||||
"""Entity matching strategy."""
|
||||
|
||||
strict = "strict"
|
||||
relaxed = "relaxed"
|
||||
|
||||
|
||||
class EntitySpan(BaseModel):
|
||||
"""A single entity mention with character offsets and type."""
|
||||
|
||||
text: str
|
||||
entity_type: str
|
||||
start_char: int
|
||||
end_char: int
|
||||
document_id: str = ""
|
||||
canonical_id: str | None = None
|
||||
is_ambiguous: bool = False
|
||||
|
||||
@property
|
||||
def span(self) -> tuple[int, int]:
|
||||
return (self.start_char, self.end_char)
|
||||
|
||||
|
||||
class TickerMention(BaseModel):
|
||||
"""A resolved ticker/company mention."""
|
||||
|
||||
text: str
|
||||
ticker: str
|
||||
start_char: int
|
||||
end_char: int
|
||||
document_id: str = ""
|
||||
canonical_company_id: str | None = None
|
||||
is_ambiguous: bool = False
|
||||
|
||||
@property
|
||||
def span(self) -> tuple[int, int]:
|
||||
return (self.start_char, self.end_char)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PRF1(BaseModel):
|
||||
"""Precision, recall, F1 triple."""
|
||||
|
||||
precision: float = Field(ge=0.0, le=1.0)
|
||||
recall: float = Field(ge=0.0, le=1.0)
|
||||
f1: float = Field(ge=0.0, le=1.0)
|
||||
support_predicted: int = Field(ge=0)
|
||||
support_gold: int = Field(ge=0)
|
||||
|
||||
|
||||
class EntityMetricsResult(BaseModel):
|
||||
"""Full entity evaluation result with per-type breakdowns."""
|
||||
|
||||
match_mode: Literal["strict", "relaxed"]
|
||||
overall: PRF1
|
||||
per_type: dict[str, PRF1]
|
||||
|
||||
|
||||
class TickerMetricsResult(BaseModel):
|
||||
"""Ticker/company resolution evaluation result."""
|
||||
|
||||
match_mode: Literal["strict", "relaxed"]
|
||||
overall: PRF1
|
||||
per_type: dict[str, PRF1] = Field(
|
||||
default_factory=dict,
|
||||
description="Breakdown by canonical company or sector if available",
|
||||
)
|
||||
|
||||
|
||||
class AmbiguityResult(BaseModel):
|
||||
"""Ambiguity detection accuracy."""
|
||||
|
||||
accuracy: float = Field(ge=0.0, le=1.0)
|
||||
true_positives: int = Field(ge=0)
|
||||
true_negatives: int = Field(ge=0)
|
||||
false_positives: int = Field(ge=0)
|
||||
false_negatives: int = Field(ge=0)
|
||||
support: int = Field(ge=0)
|
||||
|
||||
|
||||
class EntityEvaluationReport(BaseModel):
|
||||
"""Complete entity evaluation report."""
|
||||
|
||||
entity_metrics: EntityMetricsResult
|
||||
ticker_metrics: TickerMetricsResult
|
||||
ambiguity_accuracy: AmbiguityResult
|
||||
document_count: int = Field(ge=0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Matching Logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _spans_overlap(a: tuple[int, int], b: tuple[int, int]) -> bool:
|
||||
"""Return True if two character spans overlap."""
|
||||
return a[0] < b[1] and b[0] < a[1]
|
||||
|
||||
|
||||
def _entity_matches_strict(pred: EntitySpan, gold: EntitySpan) -> bool:
|
||||
"""Strict match: exact span boundaries and same entity type."""
|
||||
return (
|
||||
pred.entity_type == gold.entity_type
|
||||
and pred.start_char == gold.start_char
|
||||
and pred.end_char == gold.end_char
|
||||
)
|
||||
|
||||
|
||||
def _entity_matches_relaxed(pred: EntitySpan, gold: EntitySpan) -> bool:
|
||||
"""Relaxed match: overlapping span with same entity type."""
|
||||
return pred.entity_type == gold.entity_type and _spans_overlap(
|
||||
pred.span, gold.span
|
||||
)
|
||||
|
||||
|
||||
def _ticker_matches_strict(pred: TickerMention, gold: TickerMention) -> bool:
|
||||
"""Strict match: exact span and same resolved ticker."""
|
||||
return (
|
||||
pred.ticker == gold.ticker
|
||||
and pred.start_char == gold.start_char
|
||||
and pred.end_char == gold.end_char
|
||||
)
|
||||
|
||||
|
||||
def _ticker_matches_relaxed(pred: TickerMention, gold: TickerMention) -> bool:
|
||||
"""Relaxed match: overlapping span with same resolved ticker."""
|
||||
return pred.ticker == gold.ticker and _spans_overlap(pred.span, gold.span)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core Metric Computation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _compute_prf1(
|
||||
predicted: list[EntitySpan] | list[TickerMention],
|
||||
gold: list[EntitySpan] | list[TickerMention],
|
||||
match_fn: object,
|
||||
) -> PRF1:
|
||||
"""Compute precision, recall, F1 using greedy bipartite matching.
|
||||
|
||||
Each predicted item can match at most one gold item and vice versa.
|
||||
"""
|
||||
n_pred = len(predicted)
|
||||
n_gold = len(gold)
|
||||
|
||||
if n_pred == 0 and n_gold == 0:
|
||||
return PRF1(
|
||||
precision=1.0,
|
||||
recall=1.0,
|
||||
f1=1.0,
|
||||
support_predicted=0,
|
||||
support_gold=0,
|
||||
)
|
||||
|
||||
if n_pred == 0:
|
||||
return PRF1(
|
||||
precision=1.0,
|
||||
recall=0.0,
|
||||
f1=0.0,
|
||||
support_predicted=0,
|
||||
support_gold=n_gold,
|
||||
)
|
||||
|
||||
if n_gold == 0:
|
||||
return PRF1(
|
||||
precision=0.0,
|
||||
recall=1.0,
|
||||
f1=0.0,
|
||||
support_predicted=n_pred,
|
||||
support_gold=0,
|
||||
)
|
||||
|
||||
# Greedy matching: for each predicted, find first unmatched gold
|
||||
matched_gold: set[int] = set()
|
||||
true_positives = 0
|
||||
|
||||
for p in predicted:
|
||||
for g_idx, g in enumerate(gold):
|
||||
if g_idx in matched_gold:
|
||||
continue
|
||||
if match_fn(p, g): # type: ignore[operator]
|
||||
true_positives += 1
|
||||
matched_gold.add(g_idx)
|
||||
break
|
||||
|
||||
precision = true_positives / n_pred if n_pred > 0 else 0.0
|
||||
recall = true_positives / n_gold if n_gold > 0 else 0.0
|
||||
|
||||
if precision + recall > 0:
|
||||
f1 = 2 * precision * recall / (precision + recall)
|
||||
else:
|
||||
f1 = 0.0
|
||||
|
||||
return PRF1(
|
||||
precision=precision,
|
||||
recall=recall,
|
||||
f1=f1,
|
||||
support_predicted=n_pred,
|
||||
support_gold=n_gold,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_entity_metrics(
|
||||
predicted: list[EntitySpan],
|
||||
gold: list[EntitySpan],
|
||||
mode: MatchMode = MatchMode.strict,
|
||||
) -> EntityMetricsResult:
|
||||
"""Compute entity precision, recall, F1 with per-type breakdowns.
|
||||
|
||||
Args:
|
||||
predicted: Predicted entity spans.
|
||||
gold: Gold standard entity spans.
|
||||
mode: Matching strategy (strict or relaxed).
|
||||
|
||||
Returns:
|
||||
EntityMetricsResult with overall and per-type PRF1.
|
||||
"""
|
||||
match_fn = _entity_matches_strict if mode == MatchMode.strict else _entity_matches_relaxed
|
||||
|
||||
# Overall
|
||||
overall = _compute_prf1(predicted, gold, match_fn)
|
||||
|
||||
# Per-type breakdown
|
||||
all_types = {e.entity_type for e in predicted} | {e.entity_type for e in gold}
|
||||
per_type: dict[str, PRF1] = {}
|
||||
|
||||
for entity_type in sorted(all_types):
|
||||
type_predicted = [e for e in predicted if e.entity_type == entity_type]
|
||||
type_gold = [e for e in gold if e.entity_type == entity_type]
|
||||
per_type[entity_type] = _compute_prf1(type_predicted, type_gold, match_fn)
|
||||
|
||||
return EntityMetricsResult(
|
||||
match_mode=mode.value,
|
||||
overall=overall,
|
||||
per_type=per_type,
|
||||
)
|
||||
|
||||
|
||||
def compute_ticker_metrics(
|
||||
predicted: list[TickerMention],
|
||||
gold: list[TickerMention],
|
||||
mode: MatchMode = MatchMode.strict,
|
||||
) -> TickerMetricsResult:
|
||||
"""Compute ticker/company resolution precision, recall, F1.
|
||||
|
||||
Args:
|
||||
predicted: Predicted ticker mentions with resolved tickers.
|
||||
gold: Gold standard ticker mentions.
|
||||
mode: Matching strategy (strict or relaxed).
|
||||
|
||||
Returns:
|
||||
TickerMetricsResult with overall and optional per-ticker PRF1.
|
||||
"""
|
||||
match_fn = _ticker_matches_strict if mode == MatchMode.strict else _ticker_matches_relaxed
|
||||
|
||||
overall = _compute_prf1(predicted, gold, match_fn)
|
||||
|
||||
# Per-ticker breakdown
|
||||
all_tickers = {t.ticker for t in predicted} | {t.ticker for t in gold}
|
||||
per_type: dict[str, PRF1] = {}
|
||||
|
||||
for ticker in sorted(all_tickers):
|
||||
ticker_predicted = [t for t in predicted if t.ticker == ticker]
|
||||
ticker_gold = [t for t in gold if t.ticker == ticker]
|
||||
per_type[ticker] = _compute_prf1(ticker_predicted, ticker_gold, match_fn)
|
||||
|
||||
return TickerMetricsResult(
|
||||
match_mode=mode.value,
|
||||
overall=overall,
|
||||
per_type=per_type,
|
||||
)
|
||||
|
||||
|
||||
def compute_ambiguity_accuracy(
|
||||
predicted: list[EntitySpan] | list[TickerMention],
|
||||
gold: list[EntitySpan] | list[TickerMention],
|
||||
) -> AmbiguityResult:
|
||||
"""Compute ambiguity detection accuracy.
|
||||
|
||||
Measures how well the system identifies entities that require
|
||||
adjudication (ambiguous entities). Uses the `is_ambiguous` flag
|
||||
on each span/mention.
|
||||
|
||||
Entities are aligned by position (exact start_char, end_char match)
|
||||
to compare ambiguity labels.
|
||||
|
||||
Args:
|
||||
predicted: Predicted entities/tickers with ambiguity flags.
|
||||
gold: Gold standard entities/tickers with ambiguity flags.
|
||||
|
||||
Returns:
|
||||
AmbiguityResult with accuracy and confusion counts.
|
||||
"""
|
||||
# Build a lookup from gold spans to ambiguity flag
|
||||
gold_lookup: dict[tuple[int, int], bool] = {}
|
||||
for g in gold:
|
||||
gold_lookup[(g.start_char, g.end_char)] = g.is_ambiguous
|
||||
|
||||
tp = 0 # predicted ambiguous, gold ambiguous
|
||||
tn = 0 # predicted not ambiguous, gold not ambiguous
|
||||
fp = 0 # predicted ambiguous, gold not ambiguous
|
||||
fn = 0 # predicted not ambiguous, gold ambiguous
|
||||
|
||||
matched_count = 0
|
||||
|
||||
for p in predicted:
|
||||
key = (p.start_char, p.end_char)
|
||||
if key in gold_lookup:
|
||||
matched_count += 1
|
||||
gold_ambiguous = gold_lookup[key]
|
||||
pred_ambiguous = p.is_ambiguous
|
||||
|
||||
if pred_ambiguous and gold_ambiguous:
|
||||
tp += 1
|
||||
elif not pred_ambiguous and not gold_ambiguous:
|
||||
tn += 1
|
||||
elif pred_ambiguous and not gold_ambiguous:
|
||||
fp += 1
|
||||
else:
|
||||
fn += 1
|
||||
|
||||
support = tp + tn + fp + fn
|
||||
accuracy = (tp + tn) / support if support > 0 else 1.0
|
||||
|
||||
return AmbiguityResult(
|
||||
accuracy=accuracy,
|
||||
true_positives=tp,
|
||||
true_negatives=tn,
|
||||
false_positives=fp,
|
||||
false_negatives=fn,
|
||||
support=support,
|
||||
)
|
||||
|
||||
|
||||
def evaluate_entities(
|
||||
predicted_entities: list[EntitySpan],
|
||||
gold_entities: list[EntitySpan],
|
||||
predicted_tickers: list[TickerMention],
|
||||
gold_tickers: list[TickerMention],
|
||||
mode: MatchMode = MatchMode.strict,
|
||||
document_count: int = 1,
|
||||
) -> EntityEvaluationReport:
|
||||
"""Run full entity evaluation producing a complete report.
|
||||
|
||||
Args:
|
||||
predicted_entities: All predicted entity spans.
|
||||
gold_entities: All gold standard entity spans.
|
||||
predicted_tickers: All predicted ticker mentions.
|
||||
gold_tickers: All gold standard ticker mentions.
|
||||
mode: Matching strategy.
|
||||
document_count: Number of documents evaluated.
|
||||
|
||||
Returns:
|
||||
EntityEvaluationReport with entity metrics, ticker metrics,
|
||||
and ambiguity accuracy.
|
||||
"""
|
||||
entity_metrics = compute_entity_metrics(predicted_entities, gold_entities, mode)
|
||||
ticker_metrics = compute_ticker_metrics(predicted_tickers, gold_tickers, mode)
|
||||
ambiguity_accuracy = compute_ambiguity_accuracy(predicted_entities, gold_entities)
|
||||
|
||||
return EntityEvaluationReport(
|
||||
entity_metrics=entity_metrics,
|
||||
ticker_metrics=ticker_metrics,
|
||||
ambiguity_accuracy=ambiguity_accuracy,
|
||||
document_count=document_count,
|
||||
)
|
||||
@@ -0,0 +1,384 @@
|
||||
"""Event and relation macro/micro F1 evaluation metrics.
|
||||
|
||||
Implements evaluation metrics for event classification and relation extraction
|
||||
quality against a gold standard corpus. Supports both macro-F1 (average across
|
||||
classes) and micro-F1 (global TP/FP/FN) with per-class breakdowns.
|
||||
|
||||
Matching logic:
|
||||
- Events match if they share the same event_class AND have overlapping evidence
|
||||
spans OR the same primary company.
|
||||
- Relations match if they share the same relation_type, source_id, and target_id.
|
||||
|
||||
Validates: Requirements 16.3, 16.4
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from services.intelligence_pipeline_v3.schemas.annotations import (
|
||||
EventClass,
|
||||
RelationType,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PredictedEvent(BaseModel):
|
||||
"""A predicted event for evaluation."""
|
||||
|
||||
event_class: EventClass
|
||||
evidence_ids: list[str] = Field(default_factory=list)
|
||||
primary_company_ids: list[str] = Field(default_factory=list)
|
||||
confidence: float = Field(ge=0.0, le=1.0, default=1.0)
|
||||
|
||||
|
||||
class GoldEvent(BaseModel):
|
||||
"""A gold standard event for evaluation."""
|
||||
|
||||
event_class: EventClass
|
||||
evidence_ids: list[str] = Field(default_factory=list)
|
||||
primary_company_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class PredictedRelation(BaseModel):
|
||||
"""A predicted relation for evaluation."""
|
||||
|
||||
relation_type: RelationType
|
||||
source_id: str
|
||||
target_id: str
|
||||
confidence: float = Field(ge=0.0, le=1.0, default=1.0)
|
||||
|
||||
|
||||
class GoldRelation(BaseModel):
|
||||
"""A gold standard relation for evaluation."""
|
||||
|
||||
relation_type: RelationType
|
||||
source_id: str
|
||||
target_id: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PRF1(BaseModel):
|
||||
"""Precision, recall, F1 triple."""
|
||||
|
||||
precision: float = Field(ge=0.0, le=1.0)
|
||||
recall: float = Field(ge=0.0, le=1.0)
|
||||
f1: float = Field(ge=0.0, le=1.0)
|
||||
support_predicted: int = Field(ge=0)
|
||||
support_gold: int = Field(ge=0)
|
||||
|
||||
|
||||
class EventMetricsResult(BaseModel):
|
||||
"""Full event evaluation result with macro/micro F1 and per-class breakdown."""
|
||||
|
||||
macro_f1: float = Field(ge=0.0, le=1.0)
|
||||
micro: PRF1
|
||||
per_class: dict[str, PRF1]
|
||||
|
||||
|
||||
class RelationMetricsResult(BaseModel):
|
||||
"""Full relation evaluation result with macro/micro F1 and per-type breakdown."""
|
||||
|
||||
macro_f1: float = Field(ge=0.0, le=1.0)
|
||||
micro: PRF1
|
||||
per_type: dict[str, PRF1]
|
||||
|
||||
|
||||
class EventRelationEvaluationReport(BaseModel):
|
||||
"""Complete event and relation evaluation report."""
|
||||
|
||||
event_metrics: EventMetricsResult
|
||||
relation_metrics: RelationMetricsResult
|
||||
document_count: int = Field(ge=0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Matching Logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _events_match(pred: PredictedEvent, gold: GoldEvent) -> bool:
|
||||
"""Events match if same event_class AND overlapping evidence OR same primary company.
|
||||
|
||||
Overlap means at least one evidence_id in common, OR at least one
|
||||
primary_company_id in common.
|
||||
"""
|
||||
if pred.event_class != gold.event_class:
|
||||
return False
|
||||
|
||||
# Check overlapping evidence spans
|
||||
if pred.evidence_ids and gold.evidence_ids:
|
||||
if set(pred.evidence_ids) & set(gold.evidence_ids):
|
||||
return True
|
||||
|
||||
# Check same primary company
|
||||
if pred.primary_company_ids and gold.primary_company_ids:
|
||||
if set(pred.primary_company_ids) & set(gold.primary_company_ids):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _relations_match(pred: PredictedRelation, gold: GoldRelation) -> bool:
|
||||
"""Relations match if same type, source, and target."""
|
||||
return (
|
||||
pred.relation_type == gold.relation_type
|
||||
and pred.source_id == gold.source_id
|
||||
and pred.target_id == gold.target_id
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core Metric Computation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _compute_prf1_greedy(
|
||||
predicted: list,
|
||||
gold: list,
|
||||
match_fn: object,
|
||||
) -> PRF1:
|
||||
"""Compute precision, recall, F1 using greedy bipartite matching.
|
||||
|
||||
Each predicted item can match at most one gold item and vice versa.
|
||||
"""
|
||||
n_pred = len(predicted)
|
||||
n_gold = len(gold)
|
||||
|
||||
if n_pred == 0 and n_gold == 0:
|
||||
return PRF1(
|
||||
precision=1.0, recall=1.0, f1=1.0,
|
||||
support_predicted=0, support_gold=0,
|
||||
)
|
||||
|
||||
if n_pred == 0:
|
||||
return PRF1(
|
||||
precision=1.0, recall=0.0, f1=0.0,
|
||||
support_predicted=0, support_gold=n_gold,
|
||||
)
|
||||
|
||||
if n_gold == 0:
|
||||
return PRF1(
|
||||
precision=0.0, recall=1.0, f1=0.0,
|
||||
support_predicted=n_pred, support_gold=0,
|
||||
)
|
||||
|
||||
matched_gold: set[int] = set()
|
||||
true_positives = 0
|
||||
|
||||
for p in predicted:
|
||||
for g_idx, g in enumerate(gold):
|
||||
if g_idx in matched_gold:
|
||||
continue
|
||||
if match_fn(p, g): # type: ignore[operator]
|
||||
true_positives += 1
|
||||
matched_gold.add(g_idx)
|
||||
break
|
||||
|
||||
precision = true_positives / n_pred if n_pred > 0 else 0.0
|
||||
recall = true_positives / n_gold if n_gold > 0 else 0.0
|
||||
|
||||
if precision + recall > 0:
|
||||
f1 = 2 * precision * recall / (precision + recall)
|
||||
else:
|
||||
f1 = 0.0
|
||||
|
||||
return PRF1(
|
||||
precision=precision,
|
||||
recall=recall,
|
||||
f1=f1,
|
||||
support_predicted=n_pred,
|
||||
support_gold=n_gold,
|
||||
)
|
||||
|
||||
|
||||
def _compute_micro_prf1(
|
||||
predicted: list,
|
||||
gold: list,
|
||||
match_fn: object,
|
||||
class_key_pred: object,
|
||||
class_key_gold: object,
|
||||
all_classes: set[str],
|
||||
) -> PRF1:
|
||||
"""Compute micro-averaged PRF1 by summing TP/FP/FN across all classes."""
|
||||
total_tp = 0
|
||||
total_pred = 0
|
||||
total_gold = 0
|
||||
|
||||
for cls in all_classes:
|
||||
cls_predicted = [p for p in predicted if class_key_pred(p) == cls]
|
||||
cls_gold = [g for g in gold if class_key_gold(g) == cls]
|
||||
|
||||
total_pred += len(cls_predicted)
|
||||
total_gold += len(cls_gold)
|
||||
|
||||
# Greedy match within this class
|
||||
matched_gold: set[int] = set()
|
||||
for p in cls_predicted:
|
||||
for g_idx, g in enumerate(cls_gold):
|
||||
if g_idx in matched_gold:
|
||||
continue
|
||||
if match_fn(p, g): # type: ignore[operator]
|
||||
total_tp += 1
|
||||
matched_gold.add(g_idx)
|
||||
break
|
||||
|
||||
if total_pred == 0 and total_gold == 0:
|
||||
return PRF1(
|
||||
precision=1.0, recall=1.0, f1=1.0,
|
||||
support_predicted=0, support_gold=0,
|
||||
)
|
||||
|
||||
precision = total_tp / total_pred if total_pred > 0 else 0.0
|
||||
recall = total_tp / total_gold if total_gold > 0 else 0.0
|
||||
|
||||
if precision + recall > 0:
|
||||
f1 = 2 * precision * recall / (precision + recall)
|
||||
else:
|
||||
f1 = 0.0
|
||||
|
||||
return PRF1(
|
||||
precision=precision,
|
||||
recall=recall,
|
||||
f1=f1,
|
||||
support_predicted=total_pred,
|
||||
support_gold=total_gold,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API — Events
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_event_metrics(
|
||||
predicted: list[PredictedEvent],
|
||||
gold: list[GoldEvent],
|
||||
) -> EventMetricsResult:
|
||||
"""Compute event macro-F1, micro-F1, and per-class F1.
|
||||
|
||||
Args:
|
||||
predicted: Predicted events.
|
||||
gold: Gold standard events.
|
||||
|
||||
Returns:
|
||||
EventMetricsResult with macro, micro, and per-class breakdowns.
|
||||
"""
|
||||
all_classes = {e.value for e in EventClass}
|
||||
|
||||
# Per-class breakdown
|
||||
per_class: dict[str, PRF1] = {}
|
||||
f1_scores: list[float] = []
|
||||
|
||||
for cls in sorted(all_classes):
|
||||
cls_predicted = [p for p in predicted if p.event_class.value == cls]
|
||||
cls_gold = [g for g in gold if g.event_class.value == cls]
|
||||
prf1 = _compute_prf1_greedy(cls_predicted, cls_gold, _events_match)
|
||||
per_class[cls] = prf1
|
||||
f1_scores.append(prf1.f1)
|
||||
|
||||
# Macro-F1: average F1 across all event classes
|
||||
macro_f1 = sum(f1_scores) / len(f1_scores) if f1_scores else 0.0
|
||||
|
||||
# Micro-F1: global TP/FP/FN
|
||||
micro = _compute_micro_prf1(
|
||||
predicted, gold, _events_match,
|
||||
lambda p: p.event_class.value,
|
||||
lambda g: g.event_class.value,
|
||||
all_classes,
|
||||
)
|
||||
|
||||
return EventMetricsResult(
|
||||
macro_f1=macro_f1,
|
||||
micro=micro,
|
||||
per_class=per_class,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API — Relations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_relation_metrics(
|
||||
predicted: list[PredictedRelation],
|
||||
gold: list[GoldRelation],
|
||||
) -> RelationMetricsResult:
|
||||
"""Compute relation macro-F1, micro-F1, and per-type F1.
|
||||
|
||||
Args:
|
||||
predicted: Predicted relations.
|
||||
gold: Gold standard relations.
|
||||
|
||||
Returns:
|
||||
RelationMetricsResult with macro, micro, and per-type breakdowns.
|
||||
"""
|
||||
all_types = {r.value for r in RelationType}
|
||||
|
||||
# Per-type breakdown
|
||||
per_type: dict[str, PRF1] = {}
|
||||
f1_scores: list[float] = []
|
||||
|
||||
for rtype in sorted(all_types):
|
||||
type_predicted = [p for p in predicted if p.relation_type.value == rtype]
|
||||
type_gold = [g for g in gold if g.relation_type.value == rtype]
|
||||
prf1 = _compute_prf1_greedy(type_predicted, type_gold, _relations_match)
|
||||
per_type[rtype] = prf1
|
||||
f1_scores.append(prf1.f1)
|
||||
|
||||
# Macro-F1: average F1 across all relation types
|
||||
macro_f1 = sum(f1_scores) / len(f1_scores) if f1_scores else 0.0
|
||||
|
||||
# Micro-F1: global TP/FP/FN
|
||||
micro = _compute_micro_prf1(
|
||||
predicted, gold, _relations_match,
|
||||
lambda p: p.relation_type.value,
|
||||
lambda g: g.relation_type.value,
|
||||
all_types,
|
||||
)
|
||||
|
||||
return RelationMetricsResult(
|
||||
macro_f1=macro_f1,
|
||||
micro=micro,
|
||||
per_type=per_type,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API — Combined Report
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def evaluate_events_and_relations(
|
||||
predicted_events: list[PredictedEvent],
|
||||
gold_events: list[GoldEvent],
|
||||
predicted_relations: list[PredictedRelation],
|
||||
gold_relations: list[GoldRelation],
|
||||
document_count: int = 1,
|
||||
) -> EventRelationEvaluationReport:
|
||||
"""Run full event and relation evaluation producing a complete report.
|
||||
|
||||
Args:
|
||||
predicted_events: All predicted events.
|
||||
gold_events: All gold standard events.
|
||||
predicted_relations: All predicted relations.
|
||||
gold_relations: All gold standard relations.
|
||||
document_count: Number of documents evaluated.
|
||||
|
||||
Returns:
|
||||
EventRelationEvaluationReport with event metrics, relation metrics.
|
||||
"""
|
||||
event_metrics = compute_event_metrics(predicted_events, gold_events)
|
||||
relation_metrics = compute_relation_metrics(predicted_relations, gold_relations)
|
||||
|
||||
return EventRelationEvaluationReport(
|
||||
event_metrics=event_metrics,
|
||||
relation_metrics=relation_metrics,
|
||||
document_count=document_count,
|
||||
)
|
||||
@@ -0,0 +1,314 @@
|
||||
"""Evidence offset validity, support rate, coverage, and orphan metrics.
|
||||
|
||||
Implements evaluation metrics for evidence grounding quality:
|
||||
- Offset validity rate: proportion of spans where text matches source at offsets
|
||||
- Support rate: proportion of extracted items with at least one valid evidence span
|
||||
- Coverage score: average proportion of required fields supported by evidence
|
||||
- Orphan rate: proportion of evidence spans not referenced by any extracted item
|
||||
- Per-field support: breakdown of support rate by field type
|
||||
- Unsupported claim rate: proportion of extracted items with no valid evidence
|
||||
|
||||
Validates: Requirements 16.3, 16.4
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Domain Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FieldType(str, Enum):
|
||||
"""Types of extracted fields that can be evidence-supported."""
|
||||
|
||||
entity = "entity"
|
||||
event = "event"
|
||||
fact = "fact"
|
||||
sentiment = "sentiment"
|
||||
|
||||
|
||||
class EvidenceSpan(BaseModel):
|
||||
"""An evidence span with text and character offsets into source."""
|
||||
|
||||
span_id: str
|
||||
text: str
|
||||
start_char: int
|
||||
end_char: int
|
||||
document_id: str = ""
|
||||
|
||||
|
||||
class ExtractionResult(BaseModel):
|
||||
"""An extracted item referencing evidence spans by ID."""
|
||||
|
||||
item_id: str
|
||||
field_type: FieldType
|
||||
evidence_ids: list[str] = Field(default_factory=list)
|
||||
required_fields: list[str] = Field(default_factory=list)
|
||||
supported_fields: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class EvidenceMetricsResult(BaseModel):
|
||||
"""Complete evidence evaluation result."""
|
||||
|
||||
validity_rate: float = Field(ge=0.0, le=1.0)
|
||||
support_rate: float = Field(ge=0.0, le=1.0)
|
||||
coverage_score: float = Field(ge=0.0, le=1.0)
|
||||
orphan_rate: float = Field(ge=0.0, le=1.0)
|
||||
per_field_support: dict[str, float]
|
||||
unsupported_claim_rate: float = Field(ge=0.0, le=1.0)
|
||||
total_spans: int = Field(ge=0)
|
||||
valid_spans: int = Field(ge=0)
|
||||
total_items: int = Field(ge=0)
|
||||
supported_items: int = Field(ge=0)
|
||||
orphan_spans: int = Field(ge=0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core Metric Computation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_offset_validity(
|
||||
spans: list[EvidenceSpan],
|
||||
source_text: str,
|
||||
) -> tuple[float, int, int]:
|
||||
"""Compute the proportion of spans whose text matches source at offsets.
|
||||
|
||||
Args:
|
||||
spans: Evidence spans with text and character offsets.
|
||||
source_text: The original source document text.
|
||||
|
||||
Returns:
|
||||
Tuple of (validity_rate, valid_count, total_count).
|
||||
"""
|
||||
if not spans:
|
||||
return (1.0, 0, 0)
|
||||
|
||||
valid = 0
|
||||
for span in spans:
|
||||
start = span.start_char
|
||||
end = span.end_char
|
||||
|
||||
# Basic bounds check
|
||||
if start < 0 or end < 0 or start > end:
|
||||
continue
|
||||
if end > len(source_text):
|
||||
continue
|
||||
|
||||
source_slice = source_text[start:end]
|
||||
if source_slice == span.text:
|
||||
valid += 1
|
||||
|
||||
total = len(spans)
|
||||
rate = valid / total
|
||||
return (rate, valid, total)
|
||||
|
||||
|
||||
def compute_support_rate(
|
||||
items: list[ExtractionResult],
|
||||
valid_span_ids: set[str],
|
||||
) -> tuple[float, int, int]:
|
||||
"""Compute proportion of items with at least one valid evidence span.
|
||||
|
||||
Args:
|
||||
items: Extraction results referencing evidence span IDs.
|
||||
valid_span_ids: Set of span IDs that passed offset validity.
|
||||
|
||||
Returns:
|
||||
Tuple of (support_rate, supported_count, total_count).
|
||||
"""
|
||||
if not items:
|
||||
return (1.0, 0, 0)
|
||||
|
||||
supported = 0
|
||||
for item in items:
|
||||
if any(eid in valid_span_ids for eid in item.evidence_ids):
|
||||
supported += 1
|
||||
|
||||
total = len(items)
|
||||
rate = supported / total
|
||||
return (rate, supported, total)
|
||||
|
||||
|
||||
def compute_coverage_score(
|
||||
items: list[ExtractionResult],
|
||||
) -> float:
|
||||
"""Compute average proportion of required fields supported by evidence.
|
||||
|
||||
For each item, coverage = len(supported_fields ∩ required_fields) / len(required_fields).
|
||||
Items with no required fields are treated as fully covered.
|
||||
|
||||
Args:
|
||||
items: Extraction results with required and supported field lists.
|
||||
|
||||
Returns:
|
||||
Average coverage score across all items.
|
||||
"""
|
||||
if not items:
|
||||
return 1.0
|
||||
|
||||
total_coverage = 0.0
|
||||
for item in items:
|
||||
if not item.required_fields:
|
||||
total_coverage += 1.0
|
||||
continue
|
||||
|
||||
required = set(item.required_fields)
|
||||
supported = set(item.supported_fields)
|
||||
covered = required & supported
|
||||
total_coverage += len(covered) / len(required)
|
||||
|
||||
return total_coverage / len(items)
|
||||
|
||||
|
||||
def compute_orphan_rate(
|
||||
spans: list[EvidenceSpan],
|
||||
items: list[ExtractionResult],
|
||||
) -> tuple[float, int]:
|
||||
"""Compute proportion of evidence spans not referenced by any item.
|
||||
|
||||
Args:
|
||||
spans: All evidence spans.
|
||||
items: Extraction results referencing evidence span IDs.
|
||||
|
||||
Returns:
|
||||
Tuple of (orphan_rate, orphan_count).
|
||||
"""
|
||||
if not spans:
|
||||
return (0.0, 0)
|
||||
|
||||
referenced_ids: set[str] = set()
|
||||
for item in items:
|
||||
referenced_ids.update(item.evidence_ids)
|
||||
|
||||
orphan_count = sum(1 for span in spans if span.span_id not in referenced_ids)
|
||||
rate = orphan_count / len(spans)
|
||||
return (rate, orphan_count)
|
||||
|
||||
|
||||
def compute_per_field_support(
|
||||
items: list[ExtractionResult],
|
||||
valid_span_ids: set[str],
|
||||
) -> dict[str, float]:
|
||||
"""Compute support rate broken down by field type.
|
||||
|
||||
Args:
|
||||
items: Extraction results with field types and evidence IDs.
|
||||
valid_span_ids: Set of span IDs that passed offset validity.
|
||||
|
||||
Returns:
|
||||
Dict mapping field type name to support rate.
|
||||
"""
|
||||
by_type: dict[str, list[ExtractionResult]] = {}
|
||||
for item in items:
|
||||
key = item.field_type.value
|
||||
by_type.setdefault(key, []).append(item)
|
||||
|
||||
result: dict[str, float] = {}
|
||||
for field_type, type_items in sorted(by_type.items()):
|
||||
rate, _, _ = compute_support_rate(type_items, valid_span_ids)
|
||||
result[field_type] = rate
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def compute_unsupported_claim_rate(
|
||||
items: list[ExtractionResult],
|
||||
valid_span_ids: set[str],
|
||||
) -> float:
|
||||
"""Compute proportion of items with no valid evidence at all.
|
||||
|
||||
An item is unsupported if it has no evidence_ids OR none of its
|
||||
evidence_ids are in the valid set.
|
||||
|
||||
Args:
|
||||
items: Extraction results referencing evidence span IDs.
|
||||
valid_span_ids: Set of span IDs that passed offset validity.
|
||||
|
||||
Returns:
|
||||
Unsupported claim rate (0.0 to 1.0).
|
||||
"""
|
||||
if not items:
|
||||
return 0.0
|
||||
|
||||
unsupported = 0
|
||||
for item in items:
|
||||
if not item.evidence_ids:
|
||||
unsupported += 1
|
||||
elif not any(eid in valid_span_ids for eid in item.evidence_ids):
|
||||
unsupported += 1
|
||||
|
||||
return unsupported / len(items)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def evaluate_evidence(
|
||||
spans: list[EvidenceSpan],
|
||||
source_text: str,
|
||||
items: list[ExtractionResult],
|
||||
) -> EvidenceMetricsResult:
|
||||
"""Run full evidence evaluation producing a complete metrics report.
|
||||
|
||||
Args:
|
||||
spans: All evidence spans with text and offsets.
|
||||
source_text: The original source document text.
|
||||
items: Extraction results referencing evidence spans.
|
||||
|
||||
Returns:
|
||||
EvidenceMetricsResult with all computed metrics.
|
||||
"""
|
||||
# Step 1: Offset validity
|
||||
validity_rate, valid_count, total_spans = compute_offset_validity(spans, source_text)
|
||||
|
||||
# Step 2: Build valid span ID set
|
||||
valid_span_ids: set[str] = set()
|
||||
for span in spans:
|
||||
start = span.start_char
|
||||
end = span.end_char
|
||||
if start < 0 or end < 0 or start > end:
|
||||
continue
|
||||
if end > len(source_text):
|
||||
continue
|
||||
if source_text[start:end] == span.text:
|
||||
valid_span_ids.add(span.span_id)
|
||||
|
||||
# Step 3: Support rate
|
||||
support_rate, supported_count, total_items = compute_support_rate(items, valid_span_ids)
|
||||
|
||||
# Step 4: Coverage score
|
||||
coverage_score = compute_coverage_score(items)
|
||||
|
||||
# Step 5: Orphan rate
|
||||
orphan_rate, orphan_count = compute_orphan_rate(spans, items)
|
||||
|
||||
# Step 6: Per-field support
|
||||
per_field_support = compute_per_field_support(items, valid_span_ids)
|
||||
|
||||
# Step 7: Unsupported claim rate
|
||||
unsupported_claim_rate = compute_unsupported_claim_rate(items, valid_span_ids)
|
||||
|
||||
return EvidenceMetricsResult(
|
||||
validity_rate=validity_rate,
|
||||
support_rate=support_rate,
|
||||
coverage_score=coverage_score,
|
||||
orphan_rate=orphan_rate,
|
||||
per_field_support=per_field_support,
|
||||
unsupported_claim_rate=unsupported_claim_rate,
|
||||
total_spans=total_spans,
|
||||
valid_spans=valid_count,
|
||||
total_items=total_items,
|
||||
supported_items=supported_count,
|
||||
orphan_spans=orphan_count,
|
||||
)
|
||||
@@ -0,0 +1,459 @@
|
||||
"""Numeric exact/tolerance-aware matching metrics for extracted financial facts.
|
||||
|
||||
Implements evaluation metrics for numeric extraction quality against a gold
|
||||
standard corpus. Supports exact match, default 5% tolerance, and configurable
|
||||
tolerance matching. Provides per-fact-type breakdowns, unit consistency
|
||||
checks, and period matching.
|
||||
|
||||
Input model fields: fact_type, predicate, literal_value, normalized_value,
|
||||
unit, period, evidence_ids.
|
||||
|
||||
Validates: Requirements 16.3, 16.4
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Domain Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FactType(str, Enum):
|
||||
"""Known financial fact types for per-type breakdown."""
|
||||
|
||||
eps = "eps"
|
||||
revenue = "revenue"
|
||||
percentage_change = "percentage_change"
|
||||
price_target = "price_target"
|
||||
guidance = "guidance"
|
||||
dividend = "dividend"
|
||||
margin = "margin"
|
||||
growth_rate = "growth_rate"
|
||||
other = "other"
|
||||
|
||||
|
||||
class NumericFact(BaseModel):
|
||||
"""A single extracted numeric fact with normalization and context."""
|
||||
|
||||
fact_type: str
|
||||
predicate: str
|
||||
literal_value: str
|
||||
normalized_value: float | None = None
|
||||
unit: str | None = None
|
||||
period: str | None = None
|
||||
evidence_ids: list[str] = Field(default_factory=list)
|
||||
document_id: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class NumericMatchResult(BaseModel):
|
||||
"""Result of matching a single predicted fact against gold."""
|
||||
|
||||
exact_match: bool = False
|
||||
within_tolerance: bool = False
|
||||
tolerance_pct: float = 0.0
|
||||
unit_consistent: bool = True
|
||||
period_match: bool = True
|
||||
absolute_error: float | None = None
|
||||
relative_error_pct: float | None = None
|
||||
|
||||
|
||||
class AccuracyMetric(BaseModel):
|
||||
"""Simple accuracy metric with support count."""
|
||||
|
||||
accuracy: float = Field(ge=0.0, le=1.0)
|
||||
matches: int = Field(ge=0)
|
||||
total: int = Field(ge=0)
|
||||
|
||||
|
||||
class ToleranceDistribution(BaseModel):
|
||||
"""Distribution of relative errors across tolerance buckets."""
|
||||
|
||||
exact: int = Field(ge=0, default=0)
|
||||
within_1pct: int = Field(ge=0, default=0)
|
||||
within_5pct: int = Field(ge=0, default=0)
|
||||
within_10pct: int = Field(ge=0, default=0)
|
||||
beyond_10pct: int = Field(ge=0, default=0)
|
||||
not_comparable: int = Field(ge=0, default=0)
|
||||
|
||||
|
||||
class ErrorCategory(str, Enum):
|
||||
"""Common numeric extraction error categories."""
|
||||
|
||||
unit_mismatch = "unit_mismatch"
|
||||
period_mismatch = "period_mismatch"
|
||||
magnitude_error = "magnitude_error"
|
||||
sign_error = "sign_error"
|
||||
parsing_failure = "parsing_failure"
|
||||
missing_value = "missing_value"
|
||||
|
||||
|
||||
class ErrorBreakdown(BaseModel):
|
||||
"""Counts of errors by category."""
|
||||
|
||||
counts: dict[str, int] = Field(default_factory=dict)
|
||||
total_errors: int = Field(ge=0, default=0)
|
||||
|
||||
|
||||
class NumericEvaluationReport(BaseModel):
|
||||
"""Complete numeric extraction evaluation report."""
|
||||
|
||||
exact_match_accuracy: AccuracyMetric
|
||||
tolerance_accuracy: AccuracyMetric
|
||||
tolerance_pct_used: float = Field(ge=0.0)
|
||||
per_type_exact: dict[str, AccuracyMetric] = Field(default_factory=dict)
|
||||
per_type_tolerance: dict[str, AccuracyMetric] = Field(default_factory=dict)
|
||||
unit_consistency: AccuracyMetric
|
||||
period_match: AccuracyMetric
|
||||
tolerance_distribution: ToleranceDistribution
|
||||
error_breakdown: ErrorBreakdown
|
||||
document_count: int = Field(ge=0, default=0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Matching Logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DEFAULT_TOLERANCE_PCT = 5.0
|
||||
|
||||
|
||||
def _is_exact_match(pred_value: float, gold_value: float) -> bool:
|
||||
"""Check if predicted value exactly equals gold value (within float epsilon)."""
|
||||
return abs(pred_value - gold_value) < 1e-9
|
||||
|
||||
|
||||
def _is_within_tolerance(
|
||||
pred_value: float, gold_value: float, tolerance_pct: float
|
||||
) -> bool:
|
||||
"""Check if predicted value is within ±tolerance_pct of gold value.
|
||||
|
||||
For zero gold values, uses absolute comparison with a small epsilon
|
||||
derived from the tolerance percentage.
|
||||
"""
|
||||
if abs(gold_value) < 1e-12:
|
||||
# For zero gold, allow small absolute tolerance
|
||||
return abs(pred_value) < tolerance_pct / 100.0
|
||||
threshold = abs(gold_value) * (tolerance_pct / 100.0)
|
||||
return abs(pred_value - gold_value) <= threshold
|
||||
|
||||
|
||||
def _compute_relative_error_pct(pred_value: float, gold_value: float) -> float | None:
|
||||
"""Compute relative error as a percentage of gold value.
|
||||
|
||||
Returns None if gold value is zero (relative error undefined).
|
||||
"""
|
||||
if abs(gold_value) < 1e-12:
|
||||
return None
|
||||
return abs(pred_value - gold_value) / abs(gold_value) * 100.0
|
||||
|
||||
|
||||
def _classify_error(
|
||||
pred: NumericFact, gold: NumericFact, pred_value: float | None, gold_value: float
|
||||
) -> str | None:
|
||||
"""Classify the type of error for a mismatched prediction."""
|
||||
if pred_value is None:
|
||||
if pred.normalized_value is None:
|
||||
return ErrorCategory.parsing_failure.value
|
||||
return ErrorCategory.missing_value.value
|
||||
|
||||
# Check sign error (opposite signs, both non-zero)
|
||||
if pred_value * gold_value < 0 and abs(pred_value) > 1e-9 and abs(gold_value) > 1e-9:
|
||||
return ErrorCategory.sign_error.value
|
||||
|
||||
# Check magnitude error (off by factor of 10+)
|
||||
if abs(gold_value) > 1e-9:
|
||||
ratio = abs(pred_value / gold_value)
|
||||
if ratio >= 10.0 or ratio <= 0.1:
|
||||
return ErrorCategory.magnitude_error.value
|
||||
|
||||
# Unit mismatch (if units don't match)
|
||||
if pred.unit and gold.unit and pred.unit != gold.unit:
|
||||
return ErrorCategory.unit_mismatch.value
|
||||
|
||||
# Period mismatch
|
||||
if pred.period and gold.period and pred.period != gold.period:
|
||||
return ErrorCategory.period_mismatch.value
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _bucket_relative_error(relative_error_pct: float | None) -> str:
|
||||
"""Assign a relative error to a tolerance bucket name."""
|
||||
if relative_error_pct is None:
|
||||
return "not_comparable"
|
||||
if relative_error_pct < 1e-7:
|
||||
return "exact"
|
||||
if relative_error_pct <= 1.0:
|
||||
return "within_1pct"
|
||||
if relative_error_pct <= 5.0:
|
||||
return "within_5pct"
|
||||
if relative_error_pct <= 10.0:
|
||||
return "within_10pct"
|
||||
return "beyond_10pct"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Single Fact Matching
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def match_numeric_fact(
|
||||
pred: NumericFact,
|
||||
gold: NumericFact,
|
||||
tolerance_pct: float = DEFAULT_TOLERANCE_PCT,
|
||||
) -> NumericMatchResult:
|
||||
"""Match a predicted numeric fact against a gold standard fact.
|
||||
|
||||
Compares normalized values, checks unit consistency and period match.
|
||||
|
||||
Args:
|
||||
pred: Predicted numeric fact.
|
||||
gold: Gold standard numeric fact.
|
||||
tolerance_pct: Tolerance percentage for approximate matching.
|
||||
|
||||
Returns:
|
||||
NumericMatchResult with match details.
|
||||
"""
|
||||
# Unit consistency check
|
||||
unit_consistent = True
|
||||
if pred.unit is not None and gold.unit is not None:
|
||||
unit_consistent = pred.unit == gold.unit
|
||||
elif pred.unit is None and gold.unit is not None:
|
||||
unit_consistent = False
|
||||
# If gold has no unit, we consider it consistent regardless
|
||||
|
||||
# Period match check
|
||||
period_match = True
|
||||
if pred.period is not None and gold.period is not None:
|
||||
period_match = pred.period == gold.period
|
||||
elif pred.period is None and gold.period is not None:
|
||||
period_match = False
|
||||
|
||||
# Value comparison
|
||||
pred_value = pred.normalized_value
|
||||
gold_value = gold.normalized_value
|
||||
|
||||
if pred_value is None or gold_value is None:
|
||||
return NumericMatchResult(
|
||||
exact_match=False,
|
||||
within_tolerance=False,
|
||||
tolerance_pct=tolerance_pct,
|
||||
unit_consistent=unit_consistent,
|
||||
period_match=period_match,
|
||||
absolute_error=None,
|
||||
relative_error_pct=None,
|
||||
)
|
||||
|
||||
absolute_error = abs(pred_value - gold_value)
|
||||
relative_error_pct = _compute_relative_error_pct(pred_value, gold_value)
|
||||
exact = _is_exact_match(pred_value, gold_value)
|
||||
within_tol = _is_within_tolerance(pred_value, gold_value, tolerance_pct)
|
||||
|
||||
return NumericMatchResult(
|
||||
exact_match=exact,
|
||||
within_tolerance=within_tol,
|
||||
tolerance_pct=tolerance_pct,
|
||||
unit_consistent=unit_consistent,
|
||||
period_match=period_match,
|
||||
absolute_error=absolute_error,
|
||||
relative_error_pct=relative_error_pct,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Batch Evaluation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _align_facts(
|
||||
predicted: list[NumericFact],
|
||||
gold: list[NumericFact],
|
||||
) -> list[tuple[NumericFact, NumericFact]]:
|
||||
"""Align predicted facts to gold facts using greedy matching.
|
||||
|
||||
Matches on fact_type and predicate. Each gold fact can match at most
|
||||
one predicted fact.
|
||||
"""
|
||||
pairs: list[tuple[NumericFact, NumericFact]] = []
|
||||
matched_gold: set[int] = set()
|
||||
|
||||
for pred in predicted:
|
||||
for g_idx, g in enumerate(gold):
|
||||
if g_idx in matched_gold:
|
||||
continue
|
||||
if pred.fact_type == g.fact_type and pred.predicate == g.predicate:
|
||||
pairs.append((pred, g))
|
||||
matched_gold.add(g_idx)
|
||||
break
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def evaluate_numeric_facts(
|
||||
predicted: list[NumericFact],
|
||||
gold: list[NumericFact],
|
||||
tolerance_pct: float = DEFAULT_TOLERANCE_PCT,
|
||||
document_count: int = 1,
|
||||
) -> NumericEvaluationReport:
|
||||
"""Run full numeric extraction evaluation.
|
||||
|
||||
Aligns predicted facts to gold facts by fact_type and predicate,
|
||||
then computes exact match accuracy, tolerance-based accuracy,
|
||||
per-type breakdowns, unit consistency, period match accuracy,
|
||||
tolerance distribution histogram, and error categories.
|
||||
|
||||
Args:
|
||||
predicted: All predicted numeric facts.
|
||||
gold: All gold standard numeric facts.
|
||||
tolerance_pct: Tolerance percentage for approximate matching.
|
||||
document_count: Number of documents evaluated.
|
||||
|
||||
Returns:
|
||||
NumericEvaluationReport with complete evaluation results.
|
||||
"""
|
||||
pairs = _align_facts(predicted, gold)
|
||||
total_aligned = len(pairs)
|
||||
|
||||
# Track results
|
||||
exact_matches = 0
|
||||
tolerance_matches = 0
|
||||
unit_matches = 0
|
||||
period_matches = 0
|
||||
comparable_count = 0
|
||||
|
||||
# Per-type tracking
|
||||
per_type_exact_counts: dict[str, tuple[int, int]] = {} # type -> (matches, total)
|
||||
per_type_tol_counts: dict[str, tuple[int, int]] = {}
|
||||
|
||||
# Tolerance distribution
|
||||
dist = ToleranceDistribution()
|
||||
|
||||
# Error tracking
|
||||
error_counts: dict[str, int] = {}
|
||||
|
||||
for pred, g in pairs:
|
||||
result = match_numeric_fact(pred, g, tolerance_pct)
|
||||
|
||||
# Unit consistency
|
||||
if result.unit_consistent:
|
||||
unit_matches += 1
|
||||
|
||||
# Period match
|
||||
if result.period_match:
|
||||
period_matches += 1
|
||||
|
||||
# Only count value comparisons when both values exist
|
||||
if pred.normalized_value is not None and g.normalized_value is not None:
|
||||
comparable_count += 1
|
||||
|
||||
if result.exact_match:
|
||||
exact_matches += 1
|
||||
if result.within_tolerance:
|
||||
tolerance_matches += 1
|
||||
|
||||
# Per-type tracking
|
||||
ft = pred.fact_type
|
||||
ex_m, ex_t = per_type_exact_counts.get(ft, (0, 0))
|
||||
tol_m, tol_t = per_type_tol_counts.get(ft, (0, 0))
|
||||
per_type_exact_counts[ft] = (
|
||||
ex_m + (1 if result.exact_match else 0),
|
||||
ex_t + 1,
|
||||
)
|
||||
per_type_tol_counts[ft] = (
|
||||
tol_m + (1 if result.within_tolerance else 0),
|
||||
tol_t + 1,
|
||||
)
|
||||
|
||||
# Tolerance distribution
|
||||
bucket = _bucket_relative_error(result.relative_error_pct)
|
||||
if bucket == "exact":
|
||||
dist.exact += 1
|
||||
elif bucket == "within_1pct":
|
||||
dist.within_1pct += 1
|
||||
elif bucket == "within_5pct":
|
||||
dist.within_5pct += 1
|
||||
elif bucket == "within_10pct":
|
||||
dist.within_10pct += 1
|
||||
elif bucket == "beyond_10pct":
|
||||
dist.beyond_10pct += 1
|
||||
else:
|
||||
dist.not_comparable += 1
|
||||
|
||||
# Error classification for non-exact matches
|
||||
if not result.exact_match:
|
||||
error_cat = _classify_error(pred, g, pred.normalized_value, g.normalized_value)
|
||||
if error_cat:
|
||||
error_counts[error_cat] = error_counts.get(error_cat, 0) + 1
|
||||
else:
|
||||
dist.not_comparable += 1
|
||||
# Classify missing value error
|
||||
if pred.normalized_value is None:
|
||||
error_cat = ErrorCategory.parsing_failure.value
|
||||
elif g.normalized_value is None:
|
||||
error_cat = ErrorCategory.missing_value.value
|
||||
else:
|
||||
error_cat = None
|
||||
if error_cat:
|
||||
error_counts[error_cat] = error_counts.get(error_cat, 0) + 1
|
||||
|
||||
# Build accuracy metrics
|
||||
exact_accuracy = AccuracyMetric(
|
||||
accuracy=exact_matches / comparable_count if comparable_count > 0 else 1.0,
|
||||
matches=exact_matches,
|
||||
total=comparable_count,
|
||||
)
|
||||
tolerance_accuracy = AccuracyMetric(
|
||||
accuracy=tolerance_matches / comparable_count if comparable_count > 0 else 1.0,
|
||||
matches=tolerance_matches,
|
||||
total=comparable_count,
|
||||
)
|
||||
unit_consistency = AccuracyMetric(
|
||||
accuracy=unit_matches / total_aligned if total_aligned > 0 else 1.0,
|
||||
matches=unit_matches,
|
||||
total=total_aligned,
|
||||
)
|
||||
period_match_metric = AccuracyMetric(
|
||||
accuracy=period_matches / total_aligned if total_aligned > 0 else 1.0,
|
||||
matches=period_matches,
|
||||
total=total_aligned,
|
||||
)
|
||||
|
||||
# Per-type exact accuracy
|
||||
per_type_exact: dict[str, AccuracyMetric] = {}
|
||||
for ft, (m, t) in sorted(per_type_exact_counts.items()):
|
||||
per_type_exact[ft] = AccuracyMetric(
|
||||
accuracy=m / t if t > 0 else 1.0,
|
||||
matches=m,
|
||||
total=t,
|
||||
)
|
||||
|
||||
# Per-type tolerance accuracy
|
||||
per_type_tolerance: dict[str, AccuracyMetric] = {}
|
||||
for ft, (m, t) in sorted(per_type_tol_counts.items()):
|
||||
per_type_tolerance[ft] = AccuracyMetric(
|
||||
accuracy=m / t if t > 0 else 1.0,
|
||||
matches=m,
|
||||
total=t,
|
||||
)
|
||||
|
||||
total_errors = sum(error_counts.values())
|
||||
|
||||
return NumericEvaluationReport(
|
||||
exact_match_accuracy=exact_accuracy,
|
||||
tolerance_accuracy=tolerance_accuracy,
|
||||
tolerance_pct_used=tolerance_pct,
|
||||
per_type_exact=per_type_exact,
|
||||
per_type_tolerance=per_type_tolerance,
|
||||
unit_consistency=unit_consistency,
|
||||
period_match=period_match_metric,
|
||||
tolerance_distribution=dist,
|
||||
error_breakdown=ErrorBreakdown(counts=error_counts, total_errors=total_errors),
|
||||
document_count=document_count,
|
||||
)
|
||||
@@ -0,0 +1,599 @@
|
||||
"""Per-document-type and per-difficulty evaluation report generator.
|
||||
|
||||
Groups evaluation results by document type and difficulty bucket,
|
||||
runs all individual metric computations per group, and produces a
|
||||
FullEvaluationReport with overall + per-type + per-difficulty breakdowns.
|
||||
|
||||
Validates: Requirements 16.3, 16.4
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from enum import Enum
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from services.intelligence_pipeline_v3.evaluation.entity_metrics import (
|
||||
EntityEvaluationReport,
|
||||
EntitySpan,
|
||||
MatchMode,
|
||||
TickerMention,
|
||||
evaluate_entities,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.evaluation.event_metrics import (
|
||||
EventRelationEvaluationReport,
|
||||
GoldEvent,
|
||||
GoldRelation,
|
||||
PredictedEvent,
|
||||
PredictedRelation,
|
||||
evaluate_events_and_relations,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.evaluation.evidence_metrics import (
|
||||
EvidenceMetricsResult,
|
||||
EvidenceSpan,
|
||||
ExtractionResult,
|
||||
evaluate_evidence,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.evaluation.numeric_metrics import (
|
||||
NumericEvaluationReport,
|
||||
NumericFact,
|
||||
evaluate_numeric_facts,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.evaluation.resource_metrics import (
|
||||
ResourceEvaluationReport,
|
||||
StageTimingRecord,
|
||||
evaluate_resources,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.evaluation.sentiment_metrics import (
|
||||
SentimentEvaluationReport,
|
||||
SentimentPrediction,
|
||||
evaluate_sentiment,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Domain Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class DocumentType(str, Enum):
|
||||
"""Document types in the evaluation corpus."""
|
||||
|
||||
article = "article"
|
||||
filing = "filing"
|
||||
transcript = "transcript"
|
||||
press_release = "press_release"
|
||||
macro_event = "macro_event"
|
||||
|
||||
|
||||
class Difficulty(str, Enum):
|
||||
"""Difficulty buckets for evaluation stratification."""
|
||||
|
||||
easy = "easy"
|
||||
medium = "medium"
|
||||
hard = "hard"
|
||||
|
||||
|
||||
class DocumentResult(BaseModel):
|
||||
"""Holds all metric inputs for a single evaluated document."""
|
||||
|
||||
document_id: str
|
||||
document_type: DocumentType
|
||||
difficulty: Difficulty
|
||||
|
||||
# Entity metric inputs
|
||||
predicted_entities: list[EntitySpan] = Field(default_factory=list)
|
||||
gold_entities: list[EntitySpan] = Field(default_factory=list)
|
||||
predicted_tickers: list[TickerMention] = Field(default_factory=list)
|
||||
gold_tickers: list[TickerMention] = Field(default_factory=list)
|
||||
|
||||
# Event/relation metric inputs
|
||||
predicted_events: list[PredictedEvent] = Field(default_factory=list)
|
||||
gold_events: list[GoldEvent] = Field(default_factory=list)
|
||||
predicted_relations: list[PredictedRelation] = Field(default_factory=list)
|
||||
gold_relations: list[GoldRelation] = Field(default_factory=list)
|
||||
|
||||
# Numeric metric inputs
|
||||
predicted_numeric_facts: list[NumericFact] = Field(default_factory=list)
|
||||
gold_numeric_facts: list[NumericFact] = Field(default_factory=list)
|
||||
|
||||
# Evidence metric inputs
|
||||
evidence_spans: list[EvidenceSpan] = Field(default_factory=list)
|
||||
source_text: str = ""
|
||||
extraction_results: list[ExtractionResult] = Field(default_factory=list)
|
||||
|
||||
# Sentiment metric inputs
|
||||
predicted_sentiments: list[SentimentPrediction] = Field(default_factory=list)
|
||||
gold_sentiments: list[SentimentPrediction] = Field(default_factory=list)
|
||||
|
||||
# Resource metric inputs
|
||||
stage_timings: list[StageTimingRecord] = Field(default_factory=list)
|
||||
|
||||
model_config = {"arbitrary_types_allowed": True}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Safety Gate Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SafetyGateThresholds(BaseModel):
|
||||
"""Configurable thresholds for safety gate pass/fail."""
|
||||
|
||||
min_entity_f1: float = Field(default=0.7, ge=0.0, le=1.0)
|
||||
min_event_macro_f1: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
min_evidence_support_rate: float = Field(default=0.8, ge=0.0, le=1.0)
|
||||
max_unsupported_claim_rate: float = Field(default=0.2, ge=0.0, le=1.0)
|
||||
min_sentiment_macro_f1: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
max_calibration_ece: float = Field(default=0.15, ge=0.0, le=1.0)
|
||||
|
||||
|
||||
class SafetyGateResult(BaseModel):
|
||||
"""Result of safety gate evaluation."""
|
||||
|
||||
passed: bool
|
||||
checks: dict[str, bool] = Field(default_factory=dict)
|
||||
details: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Report Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class GroupMetrics(BaseModel):
|
||||
"""Metrics for a single group (document type or difficulty bucket)."""
|
||||
|
||||
group_name: str
|
||||
document_count: int = Field(ge=0)
|
||||
entity_metrics: EntityEvaluationReport | None = None
|
||||
event_metrics: EventRelationEvaluationReport | None = None
|
||||
numeric_metrics: NumericEvaluationReport | None = None
|
||||
evidence_metrics: EvidenceMetricsResult | None = None
|
||||
sentiment_metrics: SentimentEvaluationReport | None = None
|
||||
resource_metrics: ResourceEvaluationReport | None = None
|
||||
|
||||
|
||||
class FullEvaluationReport(BaseModel):
|
||||
"""Complete evaluation report with overall + per-type + per-difficulty breakdowns."""
|
||||
|
||||
overall: GroupMetrics
|
||||
per_document_type: dict[str, GroupMetrics] = Field(default_factory=dict)
|
||||
per_difficulty: dict[str, GroupMetrics] = Field(default_factory=dict)
|
||||
safety_gate: SafetyGateResult
|
||||
total_documents: int = Field(ge=0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core Computation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _compute_group_metrics(
|
||||
group_name: str,
|
||||
documents: list[DocumentResult],
|
||||
entity_match_mode: MatchMode = MatchMode.strict,
|
||||
) -> GroupMetrics:
|
||||
"""Compute all metrics for a group of documents.
|
||||
|
||||
Aggregates all individual document inputs into combined lists and
|
||||
runs each metric computation once for the group.
|
||||
"""
|
||||
if not documents:
|
||||
return GroupMetrics(group_name=group_name, document_count=0)
|
||||
|
||||
doc_count = len(documents)
|
||||
|
||||
# Aggregate entity inputs
|
||||
all_pred_entities: list[EntitySpan] = []
|
||||
all_gold_entities: list[EntitySpan] = []
|
||||
all_pred_tickers: list[TickerMention] = []
|
||||
all_gold_tickers: list[TickerMention] = []
|
||||
|
||||
for doc in documents:
|
||||
all_pred_entities.extend(doc.predicted_entities)
|
||||
all_gold_entities.extend(doc.gold_entities)
|
||||
all_pred_tickers.extend(doc.predicted_tickers)
|
||||
all_gold_tickers.extend(doc.gold_tickers)
|
||||
|
||||
entity_report = evaluate_entities(
|
||||
predicted_entities=all_pred_entities,
|
||||
gold_entities=all_gold_entities,
|
||||
predicted_tickers=all_pred_tickers,
|
||||
gold_tickers=all_gold_tickers,
|
||||
mode=entity_match_mode,
|
||||
document_count=doc_count,
|
||||
)
|
||||
|
||||
# Aggregate event/relation inputs
|
||||
all_pred_events: list[PredictedEvent] = []
|
||||
all_gold_events: list[GoldEvent] = []
|
||||
all_pred_relations: list[PredictedRelation] = []
|
||||
all_gold_relations: list[GoldRelation] = []
|
||||
|
||||
for doc in documents:
|
||||
all_pred_events.extend(doc.predicted_events)
|
||||
all_gold_events.extend(doc.gold_events)
|
||||
all_pred_relations.extend(doc.predicted_relations)
|
||||
all_gold_relations.extend(doc.gold_relations)
|
||||
|
||||
event_report = evaluate_events_and_relations(
|
||||
predicted_events=all_pred_events,
|
||||
gold_events=all_gold_events,
|
||||
predicted_relations=all_pred_relations,
|
||||
gold_relations=all_gold_relations,
|
||||
document_count=doc_count,
|
||||
)
|
||||
|
||||
# Aggregate numeric inputs
|
||||
all_pred_numeric: list[NumericFact] = []
|
||||
all_gold_numeric: list[NumericFact] = []
|
||||
|
||||
for doc in documents:
|
||||
all_pred_numeric.extend(doc.predicted_numeric_facts)
|
||||
all_gold_numeric.extend(doc.gold_numeric_facts)
|
||||
|
||||
numeric_report = evaluate_numeric_facts(
|
||||
predicted=all_pred_numeric,
|
||||
gold=all_gold_numeric,
|
||||
document_count=doc_count,
|
||||
)
|
||||
|
||||
# Aggregate evidence inputs — concatenate source texts with separator
|
||||
all_spans: list[EvidenceSpan] = []
|
||||
all_items: list[ExtractionResult] = []
|
||||
combined_source = ""
|
||||
|
||||
for doc in documents:
|
||||
offset = len(combined_source)
|
||||
# Adjust span offsets for combined source
|
||||
for span in doc.evidence_spans:
|
||||
all_spans.append(
|
||||
EvidenceSpan(
|
||||
span_id=span.span_id,
|
||||
text=span.text,
|
||||
start_char=span.start_char + offset,
|
||||
end_char=span.end_char + offset,
|
||||
document_id=span.document_id or doc.document_id,
|
||||
)
|
||||
)
|
||||
all_items.extend(doc.extraction_results)
|
||||
combined_source += doc.source_text
|
||||
|
||||
evidence_report = evaluate_evidence(
|
||||
spans=all_spans,
|
||||
source_text=combined_source,
|
||||
items=all_items,
|
||||
)
|
||||
|
||||
# Aggregate sentiment inputs
|
||||
all_pred_sentiments: list[SentimentPrediction] = []
|
||||
all_gold_sentiments: list[SentimentPrediction] = []
|
||||
|
||||
for doc in documents:
|
||||
all_pred_sentiments.extend(doc.predicted_sentiments)
|
||||
all_gold_sentiments.extend(doc.gold_sentiments)
|
||||
|
||||
sentiment_report = evaluate_sentiment(
|
||||
predicted=all_pred_sentiments,
|
||||
gold=all_gold_sentiments,
|
||||
document_count=doc_count,
|
||||
)
|
||||
|
||||
# Aggregate resource inputs
|
||||
all_timings: list[StageTimingRecord] = []
|
||||
for doc in documents:
|
||||
all_timings.extend(doc.stage_timings)
|
||||
|
||||
resource_report = evaluate_resources(records=all_timings)
|
||||
|
||||
return GroupMetrics(
|
||||
group_name=group_name,
|
||||
document_count=doc_count,
|
||||
entity_metrics=entity_report,
|
||||
event_metrics=event_report,
|
||||
numeric_metrics=numeric_report,
|
||||
evidence_metrics=evidence_report,
|
||||
sentiment_metrics=sentiment_report,
|
||||
resource_metrics=resource_report,
|
||||
)
|
||||
|
||||
|
||||
def _evaluate_safety_gate(
|
||||
overall: GroupMetrics,
|
||||
thresholds: SafetyGateThresholds,
|
||||
) -> SafetyGateResult:
|
||||
"""Evaluate safety gate thresholds against overall metrics."""
|
||||
checks: dict[str, bool] = {}
|
||||
details: dict[str, str] = {}
|
||||
|
||||
# Entity F1
|
||||
if overall.entity_metrics:
|
||||
entity_f1 = overall.entity_metrics.entity_metrics.overall.f1
|
||||
passed_entity = entity_f1 >= thresholds.min_entity_f1
|
||||
checks["entity_f1"] = passed_entity
|
||||
details["entity_f1"] = (
|
||||
f"{entity_f1:.3f} {'≥' if passed_entity else '<'} {thresholds.min_entity_f1:.3f}"
|
||||
)
|
||||
else:
|
||||
checks["entity_f1"] = True
|
||||
details["entity_f1"] = "No entity data"
|
||||
|
||||
# Event macro-F1
|
||||
if overall.event_metrics:
|
||||
event_f1 = overall.event_metrics.event_metrics.macro_f1
|
||||
passed_event = event_f1 >= thresholds.min_event_macro_f1
|
||||
checks["event_macro_f1"] = passed_event
|
||||
details["event_macro_f1"] = (
|
||||
f"{event_f1:.3f} {'≥' if passed_event else '<'} {thresholds.min_event_macro_f1:.3f}"
|
||||
)
|
||||
else:
|
||||
checks["event_macro_f1"] = True
|
||||
details["event_macro_f1"] = "No event data"
|
||||
|
||||
# Evidence support rate
|
||||
if overall.evidence_metrics:
|
||||
support_rate = overall.evidence_metrics.support_rate
|
||||
passed_support = support_rate >= thresholds.min_evidence_support_rate
|
||||
checks["evidence_support_rate"] = passed_support
|
||||
details["evidence_support_rate"] = (
|
||||
f"{support_rate:.3f} {'≥' if passed_support else '<'} "
|
||||
f"{thresholds.min_evidence_support_rate:.3f}"
|
||||
)
|
||||
|
||||
unsupported = overall.evidence_metrics.unsupported_claim_rate
|
||||
passed_unsupported = unsupported <= thresholds.max_unsupported_claim_rate
|
||||
checks["unsupported_claim_rate"] = passed_unsupported
|
||||
details["unsupported_claim_rate"] = (
|
||||
f"{unsupported:.3f} {'≤' if passed_unsupported else '>'} "
|
||||
f"{thresholds.max_unsupported_claim_rate:.3f}"
|
||||
)
|
||||
else:
|
||||
checks["evidence_support_rate"] = True
|
||||
checks["unsupported_claim_rate"] = True
|
||||
details["evidence_support_rate"] = "No evidence data"
|
||||
details["unsupported_claim_rate"] = "No evidence data"
|
||||
|
||||
# Sentiment macro-F1
|
||||
if overall.sentiment_metrics:
|
||||
sent_f1 = overall.sentiment_metrics.f1_metrics.macro_f1
|
||||
passed_sent = sent_f1 >= thresholds.min_sentiment_macro_f1
|
||||
checks["sentiment_macro_f1"] = passed_sent
|
||||
details["sentiment_macro_f1"] = (
|
||||
f"{sent_f1:.3f} {'≥' if passed_sent else '<'} "
|
||||
f"{thresholds.min_sentiment_macro_f1:.3f}"
|
||||
)
|
||||
|
||||
ece = overall.sentiment_metrics.calibration.ece
|
||||
passed_ece = ece <= thresholds.max_calibration_ece
|
||||
checks["calibration_ece"] = passed_ece
|
||||
details["calibration_ece"] = (
|
||||
f"{ece:.3f} {'≤' if passed_ece else '>'} {thresholds.max_calibration_ece:.3f}"
|
||||
)
|
||||
else:
|
||||
checks["sentiment_macro_f1"] = True
|
||||
checks["calibration_ece"] = True
|
||||
details["sentiment_macro_f1"] = "No sentiment data"
|
||||
details["calibration_ece"] = "No sentiment data"
|
||||
|
||||
all_passed = all(checks.values())
|
||||
|
||||
return SafetyGateResult(
|
||||
passed=all_passed,
|
||||
checks=checks,
|
||||
details=details,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def generate_evaluation_report(
|
||||
documents: list[DocumentResult],
|
||||
entity_match_mode: MatchMode = MatchMode.strict,
|
||||
safety_thresholds: SafetyGateThresholds | None = None,
|
||||
) -> FullEvaluationReport:
|
||||
"""Generate a full evaluation report with per-type and per-difficulty breakdowns.
|
||||
|
||||
Groups documents by document_type and difficulty, runs all metrics per group,
|
||||
and evaluates safety gate thresholds against overall results.
|
||||
|
||||
Args:
|
||||
documents: List of DocumentResult objects with all metric inputs.
|
||||
entity_match_mode: Matching strategy for entity metrics.
|
||||
safety_thresholds: Configurable safety gate thresholds (uses defaults if None).
|
||||
|
||||
Returns:
|
||||
FullEvaluationReport with overall, per-type, per-difficulty, and safety gate.
|
||||
"""
|
||||
if safety_thresholds is None:
|
||||
safety_thresholds = SafetyGateThresholds()
|
||||
|
||||
# Overall metrics
|
||||
overall = _compute_group_metrics("overall", documents, entity_match_mode)
|
||||
|
||||
# Group by document type
|
||||
by_type: dict[str, list[DocumentResult]] = defaultdict(list)
|
||||
for doc in documents:
|
||||
by_type[doc.document_type.value].append(doc)
|
||||
|
||||
per_document_type: dict[str, GroupMetrics] = {}
|
||||
for doc_type in DocumentType:
|
||||
type_docs = by_type.get(doc_type.value, [])
|
||||
if type_docs:
|
||||
per_document_type[doc_type.value] = _compute_group_metrics(
|
||||
doc_type.value, type_docs, entity_match_mode
|
||||
)
|
||||
|
||||
# Group by difficulty
|
||||
by_difficulty: dict[str, list[DocumentResult]] = defaultdict(list)
|
||||
for doc in documents:
|
||||
by_difficulty[doc.difficulty.value].append(doc)
|
||||
|
||||
per_difficulty: dict[str, GroupMetrics] = {}
|
||||
for diff in Difficulty:
|
||||
diff_docs = by_difficulty.get(diff.value, [])
|
||||
if diff_docs:
|
||||
per_difficulty[diff.value] = _compute_group_metrics(
|
||||
diff.value, diff_docs, entity_match_mode
|
||||
)
|
||||
|
||||
# Safety gate evaluation
|
||||
safety_gate = _evaluate_safety_gate(overall, safety_thresholds)
|
||||
|
||||
return FullEvaluationReport(
|
||||
overall=overall,
|
||||
per_document_type=per_document_type,
|
||||
per_difficulty=per_difficulty,
|
||||
safety_gate=safety_gate,
|
||||
total_documents=len(documents),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Markdown Formatter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _format_prf1_row(label: str, p: float, r: float, f1: float, support: int) -> str:
|
||||
"""Format a single PRF1 row for a markdown table."""
|
||||
return f"| {label} | {p:.3f} | {r:.3f} | {f1:.3f} | {support} |"
|
||||
|
||||
|
||||
def _format_group_section(group: GroupMetrics, heading_level: int = 3) -> str:
|
||||
"""Format a single group's metrics as markdown."""
|
||||
prefix = "#" * heading_level
|
||||
lines: list[str] = []
|
||||
lines.append(f"{prefix} {group.group_name} ({group.document_count} documents)")
|
||||
lines.append("")
|
||||
|
||||
# Entity metrics
|
||||
if group.entity_metrics:
|
||||
em = group.entity_metrics
|
||||
lines.append(f"{prefix}# Entity Metrics")
|
||||
lines.append("")
|
||||
lines.append("| Metric | Precision | Recall | F1 | Support |")
|
||||
lines.append("|--------|-----------|--------|-----|---------|")
|
||||
o = em.entity_metrics.overall
|
||||
lines.append(_format_prf1_row("Entities (overall)", o.precision, o.recall, o.f1, o.support_gold))
|
||||
t = em.ticker_metrics.overall
|
||||
lines.append(_format_prf1_row("Tickers (overall)", t.precision, t.recall, t.f1, t.support_gold))
|
||||
lines.append("")
|
||||
lines.append(f"Ambiguity accuracy: {em.ambiguity_accuracy.accuracy:.3f}")
|
||||
lines.append("")
|
||||
|
||||
# Event metrics
|
||||
if group.event_metrics:
|
||||
ev = group.event_metrics
|
||||
lines.append(f"{prefix}# Event & Relation Metrics")
|
||||
lines.append("")
|
||||
lines.append(f"- Event macro-F1: {ev.event_metrics.macro_f1:.3f}")
|
||||
micro = ev.event_metrics.micro
|
||||
lines.append(f"- Event micro-F1: {micro.f1:.3f} (P={micro.precision:.3f}, R={micro.recall:.3f})")
|
||||
lines.append(f"- Relation macro-F1: {ev.relation_metrics.macro_f1:.3f}")
|
||||
r_micro = ev.relation_metrics.micro
|
||||
lines.append(f"- Relation micro-F1: {r_micro.f1:.3f} (P={r_micro.precision:.3f}, R={r_micro.recall:.3f})")
|
||||
lines.append("")
|
||||
|
||||
# Numeric metrics
|
||||
if group.numeric_metrics:
|
||||
nm = group.numeric_metrics
|
||||
lines.append(f"{prefix}# Numeric Metrics")
|
||||
lines.append("")
|
||||
lines.append(f"- Exact match accuracy: {nm.exact_match_accuracy.accuracy:.3f} ({nm.exact_match_accuracy.matches}/{nm.exact_match_accuracy.total})")
|
||||
lines.append(f"- Tolerance accuracy ({nm.tolerance_pct_used}%): {nm.tolerance_accuracy.accuracy:.3f} ({nm.tolerance_accuracy.matches}/{nm.tolerance_accuracy.total})")
|
||||
lines.append(f"- Unit consistency: {nm.unit_consistency.accuracy:.3f}")
|
||||
lines.append(f"- Period match: {nm.period_match.accuracy:.3f}")
|
||||
lines.append("")
|
||||
|
||||
# Evidence metrics
|
||||
if group.evidence_metrics:
|
||||
ev = group.evidence_metrics
|
||||
lines.append(f"{prefix}# Evidence Metrics")
|
||||
lines.append("")
|
||||
lines.append(f"- Offset validity rate: {ev.validity_rate:.3f} ({ev.valid_spans}/{ev.total_spans})")
|
||||
lines.append(f"- Support rate: {ev.support_rate:.3f} ({ev.supported_items}/{ev.total_items})")
|
||||
lines.append(f"- Coverage score: {ev.coverage_score:.3f}")
|
||||
lines.append(f"- Orphan rate: {ev.orphan_rate:.3f} ({ev.orphan_spans} orphans)")
|
||||
lines.append(f"- Unsupported claim rate: {ev.unsupported_claim_rate:.3f}")
|
||||
lines.append("")
|
||||
|
||||
# Sentiment metrics
|
||||
if group.sentiment_metrics:
|
||||
sm = group.sentiment_metrics
|
||||
lines.append(f"{prefix}# Sentiment Metrics")
|
||||
lines.append("")
|
||||
lines.append(f"- Macro-F1: {sm.f1_metrics.macro_f1:.3f}")
|
||||
lines.append(f"- Micro-F1: {sm.f1_metrics.micro_f1:.3f}")
|
||||
lines.append(f"- Direction accuracy: {sm.direction_accuracy.accuracy:.3f}")
|
||||
lines.append(f"- Calibration ECE: {sm.calibration.ece:.3f}")
|
||||
lines.append(f"- Brier score: {sm.calibration.brier_score:.3f}")
|
||||
lines.append("")
|
||||
|
||||
# Resource metrics
|
||||
if group.resource_metrics:
|
||||
rm = group.resource_metrics
|
||||
lines.append(f"{prefix}# Resource Metrics")
|
||||
lines.append("")
|
||||
lines.append(f"- Latency p50: {rm.latency.p50:.2f}s, p95: {rm.latency.p95:.2f}s, p99: {rm.latency.p99:.2f}s")
|
||||
lines.append(f"- Throughput: {rm.throughput.documents_per_minute:.1f} docs/min")
|
||||
lines.append(f"- Total tokens: {rm.token_usage.total_tokens}")
|
||||
lines.append(f"- CPU: {rm.cpu.total_cpu_seconds:.1f}s total, {rm.cpu.mean_cpu_seconds_per_document:.2f}s/doc")
|
||||
lines.append(f"- GPU: {rm.gpu.total_gpu_seconds:.1f}s total, peak {rm.gpu.peak_gpu_memory_mb:.0f} MB")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_report_markdown(report: FullEvaluationReport) -> str:
|
||||
"""Produce a readable markdown summary of the full evaluation report.
|
||||
|
||||
Args:
|
||||
report: The complete evaluation report.
|
||||
|
||||
Returns:
|
||||
Markdown-formatted string with all sections.
|
||||
"""
|
||||
lines: list[str] = []
|
||||
|
||||
lines.append("# Intelligence Pipeline v3 — Evaluation Report")
|
||||
lines.append("")
|
||||
lines.append(f"**Total documents evaluated:** {report.total_documents}")
|
||||
lines.append("")
|
||||
|
||||
# Safety gate summary
|
||||
lines.append("## Safety Gate")
|
||||
lines.append("")
|
||||
gate = report.safety_gate
|
||||
status = "✅ PASSED" if gate.passed else "❌ FAILED"
|
||||
lines.append(f"**Status:** {status}")
|
||||
lines.append("")
|
||||
lines.append("| Check | Result | Details |")
|
||||
lines.append("|-------|--------|---------|")
|
||||
for check_name, passed in gate.checks.items():
|
||||
icon = "✅" if passed else "❌"
|
||||
detail = gate.details.get(check_name, "")
|
||||
lines.append(f"| {check_name} | {icon} | {detail} |")
|
||||
lines.append("")
|
||||
|
||||
# Overall metrics
|
||||
lines.append("## Overall Metrics")
|
||||
lines.append("")
|
||||
lines.append(_format_group_section(report.overall, heading_level=3))
|
||||
|
||||
# Per document type
|
||||
if report.per_document_type:
|
||||
lines.append("## Per Document Type")
|
||||
lines.append("")
|
||||
for doc_type, group in sorted(report.per_document_type.items()):
|
||||
lines.append(_format_group_section(group, heading_level=3))
|
||||
|
||||
# Per difficulty
|
||||
if report.per_difficulty:
|
||||
lines.append("## Per Difficulty")
|
||||
lines.append("")
|
||||
for diff, group in sorted(report.per_difficulty.items()):
|
||||
lines.append(_format_group_section(group, heading_level=3))
|
||||
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,660 @@
|
||||
"""Latency, throughput, token, CPU, GPU, and memory resource metrics.
|
||||
|
||||
Implements evaluation metrics for pipeline resource consumption and efficiency.
|
||||
Supports per-document and per-stage breakdowns with percentile calculations.
|
||||
|
||||
Validates: Requirements 16.3, 16.4
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StageTimingRecord:
|
||||
"""A single stage execution record with resource measurements.
|
||||
|
||||
Captures timing, token usage, and hardware resource consumption
|
||||
for one processing stage of one document.
|
||||
"""
|
||||
|
||||
document_id: str
|
||||
stage_name: str
|
||||
start_time: float # Unix timestamp (seconds)
|
||||
end_time: float # Unix timestamp (seconds)
|
||||
input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
gpu_memory_mb: float = 0.0
|
||||
cpu_seconds: float = 0.0
|
||||
gpu_seconds: float = 0.0
|
||||
|
||||
@property
|
||||
def duration_seconds(self) -> float:
|
||||
"""Wall-clock duration of this stage in seconds."""
|
||||
return self.end_time - self.start_time
|
||||
|
||||
@property
|
||||
def total_tokens(self) -> int:
|
||||
"""Sum of input and output tokens."""
|
||||
return self.input_tokens + self.output_tokens
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Percentile Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_percentile(values: list[float], percentile: float) -> float:
|
||||
"""Compute a percentile from a sorted list without numpy.
|
||||
|
||||
Uses linear interpolation between nearest ranks.
|
||||
|
||||
Args:
|
||||
values: List of numeric values (need not be pre-sorted).
|
||||
percentile: Percentile to compute (0-100).
|
||||
|
||||
Returns:
|
||||
The interpolated percentile value.
|
||||
|
||||
Raises:
|
||||
ValueError: If values is empty or percentile is out of range.
|
||||
"""
|
||||
if not values:
|
||||
raise ValueError("Cannot compute percentile of empty list")
|
||||
if not (0.0 <= percentile <= 100.0):
|
||||
raise ValueError(f"Percentile must be between 0 and 100, got {percentile}")
|
||||
|
||||
sorted_values = sorted(values)
|
||||
n = len(sorted_values)
|
||||
|
||||
if n == 1:
|
||||
return sorted_values[0]
|
||||
|
||||
# Compute the rank (0-indexed fractional position)
|
||||
rank = (percentile / 100.0) * (n - 1)
|
||||
lower_idx = int(rank)
|
||||
upper_idx = lower_idx + 1
|
||||
fraction = rank - lower_idx
|
||||
|
||||
if upper_idx >= n:
|
||||
return sorted_values[-1]
|
||||
|
||||
return sorted_values[lower_idx] + fraction * (
|
||||
sorted_values[upper_idx] - sorted_values[lower_idx]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class LatencyPercentiles(BaseModel):
|
||||
"""Latency percentile distribution in seconds."""
|
||||
|
||||
p50: float = Field(ge=0.0)
|
||||
p90: float = Field(ge=0.0)
|
||||
p95: float = Field(ge=0.0)
|
||||
p99: float = Field(ge=0.0)
|
||||
mean: float = Field(ge=0.0)
|
||||
max: float = Field(ge=0.0)
|
||||
min: float = Field(ge=0.0)
|
||||
count: int = Field(ge=0)
|
||||
|
||||
|
||||
class ThroughputMetrics(BaseModel):
|
||||
"""Document throughput measurements."""
|
||||
|
||||
documents_per_minute: float = Field(ge=0.0)
|
||||
documents_per_hour: float = Field(ge=0.0)
|
||||
total_documents: int = Field(ge=0)
|
||||
total_wall_seconds: float = Field(ge=0.0)
|
||||
|
||||
|
||||
class TokenUsageMetrics(BaseModel):
|
||||
"""Token consumption statistics."""
|
||||
|
||||
total_input_tokens: int = Field(ge=0)
|
||||
total_output_tokens: int = Field(ge=0)
|
||||
total_tokens: int = Field(ge=0)
|
||||
mean_input_tokens_per_document: float = Field(ge=0.0)
|
||||
mean_output_tokens_per_document: float = Field(ge=0.0)
|
||||
mean_total_tokens_per_document: float = Field(ge=0.0)
|
||||
per_stage: dict[str, "StageTokenUsage"] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class StageTokenUsage(BaseModel):
|
||||
"""Token usage breakdown for a single stage."""
|
||||
|
||||
total_input_tokens: int = Field(ge=0)
|
||||
total_output_tokens: int = Field(ge=0)
|
||||
total_tokens: int = Field(ge=0)
|
||||
mean_input_tokens: float = Field(ge=0.0)
|
||||
mean_output_tokens: float = Field(ge=0.0)
|
||||
mean_total_tokens: float = Field(ge=0.0)
|
||||
count: int = Field(ge=0)
|
||||
|
||||
|
||||
class CpuMetrics(BaseModel):
|
||||
"""CPU resource consumption metrics."""
|
||||
|
||||
total_cpu_seconds: float = Field(ge=0.0)
|
||||
mean_cpu_seconds_per_document: float = Field(ge=0.0)
|
||||
peak_cpu_seconds: float = Field(ge=0.0, description="Max CPU-seconds for a single document")
|
||||
|
||||
|
||||
class GpuMetrics(BaseModel):
|
||||
"""GPU resource consumption metrics."""
|
||||
|
||||
total_gpu_seconds: float = Field(ge=0.0)
|
||||
mean_gpu_seconds_per_document: float = Field(ge=0.0)
|
||||
peak_gpu_memory_mb: float = Field(ge=0.0)
|
||||
mean_gpu_memory_mb: float = Field(ge=0.0)
|
||||
gpu_utilization_percent: float = Field(
|
||||
ge=0.0, le=100.0,
|
||||
description="Percentage of total wall time spent on GPU",
|
||||
)
|
||||
|
||||
|
||||
class MemoryMetrics(BaseModel):
|
||||
"""Memory consumption metrics."""
|
||||
|
||||
peak_rss_memory_mb: float = Field(ge=0.0)
|
||||
mean_working_set_mb: float = Field(ge=0.0)
|
||||
|
||||
|
||||
class EfficiencyMetrics(BaseModel):
|
||||
"""Efficiency ratio metrics."""
|
||||
|
||||
tokens_per_second: float = Field(ge=0.0)
|
||||
documents_per_gpu_second: float = Field(ge=0.0)
|
||||
fast_path_cpu_seconds: float = Field(ge=0.0)
|
||||
adjudication_cpu_seconds: float = Field(ge=0.0)
|
||||
fast_path_gpu_seconds: float = Field(ge=0.0)
|
||||
adjudication_gpu_seconds: float = Field(ge=0.0)
|
||||
fast_path_fraction: float = Field(
|
||||
ge=0.0, le=1.0,
|
||||
description="Fraction of total resource usage from fast-path stages",
|
||||
)
|
||||
adjudication_fraction: float = Field(
|
||||
ge=0.0, le=1.0,
|
||||
description="Fraction of total resource usage from adjudication stages",
|
||||
)
|
||||
|
||||
|
||||
class StageLatencyBreakdown(BaseModel):
|
||||
"""Per-stage latency statistics."""
|
||||
|
||||
stage_name: str
|
||||
latency: LatencyPercentiles
|
||||
invocation_count: int = Field(ge=0)
|
||||
|
||||
|
||||
class ResourceEvaluationReport(BaseModel):
|
||||
"""Complete resource evaluation report."""
|
||||
|
||||
latency: LatencyPercentiles
|
||||
per_stage_latency: list[StageLatencyBreakdown] = Field(default_factory=list)
|
||||
throughput: ThroughputMetrics
|
||||
token_usage: TokenUsageMetrics
|
||||
cpu: CpuMetrics
|
||||
gpu: GpuMetrics
|
||||
memory: MemoryMetrics
|
||||
efficiency: EfficiencyMetrics
|
||||
document_count: int = Field(ge=0)
|
||||
|
||||
|
||||
# Rebuild model to resolve forward references
|
||||
TokenUsageMetrics.model_rebuild()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Computation Logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Stages considered as "adjudication" for resource split calculations
|
||||
ADJUDICATION_STAGES: frozenset[str] = frozenset({
|
||||
"adjudication",
|
||||
"adjudicator",
|
||||
"9b_adjudication",
|
||||
"semantic_adjudication",
|
||||
})
|
||||
|
||||
|
||||
def _is_adjudication_stage(stage_name: str) -> bool:
|
||||
"""Determine if a stage belongs to the adjudication path."""
|
||||
lower = stage_name.lower()
|
||||
return lower in ADJUDICATION_STAGES or "adjudicat" in lower
|
||||
|
||||
|
||||
def _compute_latency_percentiles(durations: list[float]) -> LatencyPercentiles:
|
||||
"""Compute latency percentile distribution from a list of durations."""
|
||||
if not durations:
|
||||
return LatencyPercentiles(
|
||||
p50=0.0, p90=0.0, p95=0.0, p99=0.0,
|
||||
mean=0.0, max=0.0, min=0.0, count=0,
|
||||
)
|
||||
|
||||
return LatencyPercentiles(
|
||||
p50=compute_percentile(durations, 50.0),
|
||||
p90=compute_percentile(durations, 90.0),
|
||||
p95=compute_percentile(durations, 95.0),
|
||||
p99=compute_percentile(durations, 99.0),
|
||||
mean=sum(durations) / len(durations),
|
||||
max=max(durations),
|
||||
min=min(durations),
|
||||
count=len(durations),
|
||||
)
|
||||
|
||||
|
||||
def _compute_document_durations(
|
||||
records: list[StageTimingRecord],
|
||||
) -> dict[str, float]:
|
||||
"""Compute total wall-clock duration per document.
|
||||
|
||||
Uses min(start_time) to max(end_time) for each document to handle
|
||||
overlapping/parallel stages.
|
||||
"""
|
||||
doc_times: dict[str, tuple[float, float]] = {}
|
||||
for r in records:
|
||||
if r.document_id not in doc_times:
|
||||
doc_times[r.document_id] = (r.start_time, r.end_time)
|
||||
else:
|
||||
existing = doc_times[r.document_id]
|
||||
doc_times[r.document_id] = (
|
||||
min(existing[0], r.start_time),
|
||||
max(existing[1], r.end_time),
|
||||
)
|
||||
return {doc_id: end - start for doc_id, (start, end) in doc_times.items()}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_latency_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
) -> tuple[LatencyPercentiles, list[StageLatencyBreakdown]]:
|
||||
"""Compute per-document and per-stage latency metrics.
|
||||
|
||||
Per-document latency is the wall-clock time from the earliest stage
|
||||
start to the latest stage end for each document.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
|
||||
Returns:
|
||||
Tuple of (overall document latency, per-stage breakdown).
|
||||
"""
|
||||
if not records:
|
||||
return (
|
||||
LatencyPercentiles(
|
||||
p50=0.0, p90=0.0, p95=0.0, p99=0.0,
|
||||
mean=0.0, max=0.0, min=0.0, count=0,
|
||||
),
|
||||
[],
|
||||
)
|
||||
|
||||
# Per-document latency
|
||||
doc_durations = _compute_document_durations(records)
|
||||
overall = _compute_latency_percentiles(list(doc_durations.values()))
|
||||
|
||||
# Per-stage latency
|
||||
stage_durations: dict[str, list[float]] = {}
|
||||
for r in records:
|
||||
stage_durations.setdefault(r.stage_name, []).append(r.duration_seconds)
|
||||
|
||||
per_stage = [
|
||||
StageLatencyBreakdown(
|
||||
stage_name=stage,
|
||||
latency=_compute_latency_percentiles(durations),
|
||||
invocation_count=len(durations),
|
||||
)
|
||||
for stage, durations in sorted(stage_durations.items())
|
||||
]
|
||||
|
||||
return overall, per_stage
|
||||
|
||||
|
||||
def compute_throughput_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
) -> ThroughputMetrics:
|
||||
"""Compute document throughput from timing records.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
|
||||
Returns:
|
||||
ThroughputMetrics with documents/minute and documents/hour.
|
||||
"""
|
||||
if not records:
|
||||
return ThroughputMetrics(
|
||||
documents_per_minute=0.0,
|
||||
documents_per_hour=0.0,
|
||||
total_documents=0,
|
||||
total_wall_seconds=0.0,
|
||||
)
|
||||
|
||||
doc_ids = {r.document_id for r in records}
|
||||
total_docs = len(doc_ids)
|
||||
|
||||
# Total wall time: earliest start to latest end across all records
|
||||
earliest = min(r.start_time for r in records)
|
||||
latest = max(r.end_time for r in records)
|
||||
total_wall = latest - earliest
|
||||
|
||||
if total_wall <= 0.0:
|
||||
return ThroughputMetrics(
|
||||
documents_per_minute=0.0,
|
||||
documents_per_hour=0.0,
|
||||
total_documents=total_docs,
|
||||
total_wall_seconds=0.0,
|
||||
)
|
||||
|
||||
docs_per_second = total_docs / total_wall
|
||||
|
||||
return ThroughputMetrics(
|
||||
documents_per_minute=docs_per_second * 60.0,
|
||||
documents_per_hour=docs_per_second * 3600.0,
|
||||
total_documents=total_docs,
|
||||
total_wall_seconds=total_wall,
|
||||
)
|
||||
|
||||
|
||||
def compute_token_usage_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
) -> TokenUsageMetrics:
|
||||
"""Compute token usage statistics per document and per stage.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
|
||||
Returns:
|
||||
TokenUsageMetrics with aggregate and per-stage breakdowns.
|
||||
"""
|
||||
if not records:
|
||||
return TokenUsageMetrics(
|
||||
total_input_tokens=0,
|
||||
total_output_tokens=0,
|
||||
total_tokens=0,
|
||||
mean_input_tokens_per_document=0.0,
|
||||
mean_output_tokens_per_document=0.0,
|
||||
mean_total_tokens_per_document=0.0,
|
||||
per_stage={},
|
||||
)
|
||||
|
||||
total_input = sum(r.input_tokens for r in records)
|
||||
total_output = sum(r.output_tokens for r in records)
|
||||
total = total_input + total_output
|
||||
|
||||
doc_ids = {r.document_id for r in records}
|
||||
n_docs = len(doc_ids)
|
||||
|
||||
# Per-stage breakdown
|
||||
stage_records: dict[str, list[StageTimingRecord]] = {}
|
||||
for r in records:
|
||||
stage_records.setdefault(r.stage_name, []).append(r)
|
||||
|
||||
per_stage: dict[str, StageTokenUsage] = {}
|
||||
for stage, stage_recs in sorted(stage_records.items()):
|
||||
s_input = sum(r.input_tokens for r in stage_recs)
|
||||
s_output = sum(r.output_tokens for r in stage_recs)
|
||||
s_total = s_input + s_output
|
||||
count = len(stage_recs)
|
||||
per_stage[stage] = StageTokenUsage(
|
||||
total_input_tokens=s_input,
|
||||
total_output_tokens=s_output,
|
||||
total_tokens=s_total,
|
||||
mean_input_tokens=s_input / count if count > 0 else 0.0,
|
||||
mean_output_tokens=s_output / count if count > 0 else 0.0,
|
||||
mean_total_tokens=s_total / count if count > 0 else 0.0,
|
||||
count=count,
|
||||
)
|
||||
|
||||
return TokenUsageMetrics(
|
||||
total_input_tokens=total_input,
|
||||
total_output_tokens=total_output,
|
||||
total_tokens=total,
|
||||
mean_input_tokens_per_document=total_input / n_docs if n_docs > 0 else 0.0,
|
||||
mean_output_tokens_per_document=total_output / n_docs if n_docs > 0 else 0.0,
|
||||
mean_total_tokens_per_document=total / n_docs if n_docs > 0 else 0.0,
|
||||
per_stage=per_stage,
|
||||
)
|
||||
|
||||
|
||||
def compute_cpu_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
) -> CpuMetrics:
|
||||
"""Compute CPU resource consumption metrics.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
|
||||
Returns:
|
||||
CpuMetrics with totals and per-document statistics.
|
||||
"""
|
||||
if not records:
|
||||
return CpuMetrics(
|
||||
total_cpu_seconds=0.0,
|
||||
mean_cpu_seconds_per_document=0.0,
|
||||
peak_cpu_seconds=0.0,
|
||||
)
|
||||
|
||||
total_cpu = sum(r.cpu_seconds for r in records)
|
||||
|
||||
# Per-document CPU totals
|
||||
doc_cpu: dict[str, float] = {}
|
||||
for r in records:
|
||||
doc_cpu[r.document_id] = doc_cpu.get(r.document_id, 0.0) + r.cpu_seconds
|
||||
|
||||
n_docs = len(doc_cpu)
|
||||
peak = max(doc_cpu.values()) if doc_cpu else 0.0
|
||||
|
||||
return CpuMetrics(
|
||||
total_cpu_seconds=total_cpu,
|
||||
mean_cpu_seconds_per_document=total_cpu / n_docs if n_docs > 0 else 0.0,
|
||||
peak_cpu_seconds=peak,
|
||||
)
|
||||
|
||||
|
||||
def compute_gpu_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
) -> GpuMetrics:
|
||||
"""Compute GPU resource consumption metrics.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
|
||||
Returns:
|
||||
GpuMetrics with totals, peaks, and utilization percentage.
|
||||
"""
|
||||
if not records:
|
||||
return GpuMetrics(
|
||||
total_gpu_seconds=0.0,
|
||||
mean_gpu_seconds_per_document=0.0,
|
||||
peak_gpu_memory_mb=0.0,
|
||||
mean_gpu_memory_mb=0.0,
|
||||
gpu_utilization_percent=0.0,
|
||||
)
|
||||
|
||||
total_gpu = sum(r.gpu_seconds for r in records)
|
||||
|
||||
# Per-document GPU totals
|
||||
doc_gpu: dict[str, float] = {}
|
||||
for r in records:
|
||||
doc_gpu[r.document_id] = doc_gpu.get(r.document_id, 0.0) + r.gpu_seconds
|
||||
|
||||
n_docs = len(doc_gpu)
|
||||
|
||||
# GPU memory stats
|
||||
gpu_mem_values = [r.gpu_memory_mb for r in records if r.gpu_memory_mb > 0.0]
|
||||
peak_gpu_mem = max(gpu_mem_values) if gpu_mem_values else 0.0
|
||||
mean_gpu_mem = (
|
||||
sum(gpu_mem_values) / len(gpu_mem_values) if gpu_mem_values else 0.0
|
||||
)
|
||||
|
||||
# GPU utilization: fraction of wall time spent on GPU work
|
||||
earliest = min(r.start_time for r in records)
|
||||
latest = max(r.end_time for r in records)
|
||||
total_wall = latest - earliest
|
||||
|
||||
utilization = (
|
||||
(total_gpu / total_wall) * 100.0 if total_wall > 0.0 else 0.0
|
||||
)
|
||||
# Cap at 100% (parallel GPU stages could theoretically exceed wall time)
|
||||
utilization = min(utilization, 100.0)
|
||||
|
||||
return GpuMetrics(
|
||||
total_gpu_seconds=total_gpu,
|
||||
mean_gpu_seconds_per_document=total_gpu / n_docs if n_docs > 0 else 0.0,
|
||||
peak_gpu_memory_mb=peak_gpu_mem,
|
||||
mean_gpu_memory_mb=mean_gpu_mem,
|
||||
gpu_utilization_percent=utilization,
|
||||
)
|
||||
|
||||
|
||||
def compute_memory_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
rss_samples_mb: list[float] | None = None,
|
||||
) -> MemoryMetrics:
|
||||
"""Compute memory consumption metrics.
|
||||
|
||||
Uses gpu_memory_mb as a proxy for working set if no explicit RSS
|
||||
samples are provided. When rss_samples_mb is given, it takes
|
||||
precedence for peak and mean calculations.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
rss_samples_mb: Optional explicit RSS memory samples in MB.
|
||||
|
||||
Returns:
|
||||
MemoryMetrics with peak and mean working set.
|
||||
"""
|
||||
if rss_samples_mb:
|
||||
return MemoryMetrics(
|
||||
peak_rss_memory_mb=max(rss_samples_mb),
|
||||
mean_working_set_mb=sum(rss_samples_mb) / len(rss_samples_mb),
|
||||
)
|
||||
|
||||
if not records:
|
||||
return MemoryMetrics(peak_rss_memory_mb=0.0, mean_working_set_mb=0.0)
|
||||
|
||||
# Use gpu_memory_mb as working set proxy
|
||||
mem_values = [r.gpu_memory_mb for r in records if r.gpu_memory_mb > 0.0]
|
||||
if not mem_values:
|
||||
return MemoryMetrics(peak_rss_memory_mb=0.0, mean_working_set_mb=0.0)
|
||||
|
||||
return MemoryMetrics(
|
||||
peak_rss_memory_mb=max(mem_values),
|
||||
mean_working_set_mb=sum(mem_values) / len(mem_values),
|
||||
)
|
||||
|
||||
|
||||
def compute_efficiency_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
) -> EfficiencyMetrics:
|
||||
"""Compute efficiency ratios including tokens/second and resource splits.
|
||||
|
||||
Fast-path vs adjudication split is determined by stage name matching.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
|
||||
Returns:
|
||||
EfficiencyMetrics with ratios and resource splits.
|
||||
"""
|
||||
if not records:
|
||||
return EfficiencyMetrics(
|
||||
tokens_per_second=0.0,
|
||||
documents_per_gpu_second=0.0,
|
||||
fast_path_cpu_seconds=0.0,
|
||||
adjudication_cpu_seconds=0.0,
|
||||
fast_path_gpu_seconds=0.0,
|
||||
adjudication_gpu_seconds=0.0,
|
||||
fast_path_fraction=0.0,
|
||||
adjudication_fraction=0.0,
|
||||
)
|
||||
|
||||
total_tokens = sum(r.total_tokens for r in records)
|
||||
total_wall = max(r.end_time for r in records) - min(r.start_time for r in records)
|
||||
total_gpu = sum(r.gpu_seconds for r in records)
|
||||
n_docs = len({r.document_id for r in records})
|
||||
|
||||
tokens_per_second = total_tokens / total_wall if total_wall > 0.0 else 0.0
|
||||
docs_per_gpu_second = n_docs / total_gpu if total_gpu > 0.0 else 0.0
|
||||
|
||||
# Resource split
|
||||
fast_cpu = 0.0
|
||||
adj_cpu = 0.0
|
||||
fast_gpu = 0.0
|
||||
adj_gpu = 0.0
|
||||
|
||||
for r in records:
|
||||
if _is_adjudication_stage(r.stage_name):
|
||||
adj_cpu += r.cpu_seconds
|
||||
adj_gpu += r.gpu_seconds
|
||||
else:
|
||||
fast_cpu += r.cpu_seconds
|
||||
fast_gpu += r.gpu_seconds
|
||||
|
||||
total_resource = fast_cpu + adj_cpu + fast_gpu + adj_gpu
|
||||
fast_total = fast_cpu + fast_gpu
|
||||
adj_total = adj_cpu + adj_gpu
|
||||
|
||||
fast_fraction = fast_total / total_resource if total_resource > 0.0 else 0.0
|
||||
adj_fraction = adj_total / total_resource if total_resource > 0.0 else 0.0
|
||||
|
||||
return EfficiencyMetrics(
|
||||
tokens_per_second=tokens_per_second,
|
||||
documents_per_gpu_second=docs_per_gpu_second,
|
||||
fast_path_cpu_seconds=fast_cpu,
|
||||
adjudication_cpu_seconds=adj_cpu,
|
||||
fast_path_gpu_seconds=fast_gpu,
|
||||
adjudication_gpu_seconds=adj_gpu,
|
||||
fast_path_fraction=fast_fraction,
|
||||
adjudication_fraction=adj_fraction,
|
||||
)
|
||||
|
||||
|
||||
def evaluate_resources(
|
||||
records: list[StageTimingRecord],
|
||||
rss_samples_mb: list[float] | None = None,
|
||||
) -> ResourceEvaluationReport:
|
||||
"""Run full resource evaluation producing a complete report.
|
||||
|
||||
Args:
|
||||
records: List of stage timing records from pipeline execution.
|
||||
rss_samples_mb: Optional explicit RSS memory samples.
|
||||
|
||||
Returns:
|
||||
ResourceEvaluationReport with all resource metrics.
|
||||
"""
|
||||
latency, per_stage_latency = compute_latency_metrics(records)
|
||||
throughput = compute_throughput_metrics(records)
|
||||
token_usage = compute_token_usage_metrics(records)
|
||||
cpu = compute_cpu_metrics(records)
|
||||
gpu = compute_gpu_metrics(records)
|
||||
memory = compute_memory_metrics(records, rss_samples_mb)
|
||||
efficiency = compute_efficiency_metrics(records)
|
||||
|
||||
doc_count = len({r.document_id for r in records}) if records else 0
|
||||
|
||||
return ResourceEvaluationReport(
|
||||
latency=latency,
|
||||
per_stage_latency=per_stage_latency,
|
||||
throughput=throughput,
|
||||
token_usage=token_usage,
|
||||
cpu=cpu,
|
||||
gpu=gpu,
|
||||
memory=memory,
|
||||
efficiency=efficiency,
|
||||
document_count=doc_count,
|
||||
)
|
||||
@@ -0,0 +1,443 @@
|
||||
"""Sentiment macro-F1, micro-F1, direction accuracy, and probability calibration metrics.
|
||||
|
||||
Implements evaluation metrics for company-specific sentiment extraction quality
|
||||
and probability calibration against a gold standard corpus. Includes Expected
|
||||
Calibration Error (ECE), Brier score, and reliability diagram data.
|
||||
|
||||
Sentiments are matched by company_entity_id between predicted and gold sets.
|
||||
|
||||
Validates: Requirements 16.3, 16.4
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Domain Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SENTIMENT_LABELS = ("positive", "negative", "neutral", "mixed")
|
||||
|
||||
|
||||
class SentimentLabel(str, Enum):
|
||||
"""Supported sentiment labels."""
|
||||
|
||||
positive = "positive"
|
||||
negative = "negative"
|
||||
neutral = "neutral"
|
||||
mixed = "mixed"
|
||||
|
||||
|
||||
class SentimentPrediction(BaseModel):
|
||||
"""A predicted or gold sentiment for a specific company entity."""
|
||||
|
||||
company_entity_id: str
|
||||
label: SentimentLabel
|
||||
positive_prob: float = Field(ge=0.0, le=1.0, default=0.0)
|
||||
negative_prob: float = Field(ge=0.0, le=1.0, default=0.0)
|
||||
neutral_prob: float = Field(ge=0.0, le=1.0, default=0.0)
|
||||
mixed_prob: float = Field(ge=0.0, le=1.0, default=0.0)
|
||||
document_id: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class LabelF1(BaseModel):
|
||||
"""Per-label precision, recall, F1."""
|
||||
|
||||
label: str
|
||||
precision: float = Field(ge=0.0, le=1.0)
|
||||
recall: float = Field(ge=0.0, le=1.0)
|
||||
f1: float = Field(ge=0.0, le=1.0)
|
||||
support_predicted: int = Field(ge=0)
|
||||
support_gold: int = Field(ge=0)
|
||||
|
||||
|
||||
class SentimentF1Result(BaseModel):
|
||||
"""Sentiment classification F1 metrics."""
|
||||
|
||||
macro_f1: float = Field(ge=0.0, le=1.0)
|
||||
micro_f1: float = Field(ge=0.0, le=1.0)
|
||||
per_label: dict[str, LabelF1]
|
||||
support: int = Field(ge=0)
|
||||
|
||||
|
||||
class DirectionAccuracyResult(BaseModel):
|
||||
"""Binary direction accuracy (positive vs negative, ignoring neutral/mixed)."""
|
||||
|
||||
accuracy: float = Field(ge=0.0, le=1.0)
|
||||
correct: int = Field(ge=0)
|
||||
total: int = Field(ge=0)
|
||||
|
||||
|
||||
class CalibrationBin(BaseModel):
|
||||
"""A single bin in the reliability diagram."""
|
||||
|
||||
bin_lower: float = Field(ge=0.0, le=1.0)
|
||||
bin_upper: float = Field(ge=0.0, le=1.0)
|
||||
mean_predicted_prob: float = Field(ge=0.0, le=1.0)
|
||||
fraction_positive: float = Field(ge=0.0, le=1.0)
|
||||
count: int = Field(ge=0)
|
||||
|
||||
|
||||
class CalibrationResult(BaseModel):
|
||||
"""Probability calibration metrics."""
|
||||
|
||||
ece: float = Field(ge=0.0, le=1.0, description="Expected Calibration Error")
|
||||
brier_score: float = Field(ge=0.0, description="Brier score (mean squared error)")
|
||||
reliability_bins: list[CalibrationBin]
|
||||
n_samples: int = Field(ge=0)
|
||||
|
||||
|
||||
class SentimentEvaluationReport(BaseModel):
|
||||
"""Complete sentiment evaluation report."""
|
||||
|
||||
f1_metrics: SentimentF1Result
|
||||
direction_accuracy: DirectionAccuracyResult
|
||||
calibration: CalibrationResult
|
||||
document_count: int = Field(ge=0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core Metric Computation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _compute_label_f1(
|
||||
predicted_labels: list[str],
|
||||
gold_labels: list[str],
|
||||
label: str,
|
||||
) -> LabelF1:
|
||||
"""Compute precision, recall, F1 for a single label (one-vs-rest)."""
|
||||
tp = 0
|
||||
fp = 0
|
||||
fn = 0
|
||||
|
||||
for pred, gold in zip(predicted_labels, gold_labels):
|
||||
if pred == label and gold == label:
|
||||
tp += 1
|
||||
elif pred == label and gold != label:
|
||||
fp += 1
|
||||
elif pred != label and gold == label:
|
||||
fn += 1
|
||||
|
||||
support_predicted = tp + fp
|
||||
support_gold = tp + fn
|
||||
|
||||
precision = tp / (tp + fp) if (tp + fp) > 0 else 1.0
|
||||
recall = tp / (tp + fn) if (tp + fn) > 0 else 1.0
|
||||
|
||||
if precision + recall > 0:
|
||||
f1 = 2 * precision * recall / (precision + recall)
|
||||
else:
|
||||
f1 = 0.0
|
||||
|
||||
return LabelF1(
|
||||
label=label,
|
||||
precision=precision,
|
||||
recall=recall,
|
||||
f1=f1,
|
||||
support_predicted=support_predicted,
|
||||
support_gold=support_gold,
|
||||
)
|
||||
|
||||
|
||||
def compute_sentiment_f1(
|
||||
predicted: list[SentimentPrediction],
|
||||
gold: list[SentimentPrediction],
|
||||
) -> SentimentF1Result:
|
||||
"""Compute macro-F1, micro-F1, and per-label F1 for sentiment classification.
|
||||
|
||||
Matches predictions to gold by company_entity_id. Only matched pairs are
|
||||
evaluated (unmatched predictions/gold are ignored).
|
||||
|
||||
Args:
|
||||
predicted: Predicted sentiment labels with probabilities.
|
||||
gold: Gold standard sentiment labels.
|
||||
|
||||
Returns:
|
||||
SentimentF1Result with macro-F1, micro-F1, and per-label breakdown.
|
||||
"""
|
||||
# Match by company_entity_id
|
||||
gold_by_id = {g.company_entity_id: g for g in gold}
|
||||
matched_pred_labels: list[str] = []
|
||||
matched_gold_labels: list[str] = []
|
||||
|
||||
for p in predicted:
|
||||
if p.company_entity_id in gold_by_id:
|
||||
matched_pred_labels.append(p.label.value)
|
||||
matched_gold_labels.append(gold_by_id[p.company_entity_id].label.value)
|
||||
|
||||
support = len(matched_pred_labels)
|
||||
|
||||
if support == 0:
|
||||
empty_per_label = {
|
||||
label: LabelF1(
|
||||
label=label, precision=1.0, recall=1.0, f1=1.0,
|
||||
support_predicted=0, support_gold=0,
|
||||
)
|
||||
for label in SENTIMENT_LABELS
|
||||
}
|
||||
return SentimentF1Result(
|
||||
macro_f1=1.0,
|
||||
micro_f1=1.0,
|
||||
per_label=empty_per_label,
|
||||
support=0,
|
||||
)
|
||||
|
||||
# Per-label F1
|
||||
per_label: dict[str, LabelF1] = {}
|
||||
for label in SENTIMENT_LABELS:
|
||||
per_label[label] = _compute_label_f1(matched_pred_labels, matched_gold_labels, label)
|
||||
|
||||
# Macro-F1: average of per-label F1 scores (only labels with support)
|
||||
active_labels = [
|
||||
label for label in SENTIMENT_LABELS
|
||||
if per_label[label].support_predicted > 0 or per_label[label].support_gold > 0
|
||||
]
|
||||
if active_labels:
|
||||
label_f1_values = [per_label[label].f1 for label in active_labels]
|
||||
macro_f1 = sum(label_f1_values) / len(label_f1_values)
|
||||
else:
|
||||
macro_f1 = 1.0
|
||||
|
||||
# Micro-F1: global TP, FP, FN across all labels
|
||||
total_tp = 0
|
||||
total_fp = 0
|
||||
total_fn = 0
|
||||
|
||||
for label in SENTIMENT_LABELS:
|
||||
for pred, gold_label in zip(matched_pred_labels, matched_gold_labels):
|
||||
if pred == label and gold_label == label:
|
||||
total_tp += 1
|
||||
elif pred == label and gold_label != label:
|
||||
total_fp += 1
|
||||
elif pred != label and gold_label == label:
|
||||
total_fn += 1
|
||||
|
||||
micro_precision = total_tp / (total_tp + total_fp) if (total_tp + total_fp) > 0 else 1.0
|
||||
micro_recall = total_tp / (total_tp + total_fn) if (total_tp + total_fn) > 0 else 1.0
|
||||
|
||||
if micro_precision + micro_recall > 0:
|
||||
micro_f1 = 2 * micro_precision * micro_recall / (micro_precision + micro_recall)
|
||||
else:
|
||||
micro_f1 = 0.0
|
||||
|
||||
return SentimentF1Result(
|
||||
macro_f1=macro_f1,
|
||||
micro_f1=micro_f1,
|
||||
per_label=per_label,
|
||||
support=support,
|
||||
)
|
||||
|
||||
|
||||
def compute_direction_accuracy(
|
||||
predicted: list[SentimentPrediction],
|
||||
gold: list[SentimentPrediction],
|
||||
) -> DirectionAccuracyResult:
|
||||
"""Compute binary direction accuracy (positive vs negative).
|
||||
|
||||
Only considers matched pairs where BOTH predicted and gold labels are
|
||||
either 'positive' or 'negative'. Neutral and mixed are ignored.
|
||||
|
||||
Args:
|
||||
predicted: Predicted sentiment labels.
|
||||
gold: Gold standard sentiment labels.
|
||||
|
||||
Returns:
|
||||
DirectionAccuracyResult with accuracy and counts.
|
||||
"""
|
||||
gold_by_id = {g.company_entity_id: g for g in gold}
|
||||
correct = 0
|
||||
total = 0
|
||||
|
||||
directional_labels = {SentimentLabel.positive, SentimentLabel.negative}
|
||||
|
||||
for p in predicted:
|
||||
if p.company_entity_id not in gold_by_id:
|
||||
continue
|
||||
g = gold_by_id[p.company_entity_id]
|
||||
# Both must be directional (positive or negative)
|
||||
if p.label in directional_labels and g.label in directional_labels:
|
||||
total += 1
|
||||
if p.label == g.label:
|
||||
correct += 1
|
||||
|
||||
accuracy = correct / total if total > 0 else 1.0
|
||||
|
||||
return DirectionAccuracyResult(
|
||||
accuracy=accuracy,
|
||||
correct=correct,
|
||||
total=total,
|
||||
)
|
||||
|
||||
|
||||
def compute_calibration(
|
||||
predicted: list[SentimentPrediction],
|
||||
gold: list[SentimentPrediction],
|
||||
n_bins: int = 10,
|
||||
) -> CalibrationResult:
|
||||
"""Compute Expected Calibration Error (ECE), Brier score, and reliability diagram.
|
||||
|
||||
For each matched pair, we evaluate how well the predicted probability for
|
||||
the true label reflects observed frequency. Uses the maximum predicted
|
||||
probability (confidence) and checks if the predicted label matches gold.
|
||||
|
||||
Args:
|
||||
predicted: Predicted sentiments with probability distributions.
|
||||
gold: Gold standard sentiments.
|
||||
n_bins: Number of bins for ECE and reliability diagram.
|
||||
|
||||
Returns:
|
||||
CalibrationResult with ECE, Brier score, and per-bin data.
|
||||
"""
|
||||
gold_by_id = {g.company_entity_id: g for g in gold}
|
||||
|
||||
# Collect (confidence, correct) pairs
|
||||
confidences: list[float] = []
|
||||
corrects: list[int] = []
|
||||
brier_terms: list[float] = []
|
||||
|
||||
for p in predicted:
|
||||
if p.company_entity_id not in gold_by_id:
|
||||
continue
|
||||
g = gold_by_id[p.company_entity_id]
|
||||
|
||||
# Confidence = probability assigned to the predicted label
|
||||
confidence = _get_label_prob(p, p.label)
|
||||
is_correct = 1 if p.label == g.label else 0
|
||||
|
||||
confidences.append(confidence)
|
||||
corrects.append(is_correct)
|
||||
|
||||
# Brier score: sum of squared errors across all label probabilities
|
||||
# For each label, the "true" probability is 1 if it matches gold, else 0
|
||||
brier_term = 0.0
|
||||
for label in SENTIMENT_LABELS:
|
||||
pred_prob = _get_label_prob(p, SentimentLabel(label))
|
||||
true_indicator = 1.0 if label == g.label.value else 0.0
|
||||
brier_term += (pred_prob - true_indicator) ** 2
|
||||
brier_terms.append(brier_term)
|
||||
|
||||
n_samples = len(confidences)
|
||||
|
||||
if n_samples == 0:
|
||||
return CalibrationResult(
|
||||
ece=0.0,
|
||||
brier_score=0.0,
|
||||
reliability_bins=[],
|
||||
n_samples=0,
|
||||
)
|
||||
|
||||
# Brier score: mean of per-sample squared error sums
|
||||
brier_score = sum(brier_terms) / n_samples
|
||||
|
||||
# ECE and reliability diagram
|
||||
bin_width = 1.0 / n_bins
|
||||
reliability_bins: list[CalibrationBin] = []
|
||||
weighted_abs_diff_sum = 0.0
|
||||
|
||||
for i in range(n_bins):
|
||||
bin_lower = i * bin_width
|
||||
bin_upper = (i + 1) * bin_width
|
||||
|
||||
# Collect samples in this bin
|
||||
bin_confidences: list[float] = []
|
||||
bin_corrects: list[int] = []
|
||||
|
||||
for conf, correct in zip(confidences, corrects):
|
||||
# Include in bin if conf is in [bin_lower, bin_upper)
|
||||
# Last bin includes the upper boundary
|
||||
if i == n_bins - 1:
|
||||
in_bin = bin_lower <= conf <= bin_upper
|
||||
else:
|
||||
in_bin = bin_lower <= conf < bin_upper
|
||||
if in_bin:
|
||||
bin_confidences.append(conf)
|
||||
bin_corrects.append(correct)
|
||||
|
||||
bin_count = len(bin_confidences)
|
||||
|
||||
if bin_count > 0:
|
||||
mean_predicted = sum(bin_confidences) / bin_count
|
||||
fraction_positive = sum(bin_corrects) / bin_count
|
||||
weighted_abs_diff_sum += bin_count * abs(mean_predicted - fraction_positive)
|
||||
else:
|
||||
mean_predicted = (bin_lower + bin_upper) / 2
|
||||
fraction_positive = 0.0
|
||||
|
||||
reliability_bins.append(
|
||||
CalibrationBin(
|
||||
bin_lower=bin_lower,
|
||||
bin_upper=bin_upper,
|
||||
mean_predicted_prob=mean_predicted,
|
||||
fraction_positive=fraction_positive,
|
||||
count=bin_count,
|
||||
)
|
||||
)
|
||||
|
||||
ece = weighted_abs_diff_sum / n_samples
|
||||
|
||||
return CalibrationResult(
|
||||
ece=ece,
|
||||
brier_score=brier_score,
|
||||
reliability_bins=reliability_bins,
|
||||
n_samples=n_samples,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def evaluate_sentiment(
|
||||
predicted: list[SentimentPrediction],
|
||||
gold: list[SentimentPrediction],
|
||||
n_bins: int = 10,
|
||||
document_count: int = 1,
|
||||
) -> SentimentEvaluationReport:
|
||||
"""Run full sentiment evaluation producing a complete report.
|
||||
|
||||
Args:
|
||||
predicted: All predicted sentiment records.
|
||||
gold: All gold standard sentiment records.
|
||||
n_bins: Number of bins for calibration metrics.
|
||||
document_count: Number of documents evaluated.
|
||||
|
||||
Returns:
|
||||
SentimentEvaluationReport with F1, direction accuracy, and calibration.
|
||||
"""
|
||||
f1_metrics = compute_sentiment_f1(predicted, gold)
|
||||
direction_accuracy = compute_direction_accuracy(predicted, gold)
|
||||
calibration = compute_calibration(predicted, gold, n_bins=n_bins)
|
||||
|
||||
return SentimentEvaluationReport(
|
||||
f1_metrics=f1_metrics,
|
||||
direction_accuracy=direction_accuracy,
|
||||
calibration=calibration,
|
||||
document_count=document_count,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_label_prob(prediction: SentimentPrediction, label: SentimentLabel) -> float:
|
||||
"""Get the predicted probability for a specific label."""
|
||||
if label == SentimentLabel.positive:
|
||||
return prediction.positive_prob
|
||||
elif label == SentimentLabel.negative:
|
||||
return prediction.negative_prob
|
||||
elif label == SentimentLabel.neutral:
|
||||
return prediction.neutral_prob
|
||||
elif label == SentimentLabel.mixed:
|
||||
return prediction.mixed_prob
|
||||
return 0.0
|
||||
Reference in New Issue
Block a user