"""Evidence coverage and unsupported-claim metrics. Computes the proportion of extracted fields that have valid evidence support, and identifies unsupported claims for audit and active learning. """ from __future__ import annotations from dataclasses import dataclass, field @dataclass(frozen=True) class CoverageMetrics: """Evidence coverage statistics for a document's extraction. Attributes: total_fields: Total number of fields requiring evidence support. supported_fields: Fields with at least one valid evidence span. coverage_rate: Proportion of fields with valid evidence (0.0 to 1.0). unsupported_claims: List of field identifiers lacking evidence. unsupported_rate: Proportion of fields without valid evidence. """ total_fields: int supported_fields: int coverage_rate: float unsupported_claims: list[str] unsupported_rate: float @dataclass class FieldEvidence: """A field that requires evidence support.""" field_id: str field_name: str evidence_ids: list[str] = field(default_factory=list) def compute_coverage( fields: list[FieldEvidence], verified_span_ids: set[str] ) -> CoverageMetrics: """Compute evidence coverage metrics for a set of extraction fields. A field is considered "supported" if it references at least one span ID that passed offset verification (i.e., is in verified_span_ids). Args: fields: List of fields with their linked evidence IDs. verified_span_ids: Set of span IDs that passed offset verification. Returns: CoverageMetrics with coverage rate and unsupported claims. """ total = len(fields) if total == 0: return CoverageMetrics( total_fields=0, supported_fields=0, coverage_rate=1.0, unsupported_claims=[], unsupported_rate=0.0, ) supported = 0 unsupported: list[str] = [] for f in fields: # A field is supported if any of its evidence IDs are in the verified set has_valid_evidence = any(eid in verified_span_ids for eid in f.evidence_ids) if has_valid_evidence: supported += 1 else: unsupported.append(f.field_id) coverage_rate = supported / total unsupported_rate = 1.0 - coverage_rate return CoverageMetrics( total_fields=total, supported_fields=supported, coverage_rate=coverage_rate, unsupported_claims=unsupported, unsupported_rate=unsupported_rate, )