"""Verification metrics: unsupported-claim rate, evidence coverage, and per-reason breakdown. Provides aggregated metrics over a batch of verification results to support monitoring, alerting, and promotion gates. """ from __future__ import annotations from dataclasses import dataclass, field from services.intelligence_pipeline_v3.verification.models import ( RejectionReason, VerificationReport, ) @dataclass(frozen=True) class VerificationMetrics: """Aggregated verification metrics over a collection of verification results. Attributes: total_checked: Total number of candidates checked across all reports. passed_count: Candidates that passed verification. failed_count: Candidates that failed verification. unsupported_claim_rate: Proportion of candidates rejected as unsupported claims. evidence_coverage_rate: Proportion of candidates with valid evidence support. per_reason_counts: Breakdown of rejection counts by reason code. """ total_checked: int passed_count: int failed_count: int unsupported_claim_rate: float evidence_coverage_rate: float per_reason_counts: dict[str, int] = field(default_factory=dict) def compute_verification_metrics(results: list[VerificationReport]) -> VerificationMetrics: """Compute aggregated verification metrics from a list of verification reports. Each VerificationReport represents the outcome of verifying one document's candidates. This function aggregates across all documents to produce pipeline-wide metrics suitable for dashboards and promotion gates. Args: results: List of VerificationReport objects from individual document verifications. Returns: VerificationMetrics with totals, rates, and per-reason breakdown. """ if not results: return VerificationMetrics( total_checked=0, passed_count=0, failed_count=0, unsupported_claim_rate=0.0, evidence_coverage_rate=1.0, per_reason_counts={}, ) total_checked = 0 passed_count = 0 failed_count = 0 per_reason_counts: dict[str, int] = {} for report in results: total_checked += report.total_candidates passed_count += report.verified failed_count += report.rejected for reason_code, count in report.rejection_breakdown.items(): per_reason_counts[reason_code] = per_reason_counts.get(reason_code, 0) + count # Unsupported claim rate: proportion of total candidates that were rejected # specifically for unsupported_claim reason unsupported_count = per_reason_counts.get(RejectionReason.UNSUPPORTED_CLAIM.value, 0) unsupported_claim_rate = unsupported_count / total_checked if total_checked > 0 else 0.0 # Evidence coverage rate: proportion of candidates that passed verification evidence_coverage_rate = passed_count / total_checked if total_checked > 0 else 1.0 return VerificationMetrics( total_checked=total_checked, passed_count=passed_count, failed_count=failed_count, unsupported_claim_rate=unsupported_claim_rate, evidence_coverage_rate=evidence_coverage_rate, per_reason_counts=per_reason_counts, )