Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
89 lines
3.2 KiB
Python
89 lines
3.2 KiB
Python
"""Verification metrics: unsupported-claim rate, evidence coverage, and per-reason breakdown.
|
|
|
|
Provides aggregated metrics over a batch of verification results to support
|
|
monitoring, alerting, and promotion gates.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
|
|
from services.intelligence_pipeline_v3.verification.models import (
|
|
RejectionReason,
|
|
VerificationReport,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VerificationMetrics:
|
|
"""Aggregated verification metrics over a collection of verification results.
|
|
|
|
Attributes:
|
|
total_checked: Total number of candidates checked across all reports.
|
|
passed_count: Candidates that passed verification.
|
|
failed_count: Candidates that failed verification.
|
|
unsupported_claim_rate: Proportion of candidates rejected as unsupported claims.
|
|
evidence_coverage_rate: Proportion of candidates with valid evidence support.
|
|
per_reason_counts: Breakdown of rejection counts by reason code.
|
|
"""
|
|
|
|
total_checked: int
|
|
passed_count: int
|
|
failed_count: int
|
|
unsupported_claim_rate: float
|
|
evidence_coverage_rate: float
|
|
per_reason_counts: dict[str, int] = field(default_factory=dict)
|
|
|
|
|
|
def compute_verification_metrics(results: list[VerificationReport]) -> VerificationMetrics:
|
|
"""Compute aggregated verification metrics from a list of verification reports.
|
|
|
|
Each VerificationReport represents the outcome of verifying one document's
|
|
candidates. This function aggregates across all documents to produce
|
|
pipeline-wide metrics suitable for dashboards and promotion gates.
|
|
|
|
Args:
|
|
results: List of VerificationReport objects from individual document verifications.
|
|
|
|
Returns:
|
|
VerificationMetrics with totals, rates, and per-reason breakdown.
|
|
"""
|
|
if not results:
|
|
return VerificationMetrics(
|
|
total_checked=0,
|
|
passed_count=0,
|
|
failed_count=0,
|
|
unsupported_claim_rate=0.0,
|
|
evidence_coverage_rate=1.0,
|
|
per_reason_counts={},
|
|
)
|
|
|
|
total_checked = 0
|
|
passed_count = 0
|
|
failed_count = 0
|
|
per_reason_counts: dict[str, int] = {}
|
|
|
|
for report in results:
|
|
total_checked += report.total_candidates
|
|
passed_count += report.verified
|
|
failed_count += report.rejected
|
|
for reason_code, count in report.rejection_breakdown.items():
|
|
per_reason_counts[reason_code] = per_reason_counts.get(reason_code, 0) + count
|
|
|
|
# Unsupported claim rate: proportion of total candidates that were rejected
|
|
# specifically for unsupported_claim reason
|
|
unsupported_count = per_reason_counts.get(RejectionReason.UNSUPPORTED_CLAIM.value, 0)
|
|
unsupported_claim_rate = unsupported_count / total_checked if total_checked > 0 else 0.0
|
|
|
|
# Evidence coverage rate: proportion of candidates that passed verification
|
|
evidence_coverage_rate = passed_count / total_checked if total_checked > 0 else 1.0
|
|
|
|
return VerificationMetrics(
|
|
total_checked=total_checked,
|
|
passed_count=passed_count,
|
|
failed_count=failed_count,
|
|
unsupported_claim_rate=unsupported_claim_rate,
|
|
evidence_coverage_rate=evidence_coverage_rate,
|
|
per_reason_counts=per_reason_counts,
|
|
)
|