feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,53 @@
|
||||
"""Evidence verification and grounding for Intelligence Pipeline v3.
|
||||
|
||||
This package provides:
|
||||
- Offset, entity association, and numeric consistency verification
|
||||
- Rejected-candidate storage with structured reason codes
|
||||
- A compact entailment verifier scaffold (keyword-overlap baseline)
|
||||
- Evidence coverage and unsupported-claim metrics
|
||||
- Aggregated verification metrics for dashboards and promotion gates
|
||||
"""
|
||||
|
||||
from services.intelligence_pipeline_v3.verification.coverage import (
|
||||
CoverageMetrics,
|
||||
FieldEvidence,
|
||||
compute_coverage,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.verification.entailment import (
|
||||
EntailmentResult,
|
||||
EntailmentVerifier,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.verification.metrics import (
|
||||
VerificationMetrics,
|
||||
compute_verification_metrics,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.verification.models import (
|
||||
AssociationVerification,
|
||||
NumericVerification,
|
||||
OffsetVerification,
|
||||
RejectedCandidate,
|
||||
RejectionReason,
|
||||
VerificationReport,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.verification.rejected_store import (
|
||||
RejectedCandidateStore,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.verification.verifier import EvidenceVerifier
|
||||
|
||||
__all__ = [
|
||||
"AssociationVerification",
|
||||
"CoverageMetrics",
|
||||
"EntailmentResult",
|
||||
"EntailmentVerifier",
|
||||
"EvidenceVerifier",
|
||||
"FieldEvidence",
|
||||
"NumericVerification",
|
||||
"OffsetVerification",
|
||||
"RejectedCandidate",
|
||||
"RejectedCandidateStore",
|
||||
"RejectionReason",
|
||||
"VerificationMetrics",
|
||||
"VerificationReport",
|
||||
"compute_coverage",
|
||||
"compute_verification_metrics",
|
||||
]
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Evidence coverage and unsupported-claim metrics.
|
||||
|
||||
Computes the proportion of extracted fields that have valid evidence support,
|
||||
and identifies unsupported claims for audit and active learning.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CoverageMetrics:
|
||||
"""Evidence coverage statistics for a document's extraction.
|
||||
|
||||
Attributes:
|
||||
total_fields: Total number of fields requiring evidence support.
|
||||
supported_fields: Fields with at least one valid evidence span.
|
||||
coverage_rate: Proportion of fields with valid evidence (0.0 to 1.0).
|
||||
unsupported_claims: List of field identifiers lacking evidence.
|
||||
unsupported_rate: Proportion of fields without valid evidence.
|
||||
"""
|
||||
|
||||
total_fields: int
|
||||
supported_fields: int
|
||||
coverage_rate: float
|
||||
unsupported_claims: list[str]
|
||||
unsupported_rate: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class FieldEvidence:
|
||||
"""A field that requires evidence support."""
|
||||
|
||||
field_id: str
|
||||
field_name: str
|
||||
evidence_ids: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def compute_coverage(
|
||||
fields: list[FieldEvidence], verified_span_ids: set[str]
|
||||
) -> CoverageMetrics:
|
||||
"""Compute evidence coverage metrics for a set of extraction fields.
|
||||
|
||||
A field is considered "supported" if it references at least one span ID
|
||||
that passed offset verification (i.e., is in verified_span_ids).
|
||||
|
||||
Args:
|
||||
fields: List of fields with their linked evidence IDs.
|
||||
verified_span_ids: Set of span IDs that passed offset verification.
|
||||
|
||||
Returns:
|
||||
CoverageMetrics with coverage rate and unsupported claims.
|
||||
"""
|
||||
total = len(fields)
|
||||
if total == 0:
|
||||
return CoverageMetrics(
|
||||
total_fields=0,
|
||||
supported_fields=0,
|
||||
coverage_rate=1.0,
|
||||
unsupported_claims=[],
|
||||
unsupported_rate=0.0,
|
||||
)
|
||||
|
||||
supported = 0
|
||||
unsupported: list[str] = []
|
||||
|
||||
for f in fields:
|
||||
# A field is supported if any of its evidence IDs are in the verified set
|
||||
has_valid_evidence = any(eid in verified_span_ids for eid in f.evidence_ids)
|
||||
if has_valid_evidence:
|
||||
supported += 1
|
||||
else:
|
||||
unsupported.append(f.field_id)
|
||||
|
||||
coverage_rate = supported / total
|
||||
unsupported_rate = 1.0 - coverage_rate
|
||||
|
||||
return CoverageMetrics(
|
||||
total_fields=total,
|
||||
supported_fields=supported,
|
||||
coverage_rate=coverage_rate,
|
||||
unsupported_claims=unsupported,
|
||||
unsupported_rate=unsupported_rate,
|
||||
)
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Compact entailment verifier scaffold for claims that require semantic validation.
|
||||
|
||||
In production, this would load a compact NLI model (e.g., deberta-v3-xsmall-mnli).
|
||||
In this implementation, uses a keyword overlap heuristic as baseline for benchmarking.
|
||||
|
||||
The entailment verifier handles claims that exact matching cannot validate — e.g.,
|
||||
"revenue grew significantly" should be entailed by "revenue increased 15% year-over-year".
|
||||
|
||||
Benchmark plan for production:
|
||||
- Evaluate DeBERTa-v3-xsmall-mnli-2way for CPU-efficient NLI
|
||||
- Target: >85% accuracy on financial claim-evidence pairs
|
||||
- Constraint: <50ms per claim on CPU (no GPU required)
|
||||
- Compare against keyword overlap baseline on the Gold_Corpus entailment subset
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Stopwords to exclude from keyword overlap calculation
|
||||
_STOPWORDS = frozenset(
|
||||
{
|
||||
"a",
|
||||
"an",
|
||||
"the",
|
||||
"is",
|
||||
"are",
|
||||
"was",
|
||||
"were",
|
||||
"be",
|
||||
"been",
|
||||
"being",
|
||||
"have",
|
||||
"has",
|
||||
"had",
|
||||
"do",
|
||||
"does",
|
||||
"did",
|
||||
"will",
|
||||
"would",
|
||||
"could",
|
||||
"should",
|
||||
"may",
|
||||
"might",
|
||||
"shall",
|
||||
"can",
|
||||
"to",
|
||||
"of",
|
||||
"in",
|
||||
"for",
|
||||
"on",
|
||||
"with",
|
||||
"at",
|
||||
"by",
|
||||
"from",
|
||||
"as",
|
||||
"into",
|
||||
"through",
|
||||
"during",
|
||||
"before",
|
||||
"after",
|
||||
"and",
|
||||
"but",
|
||||
"or",
|
||||
"nor",
|
||||
"not",
|
||||
"so",
|
||||
"yet",
|
||||
"both",
|
||||
"either",
|
||||
"neither",
|
||||
"each",
|
||||
"every",
|
||||
"all",
|
||||
"any",
|
||||
"few",
|
||||
"more",
|
||||
"most",
|
||||
"other",
|
||||
"some",
|
||||
"such",
|
||||
"no",
|
||||
"only",
|
||||
"own",
|
||||
"same",
|
||||
"than",
|
||||
"too",
|
||||
"very",
|
||||
"just",
|
||||
"that",
|
||||
"this",
|
||||
"these",
|
||||
"those",
|
||||
"it",
|
||||
"its",
|
||||
"they",
|
||||
"them",
|
||||
"their",
|
||||
"we",
|
||||
"us",
|
||||
"our",
|
||||
"he",
|
||||
"him",
|
||||
"his",
|
||||
"she",
|
||||
"her",
|
||||
}
|
||||
)
|
||||
|
||||
_WORD_RE = re.compile(r"\b[a-z0-9]+(?:[-'][a-z0-9]+)*\b")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EntailmentResult:
|
||||
"""Result of an entailment check between a claim and evidence.
|
||||
|
||||
Attributes:
|
||||
entailed: Whether the evidence supports the claim.
|
||||
confidence: Confidence in the entailment decision [0.0, 1.0].
|
||||
method: The method used for verification (exact_match, keyword_overlap, nli_model).
|
||||
model_version: Version identifier for the verification model/method used.
|
||||
"""
|
||||
|
||||
entailed: bool
|
||||
confidence: float
|
||||
method: str
|
||||
model_version: str = "keyword_overlap_v1"
|
||||
|
||||
|
||||
def _tokenize(text: str) -> set[str]:
|
||||
"""Extract lowercased non-stopword tokens from text."""
|
||||
words = set(_WORD_RE.findall(text.lower()))
|
||||
return words - _STOPWORDS
|
||||
|
||||
|
||||
class EntailmentVerifier:
|
||||
"""Verifies whether evidence entails a claim using available methods.
|
||||
|
||||
The verification strategy is:
|
||||
1. Try exact matching first (claim text appears verbatim in evidence).
|
||||
2. Fall back to keyword overlap heuristic as baseline.
|
||||
3. In production: would use a compact NLI model for higher accuracy.
|
||||
|
||||
The keyword overlap heuristic computes the proportion of content words
|
||||
in the claim that also appear in the evidence. This serves as the initial
|
||||
benchmark baseline.
|
||||
"""
|
||||
|
||||
def __init__(self, keyword_threshold: float = 0.6) -> None:
|
||||
"""Initialize the entailment verifier.
|
||||
|
||||
Args:
|
||||
keyword_threshold: Minimum keyword overlap ratio to consider
|
||||
a claim entailed (default 0.6 = 60% overlap).
|
||||
"""
|
||||
self._keyword_threshold = keyword_threshold
|
||||
|
||||
def verify_claim(self, claim: str, evidence: str) -> EntailmentResult:
|
||||
"""Verify whether evidence supports a given claim.
|
||||
|
||||
Attempts exact match first, then keyword overlap. A production deployment
|
||||
would additionally run a compact NLI model for claims the heuristic cannot
|
||||
confidently classify.
|
||||
|
||||
Args:
|
||||
claim: The claim to verify (e.g., "Apple reported record revenue").
|
||||
evidence: The evidence text to check against.
|
||||
|
||||
Returns:
|
||||
EntailmentResult with entailment decision, confidence, and method used.
|
||||
"""
|
||||
if not claim or not evidence:
|
||||
return EntailmentResult(entailed=False, confidence=0.0, method="exact_match")
|
||||
|
||||
# Method 1: Exact match — claim text appears verbatim
|
||||
if claim.lower() in evidence.lower():
|
||||
return EntailmentResult(entailed=True, confidence=1.0, method="exact_match")
|
||||
|
||||
# Method 2: Keyword overlap heuristic
|
||||
claim_tokens = _tokenize(claim)
|
||||
if not claim_tokens:
|
||||
return EntailmentResult(entailed=False, confidence=0.0, method="keyword_overlap")
|
||||
|
||||
evidence_tokens = _tokenize(evidence)
|
||||
overlap = claim_tokens & evidence_tokens
|
||||
overlap_ratio = len(overlap) / len(claim_tokens)
|
||||
|
||||
entailed = overlap_ratio >= self._keyword_threshold
|
||||
# Confidence is the overlap ratio itself (higher overlap = higher confidence)
|
||||
confidence = min(overlap_ratio, 1.0)
|
||||
|
||||
return EntailmentResult(
|
||||
entailed=entailed, confidence=confidence, method="keyword_overlap"
|
||||
)
|
||||
|
||||
def verify_claims_batch(
|
||||
self, claims: list[str], evidence: str
|
||||
) -> list[EntailmentResult]:
|
||||
"""Verify multiple claims against the same evidence text.
|
||||
|
||||
Args:
|
||||
claims: List of claims to verify.
|
||||
evidence: The evidence text to check against.
|
||||
|
||||
Returns:
|
||||
List of EntailmentResult, one per claim.
|
||||
"""
|
||||
return [self.verify_claim(claim, evidence) for claim in claims]
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Verification metrics: unsupported-claim rate, evidence coverage, and per-reason breakdown.
|
||||
|
||||
Provides aggregated metrics over a batch of verification results to support
|
||||
monitoring, alerting, and promotion gates.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from services.intelligence_pipeline_v3.verification.models import (
|
||||
RejectionReason,
|
||||
VerificationReport,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VerificationMetrics:
|
||||
"""Aggregated verification metrics over a collection of verification results.
|
||||
|
||||
Attributes:
|
||||
total_checked: Total number of candidates checked across all reports.
|
||||
passed_count: Candidates that passed verification.
|
||||
failed_count: Candidates that failed verification.
|
||||
unsupported_claim_rate: Proportion of candidates rejected as unsupported claims.
|
||||
evidence_coverage_rate: Proportion of candidates with valid evidence support.
|
||||
per_reason_counts: Breakdown of rejection counts by reason code.
|
||||
"""
|
||||
|
||||
total_checked: int
|
||||
passed_count: int
|
||||
failed_count: int
|
||||
unsupported_claim_rate: float
|
||||
evidence_coverage_rate: float
|
||||
per_reason_counts: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
|
||||
def compute_verification_metrics(results: list[VerificationReport]) -> VerificationMetrics:
|
||||
"""Compute aggregated verification metrics from a list of verification reports.
|
||||
|
||||
Each VerificationReport represents the outcome of verifying one document's
|
||||
candidates. This function aggregates across all documents to produce
|
||||
pipeline-wide metrics suitable for dashboards and promotion gates.
|
||||
|
||||
Args:
|
||||
results: List of VerificationReport objects from individual document verifications.
|
||||
|
||||
Returns:
|
||||
VerificationMetrics with totals, rates, and per-reason breakdown.
|
||||
"""
|
||||
if not results:
|
||||
return VerificationMetrics(
|
||||
total_checked=0,
|
||||
passed_count=0,
|
||||
failed_count=0,
|
||||
unsupported_claim_rate=0.0,
|
||||
evidence_coverage_rate=1.0,
|
||||
per_reason_counts={},
|
||||
)
|
||||
|
||||
total_checked = 0
|
||||
passed_count = 0
|
||||
failed_count = 0
|
||||
per_reason_counts: dict[str, int] = {}
|
||||
|
||||
for report in results:
|
||||
total_checked += report.total_candidates
|
||||
passed_count += report.verified
|
||||
failed_count += report.rejected
|
||||
for reason_code, count in report.rejection_breakdown.items():
|
||||
per_reason_counts[reason_code] = per_reason_counts.get(reason_code, 0) + count
|
||||
|
||||
# Unsupported claim rate: proportion of total candidates that were rejected
|
||||
# specifically for unsupported_claim reason
|
||||
unsupported_count = per_reason_counts.get(RejectionReason.UNSUPPORTED_CLAIM.value, 0)
|
||||
unsupported_claim_rate = unsupported_count / total_checked if total_checked > 0 else 0.0
|
||||
|
||||
# Evidence coverage rate: proportion of candidates that passed verification
|
||||
evidence_coverage_rate = passed_count / total_checked if total_checked > 0 else 1.0
|
||||
|
||||
return VerificationMetrics(
|
||||
total_checked=total_checked,
|
||||
passed_count=passed_count,
|
||||
failed_count=failed_count,
|
||||
unsupported_claim_rate=unsupported_claim_rate,
|
||||
evidence_coverage_rate=evidence_coverage_rate,
|
||||
per_reason_counts=per_reason_counts,
|
||||
)
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Data models for evidence verification results and rejected candidates.
|
||||
|
||||
Provides structured types for offset verification, entity-evidence association,
|
||||
numeric consistency checks, rejected-candidate storage, and verification reports.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class RejectionReason(str, Enum):
|
||||
"""Structured reason codes for candidate rejection during verification."""
|
||||
|
||||
INVALID_OFFSET = "invalid_offset"
|
||||
TEXT_MISMATCH = "text_mismatch"
|
||||
ENTITY_NOT_IN_EVIDENCE = "entity_not_in_evidence"
|
||||
NUMERIC_INCONSISTENCY = "numeric_inconsistency"
|
||||
UNSUPPORTED_CLAIM = "unsupported_claim"
|
||||
SCHEMA_VIOLATION = "schema_violation"
|
||||
CONFIDENCE_BELOW_THRESHOLD = "confidence_below_threshold"
|
||||
|
||||
|
||||
class OffsetVerification(BaseModel):
|
||||
"""Result of verifying a single evidence span's offsets against source text."""
|
||||
|
||||
span_id: str = Field(description="ID of the evidence span being verified.")
|
||||
valid: bool = Field(description="Whether the span text matches source at offsets.")
|
||||
reason: str | None = Field(
|
||||
default=None,
|
||||
description="Explanation when invalid (e.g., 'text mismatch at offset 42').",
|
||||
)
|
||||
|
||||
|
||||
class AssociationVerification(BaseModel):
|
||||
"""Result of verifying that an entity appears in at least one linked evidence span."""
|
||||
|
||||
entity_id: str = Field(description="ID of the entity being verified.")
|
||||
valid: bool = Field(description="Whether entity text was found in any linked span.")
|
||||
reason: str | None = Field(
|
||||
default=None,
|
||||
description="Explanation when invalid (e.g., 'entity text not in any linked span').",
|
||||
)
|
||||
|
||||
|
||||
class NumericVerification(BaseModel):
|
||||
"""Result of verifying a numeric fact against its evidence spans."""
|
||||
|
||||
fact_id: str = Field(description="ID of the numeric fact being verified.")
|
||||
valid: bool = Field(description="Whether the numeric value was found in evidence text.")
|
||||
found_value: str | None = Field(
|
||||
default=None,
|
||||
description="The value found in the evidence text, if any.",
|
||||
)
|
||||
expected_value: str = Field(description="The expected normalized value from the fact.")
|
||||
reason: str | None = Field(
|
||||
default=None,
|
||||
description="Explanation when invalid (e.g., 'value 3.14 not found in evidence').",
|
||||
)
|
||||
|
||||
|
||||
class RejectedCandidate(BaseModel):
|
||||
"""A candidate that was rejected during verification, stored for audit and learning."""
|
||||
|
||||
candidate_type: str = Field(
|
||||
description="Type of candidate: entity, fact, event, relation, sentiment."
|
||||
)
|
||||
candidate_data: dict = Field(
|
||||
description="Serialized candidate data for audit trail."
|
||||
)
|
||||
rejection_reason: RejectionReason = Field(
|
||||
description="Structured reason code for rejection."
|
||||
)
|
||||
stage: str = Field(
|
||||
description="Pipeline stage where rejection occurred (e.g., 'offset_verification')."
|
||||
)
|
||||
timestamp: datetime = Field(
|
||||
default_factory=lambda: datetime.now(tz=timezone.utc),
|
||||
description="When the rejection was recorded.",
|
||||
)
|
||||
|
||||
|
||||
class VerificationReport(BaseModel):
|
||||
"""Summary report from a full verification pass over extraction candidates."""
|
||||
|
||||
total_candidates: int = Field(ge=0, description="Total candidates evaluated.")
|
||||
verified: int = Field(ge=0, description="Candidates that passed verification.")
|
||||
rejected: int = Field(ge=0, description="Candidates that failed verification.")
|
||||
coverage_rate: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Proportion of candidates with valid evidence support.",
|
||||
)
|
||||
rejection_breakdown: dict[str, int] = Field(
|
||||
default_factory=dict,
|
||||
description="Count of rejections per RejectionReason code.",
|
||||
)
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Rejected candidate storage for audit, learning, and debugging.
|
||||
|
||||
Stores candidates rejected during evidence verification with structured reason codes.
|
||||
Currently in-memory; designed for future persistence to v3_rejected_candidates table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
from services.intelligence_pipeline_v3.verification.models import (
|
||||
RejectedCandidate,
|
||||
RejectionReason,
|
||||
)
|
||||
|
||||
|
||||
class RejectedCandidateStore:
|
||||
"""In-memory store for rejected candidates, queryable by pipeline run and reason.
|
||||
|
||||
Designed to be replaced with a database-backed implementation once
|
||||
the v3_rejected_candidates table is deployed. The interface is stable.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._by_run: dict[str, list[RejectedCandidate]] = defaultdict(list)
|
||||
self._by_reason: dict[RejectionReason, list[RejectedCandidate]] = defaultdict(list)
|
||||
self._all: list[RejectedCandidate] = []
|
||||
|
||||
def store(self, rejected: RejectedCandidate, run_id: str = "default") -> None:
|
||||
"""Store a rejected candidate.
|
||||
|
||||
Args:
|
||||
rejected: The rejected candidate to store.
|
||||
run_id: Pipeline run identifier for grouping.
|
||||
"""
|
||||
self._all.append(rejected)
|
||||
self._by_run[run_id].append(rejected)
|
||||
self._by_reason[rejected.rejection_reason].append(rejected)
|
||||
|
||||
def store_batch(self, rejected_list: list[RejectedCandidate], run_id: str = "default") -> None:
|
||||
"""Store multiple rejected candidates in one call.
|
||||
|
||||
Args:
|
||||
rejected_list: List of rejected candidates to store.
|
||||
run_id: Pipeline run identifier for grouping.
|
||||
"""
|
||||
for r in rejected_list:
|
||||
self.store(r, run_id)
|
||||
|
||||
def get_by_pipeline_run(self, run_id: str) -> list[RejectedCandidate]:
|
||||
"""Retrieve all rejected candidates for a given pipeline run.
|
||||
|
||||
Args:
|
||||
run_id: Pipeline run identifier.
|
||||
|
||||
Returns:
|
||||
List of rejected candidates for that run (empty if none).
|
||||
"""
|
||||
return list(self._by_run.get(run_id, []))
|
||||
|
||||
def get_by_reason(self, reason: RejectionReason) -> list[RejectedCandidate]:
|
||||
"""Retrieve all rejected candidates with a specific rejection reason.
|
||||
|
||||
Args:
|
||||
reason: The rejection reason code to filter by.
|
||||
|
||||
Returns:
|
||||
List of rejected candidates with that reason (empty if none).
|
||||
"""
|
||||
return list(self._by_reason.get(reason, []))
|
||||
|
||||
def get_all(self) -> list[RejectedCandidate]:
|
||||
"""Retrieve all stored rejected candidates.
|
||||
|
||||
Returns:
|
||||
List of all rejected candidates.
|
||||
"""
|
||||
return list(self._all)
|
||||
|
||||
def count(self) -> int:
|
||||
"""Total number of rejected candidates stored."""
|
||||
return len(self._all)
|
||||
|
||||
def count_by_reason(self) -> dict[str, int]:
|
||||
"""Count of rejected candidates grouped by reason code.
|
||||
|
||||
Returns:
|
||||
Mapping from reason code string to count.
|
||||
"""
|
||||
return {reason.value: len(items) for reason, items in self._by_reason.items()}
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Remove all stored rejected candidates."""
|
||||
self._by_run.clear()
|
||||
self._by_reason.clear()
|
||||
self._all.clear()
|
||||
@@ -0,0 +1,397 @@
|
||||
"""Core evidence verifier for Intelligence Pipeline v3.
|
||||
|
||||
Validates offsets, entity-evidence associations, and numeric consistency
|
||||
for extracted candidates against source text and evidence spans.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from services.intelligence_pipeline_v3.verification.models import (
|
||||
AssociationVerification,
|
||||
NumericVerification,
|
||||
OffsetVerification,
|
||||
RejectedCandidate,
|
||||
RejectionReason,
|
||||
VerificationReport,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvidenceSpan:
|
||||
"""Minimal evidence span representation for verification."""
|
||||
|
||||
id: str
|
||||
start_char: int
|
||||
end_char: int
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Entity:
|
||||
"""Minimal entity representation for verification."""
|
||||
|
||||
id: str
|
||||
literal_text: str
|
||||
evidence_ids: list[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class NumericFact:
|
||||
"""Minimal numeric fact representation for verification."""
|
||||
|
||||
id: str
|
||||
literal_value: str
|
||||
normalized_value: float | None
|
||||
evidence_ids: list[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Candidate:
|
||||
"""Generic candidate for full verification."""
|
||||
|
||||
candidate_type: str
|
||||
candidate_id: str
|
||||
candidate_data: dict = field(default_factory=dict)
|
||||
evidence_ids: list[str] = field(default_factory=list)
|
||||
literal_text: str | None = None
|
||||
normalized_value: float | None = None
|
||||
|
||||
|
||||
# Regex for extracting numbers from text (integers, decimals, negative)
|
||||
_NUMBER_RE = re.compile(r"-?\d[\d,]*\.?\d*")
|
||||
|
||||
|
||||
def _extract_numbers(text: str) -> list[float]:
|
||||
"""Extract all numeric values from a string."""
|
||||
results: list[float] = []
|
||||
for match in _NUMBER_RE.finditer(text):
|
||||
try:
|
||||
# Remove commas before parsing
|
||||
cleaned = match.group().replace(",", "")
|
||||
results.append(float(cleaned))
|
||||
except ValueError:
|
||||
continue
|
||||
return results
|
||||
|
||||
|
||||
class EvidenceVerifier:
|
||||
"""Verifies evidence spans, entity associations, and numeric consistency.
|
||||
|
||||
Implements requirement 8 (Evidence Verification and Grounding):
|
||||
- Validates span offsets match source text
|
||||
- Checks entity text appears in linked evidence spans
|
||||
- Verifies numeric values can be found in evidence text
|
||||
- Produces a full verification report with rejected candidates
|
||||
"""
|
||||
|
||||
def __init__(self, numeric_tolerance: float = 0.01) -> None:
|
||||
"""Initialize the verifier.
|
||||
|
||||
Args:
|
||||
numeric_tolerance: Relative tolerance for numeric comparison (default 1%).
|
||||
"""
|
||||
self._numeric_tolerance = numeric_tolerance
|
||||
self._rejected: list[RejectedCandidate] = []
|
||||
|
||||
@property
|
||||
def rejected_candidates(self) -> list[RejectedCandidate]:
|
||||
"""All rejected candidates accumulated during verification."""
|
||||
return list(self._rejected)
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Clear accumulated rejected candidates."""
|
||||
self._rejected = []
|
||||
|
||||
def verify_offsets(
|
||||
self, spans: list[EvidenceSpan], source_text: str
|
||||
) -> list[OffsetVerification]:
|
||||
"""Check that each span's text matches the source at declared offsets.
|
||||
|
||||
Args:
|
||||
spans: Evidence spans with start_char, end_char, and text.
|
||||
source_text: The full original document text.
|
||||
|
||||
Returns:
|
||||
A list of OffsetVerification results, one per span.
|
||||
"""
|
||||
results: list[OffsetVerification] = []
|
||||
for span in spans:
|
||||
# Bounds check
|
||||
if span.start_char < 0 or span.end_char > len(source_text):
|
||||
results.append(
|
||||
OffsetVerification(
|
||||
span_id=span.id,
|
||||
valid=False,
|
||||
reason=(
|
||||
f"Offset out of bounds: start={span.start_char}, "
|
||||
f"end={span.end_char}, source_length={len(source_text)}"
|
||||
),
|
||||
)
|
||||
)
|
||||
self._reject_span(span, RejectionReason.INVALID_OFFSET)
|
||||
continue
|
||||
|
||||
if span.end_char <= span.start_char:
|
||||
results.append(
|
||||
OffsetVerification(
|
||||
span_id=span.id,
|
||||
valid=False,
|
||||
reason=f"Invalid range: end_char ({span.end_char}) <= start_char ({span.start_char})",
|
||||
)
|
||||
)
|
||||
self._reject_span(span, RejectionReason.INVALID_OFFSET)
|
||||
continue
|
||||
|
||||
# Extract text at offsets and compare
|
||||
actual_text = source_text[span.start_char : span.end_char]
|
||||
if actual_text == span.text:
|
||||
results.append(OffsetVerification(span_id=span.id, valid=True))
|
||||
else:
|
||||
results.append(
|
||||
OffsetVerification(
|
||||
span_id=span.id,
|
||||
valid=False,
|
||||
reason=(
|
||||
f"Text mismatch at [{span.start_char}:{span.end_char}]: "
|
||||
f"expected {span.text!r}, found {actual_text!r}"
|
||||
),
|
||||
)
|
||||
)
|
||||
self._reject_span(span, RejectionReason.TEXT_MISMATCH)
|
||||
|
||||
return results
|
||||
|
||||
def verify_entity_association(
|
||||
self, entity: Entity, evidence_spans: list[EvidenceSpan]
|
||||
) -> AssociationVerification:
|
||||
"""Check that entity text appears in at least one linked evidence span.
|
||||
|
||||
The check is case-insensitive to handle variation in capitalization.
|
||||
|
||||
Args:
|
||||
entity: Entity with literal_text and evidence_ids.
|
||||
evidence_spans: All available evidence spans.
|
||||
|
||||
Returns:
|
||||
AssociationVerification indicating whether the entity is supported.
|
||||
"""
|
||||
# Build a map of span_id -> span for quick lookup
|
||||
span_map = {s.id: s for s in evidence_spans}
|
||||
entity_lower = entity.literal_text.lower()
|
||||
|
||||
for eid in entity.evidence_ids:
|
||||
span = span_map.get(eid)
|
||||
if span and entity_lower in span.text.lower():
|
||||
return AssociationVerification(entity_id=entity.id, valid=True)
|
||||
|
||||
reason = (
|
||||
f"Entity text {entity.literal_text!r} not found in any linked evidence span "
|
||||
f"(checked {len(entity.evidence_ids)} spans)"
|
||||
)
|
||||
self._rejected.append(
|
||||
RejectedCandidate(
|
||||
candidate_type="entity",
|
||||
candidate_data={
|
||||
"entity_id": entity.id,
|
||||
"literal_text": entity.literal_text,
|
||||
"evidence_ids": entity.evidence_ids,
|
||||
},
|
||||
rejection_reason=RejectionReason.ENTITY_NOT_IN_EVIDENCE,
|
||||
stage="entity_association_verification",
|
||||
timestamp=datetime.now(tz=timezone.utc),
|
||||
)
|
||||
)
|
||||
return AssociationVerification(
|
||||
entity_id=entity.id, valid=False, reason=reason
|
||||
)
|
||||
|
||||
def verify_numeric_consistency(
|
||||
self, fact: NumericFact, evidence_spans: list[EvidenceSpan]
|
||||
) -> NumericVerification:
|
||||
"""Check that a numeric fact's value can be found in its evidence text.
|
||||
|
||||
Looks for the literal value or the normalized value within the text of
|
||||
linked evidence spans. Uses tolerance-aware comparison for normalized values.
|
||||
|
||||
Args:
|
||||
fact: Numeric fact with literal_value, normalized_value, and evidence_ids.
|
||||
evidence_spans: All available evidence spans.
|
||||
|
||||
Returns:
|
||||
NumericVerification indicating whether the value was found.
|
||||
"""
|
||||
span_map = {s.id: s for s in evidence_spans}
|
||||
expected_str = fact.literal_value
|
||||
|
||||
# First: check if literal value string appears in any linked span
|
||||
for eid in fact.evidence_ids:
|
||||
span = span_map.get(eid)
|
||||
if span and fact.literal_value in span.text:
|
||||
return NumericVerification(
|
||||
fact_id=fact.id,
|
||||
valid=True,
|
||||
found_value=fact.literal_value,
|
||||
expected_value=expected_str,
|
||||
)
|
||||
|
||||
# Second: if we have a normalized value, look for numeric matches
|
||||
if fact.normalized_value is not None:
|
||||
for eid in fact.evidence_ids:
|
||||
span = span_map.get(eid)
|
||||
if not span:
|
||||
continue
|
||||
numbers = _extract_numbers(span.text)
|
||||
for num in numbers:
|
||||
if self._values_match(num, fact.normalized_value):
|
||||
return NumericVerification(
|
||||
fact_id=fact.id,
|
||||
valid=True,
|
||||
found_value=str(num),
|
||||
expected_value=expected_str,
|
||||
)
|
||||
|
||||
# Rejection
|
||||
reason = (
|
||||
f"Value {expected_str!r} (normalized={fact.normalized_value}) "
|
||||
f"not found in evidence spans"
|
||||
)
|
||||
self._rejected.append(
|
||||
RejectedCandidate(
|
||||
candidate_type="fact",
|
||||
candidate_data={
|
||||
"fact_id": fact.id,
|
||||
"literal_value": fact.literal_value,
|
||||
"normalized_value": fact.normalized_value,
|
||||
"evidence_ids": fact.evidence_ids,
|
||||
},
|
||||
rejection_reason=RejectionReason.NUMERIC_INCONSISTENCY,
|
||||
stage="numeric_consistency_verification",
|
||||
timestamp=datetime.now(tz=timezone.utc),
|
||||
)
|
||||
)
|
||||
return NumericVerification(
|
||||
fact_id=fact.id,
|
||||
valid=False,
|
||||
found_value=None,
|
||||
expected_value=expected_str,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
def verify_all(
|
||||
self,
|
||||
candidates: list[Candidate],
|
||||
evidence_spans: list[EvidenceSpan],
|
||||
source_text: str,
|
||||
) -> VerificationReport:
|
||||
"""Run full verification across all candidates.
|
||||
|
||||
Performs offset verification on spans, then entity association and
|
||||
numeric consistency for each candidate as appropriate.
|
||||
|
||||
Args:
|
||||
candidates: All extraction candidates to verify.
|
||||
evidence_spans: All evidence spans in the document.
|
||||
source_text: Full original document text.
|
||||
|
||||
Returns:
|
||||
A VerificationReport summarizing the verification results.
|
||||
"""
|
||||
self.reset()
|
||||
|
||||
# Step 1: Verify all span offsets
|
||||
offset_results = self.verify_offsets(evidence_spans, source_text)
|
||||
valid_span_ids = {r.span_id for r in offset_results if r.valid}
|
||||
|
||||
# Step 2: Filter spans to only valid ones for downstream checks
|
||||
valid_spans = [s for s in evidence_spans if s.id in valid_span_ids]
|
||||
|
||||
total = len(candidates)
|
||||
verified = 0
|
||||
|
||||
for candidate in candidates:
|
||||
# Filter candidate's evidence_ids to only valid spans
|
||||
linked_valid_spans = [
|
||||
s for s in valid_spans if s.id in set(candidate.evidence_ids)
|
||||
]
|
||||
|
||||
if not linked_valid_spans and candidate.evidence_ids:
|
||||
# All linked spans were invalid
|
||||
self._rejected.append(
|
||||
RejectedCandidate(
|
||||
candidate_type=candidate.candidate_type,
|
||||
candidate_data=candidate.candidate_data,
|
||||
rejection_reason=RejectionReason.INVALID_OFFSET,
|
||||
stage="full_verification",
|
||||
timestamp=datetime.now(tz=timezone.utc),
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
# Entity association check
|
||||
if candidate.literal_text is not None:
|
||||
entity = Entity(
|
||||
id=candidate.candidate_id,
|
||||
literal_text=candidate.literal_text,
|
||||
evidence_ids=candidate.evidence_ids,
|
||||
)
|
||||
assoc = self.verify_entity_association(entity, valid_spans)
|
||||
if not assoc.valid:
|
||||
continue
|
||||
|
||||
# Numeric consistency check
|
||||
if candidate.normalized_value is not None:
|
||||
fact = NumericFact(
|
||||
id=candidate.candidate_id,
|
||||
literal_value=candidate.literal_text or str(candidate.normalized_value),
|
||||
normalized_value=candidate.normalized_value,
|
||||
evidence_ids=candidate.evidence_ids,
|
||||
)
|
||||
num_check = self.verify_numeric_consistency(fact, valid_spans)
|
||||
if not num_check.valid:
|
||||
continue
|
||||
|
||||
verified += 1
|
||||
|
||||
rejected = total - verified
|
||||
coverage_rate = verified / total if total > 0 else 1.0
|
||||
|
||||
# Build rejection breakdown
|
||||
breakdown: dict[str, int] = {}
|
||||
for rc in self._rejected:
|
||||
key = rc.rejection_reason.value
|
||||
breakdown[key] = breakdown.get(key, 0) + 1
|
||||
|
||||
return VerificationReport(
|
||||
total_candidates=total,
|
||||
verified=verified,
|
||||
rejected=rejected,
|
||||
coverage_rate=coverage_rate,
|
||||
rejection_breakdown=breakdown,
|
||||
)
|
||||
|
||||
def _values_match(self, found: float, expected: float) -> bool:
|
||||
"""Compare two numeric values with tolerance."""
|
||||
if expected == 0:
|
||||
return abs(found) < self._numeric_tolerance
|
||||
return abs(found - expected) / abs(expected) <= self._numeric_tolerance
|
||||
|
||||
def _reject_span(self, span: EvidenceSpan, reason: RejectionReason) -> None:
|
||||
"""Record a span rejection."""
|
||||
self._rejected.append(
|
||||
RejectedCandidate(
|
||||
candidate_type="evidence_span",
|
||||
candidate_data={
|
||||
"span_id": span.id,
|
||||
"start_char": span.start_char,
|
||||
"end_char": span.end_char,
|
||||
"text": span.text[:100], # Truncate for storage
|
||||
},
|
||||
rejection_reason=reason,
|
||||
stage="offset_verification",
|
||||
timestamp=datetime.now(tz=timezone.utc),
|
||||
)
|
||||
)
|
||||
Reference in New Issue
Block a user