"""Compact entailment verifier scaffold for claims that require semantic validation. In production, this would load a compact NLI model (e.g., deberta-v3-xsmall-mnli). In this implementation, uses a keyword overlap heuristic as baseline for benchmarking. The entailment verifier handles claims that exact matching cannot validate — e.g., "revenue grew significantly" should be entailed by "revenue increased 15% year-over-year". Benchmark plan for production: - Evaluate DeBERTa-v3-xsmall-mnli-2way for CPU-efficient NLI - Target: >85% accuracy on financial claim-evidence pairs - Constraint: <50ms per claim on CPU (no GPU required) - Compare against keyword overlap baseline on the Gold_Corpus entailment subset """ from __future__ import annotations import re from dataclasses import dataclass # Stopwords to exclude from keyword overlap calculation _STOPWORDS = frozenset( { "a", "an", "the", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "do", "does", "did", "will", "would", "could", "should", "may", "might", "shall", "can", "to", "of", "in", "for", "on", "with", "at", "by", "from", "as", "into", "through", "during", "before", "after", "and", "but", "or", "nor", "not", "so", "yet", "both", "either", "neither", "each", "every", "all", "any", "few", "more", "most", "other", "some", "such", "no", "only", "own", "same", "than", "too", "very", "just", "that", "this", "these", "those", "it", "its", "they", "them", "their", "we", "us", "our", "he", "him", "his", "she", "her", } ) _WORD_RE = re.compile(r"\b[a-z0-9]+(?:[-'][a-z0-9]+)*\b") @dataclass(frozen=True) class EntailmentResult: """Result of an entailment check between a claim and evidence. Attributes: entailed: Whether the evidence supports the claim. confidence: Confidence in the entailment decision [0.0, 1.0]. method: The method used for verification (exact_match, keyword_overlap, nli_model). model_version: Version identifier for the verification model/method used. """ entailed: bool confidence: float method: str model_version: str = "keyword_overlap_v1" def _tokenize(text: str) -> set[str]: """Extract lowercased non-stopword tokens from text.""" words = set(_WORD_RE.findall(text.lower())) return words - _STOPWORDS class EntailmentVerifier: """Verifies whether evidence entails a claim using available methods. The verification strategy is: 1. Try exact matching first (claim text appears verbatim in evidence). 2. Fall back to keyword overlap heuristic as baseline. 3. In production: would use a compact NLI model for higher accuracy. The keyword overlap heuristic computes the proportion of content words in the claim that also appear in the evidence. This serves as the initial benchmark baseline. """ def __init__(self, keyword_threshold: float = 0.6) -> None: """Initialize the entailment verifier. Args: keyword_threshold: Minimum keyword overlap ratio to consider a claim entailed (default 0.6 = 60% overlap). """ self._keyword_threshold = keyword_threshold def verify_claim(self, claim: str, evidence: str) -> EntailmentResult: """Verify whether evidence supports a given claim. Attempts exact match first, then keyword overlap. A production deployment would additionally run a compact NLI model for claims the heuristic cannot confidently classify. Args: claim: The claim to verify (e.g., "Apple reported record revenue"). evidence: The evidence text to check against. Returns: EntailmentResult with entailment decision, confidence, and method used. """ if not claim or not evidence: return EntailmentResult(entailed=False, confidence=0.0, method="exact_match") # Method 1: Exact match — claim text appears verbatim if claim.lower() in evidence.lower(): return EntailmentResult(entailed=True, confidence=1.0, method="exact_match") # Method 2: Keyword overlap heuristic claim_tokens = _tokenize(claim) if not claim_tokens: return EntailmentResult(entailed=False, confidence=0.0, method="keyword_overlap") evidence_tokens = _tokenize(evidence) overlap = claim_tokens & evidence_tokens overlap_ratio = len(overlap) / len(claim_tokens) entailed = overlap_ratio >= self._keyword_threshold # Confidence is the overlap ratio itself (higher overlap = higher confidence) confidence = min(overlap_ratio, 1.0) return EntailmentResult( entailed=entailed, confidence=confidence, method="keyword_overlap" ) def verify_claims_batch( self, claims: list[str], evidence: str ) -> list[EntailmentResult]: """Verify multiple claims against the same evidence text. Args: claims: List of claims to verify. evidence: The evidence text to check against. Returns: List of EntailmentResult, one per claim. """ return [self.verify_claim(claim, evidence) for claim in claims]