stonks-oracle/services/intelligence_pipeline_v3/verification/entailment.py

"""Compact entailment verifier scaffold for claims that require semantic validation.

In production, this would load a compact NLI model (e.g., deberta-v3-xsmall-mnli).
In this implementation, uses a keyword overlap heuristic as baseline for benchmarking.

The entailment verifier handles claims that exact matching cannot validate — e.g.,
"revenue grew significantly" should be entailed by "revenue increased 15% year-over-year".

Benchmark plan for production:
- Evaluate DeBERTa-v3-xsmall-mnli-2way for CPU-efficient NLI
- Target: >85% accuracy on financial claim-evidence pairs
- Constraint: <50ms per claim on CPU (no GPU required)
- Compare against keyword overlap baseline on the Gold_Corpus entailment subset
"""

from __future__ import annotations

import re
from dataclasses import dataclass

# Stopwords to exclude from keyword overlap calculation
_STOPWORDS = frozenset(
    {
        "a",
        "an",
        "the",
        "is",
        "are",
        "was",
        "were",
        "be",
        "been",
        "being",
        "have",
        "has",
        "had",
        "do",
        "does",
        "did",
        "will",
        "would",
        "could",
        "should",
        "may",
        "might",
        "shall",
        "can",
        "to",
        "of",
        "in",
        "for",
        "on",
        "with",
        "at",
        "by",
        "from",
        "as",
        "into",
        "through",
        "during",
        "before",
        "after",
        "and",
        "but",
        "or",
        "nor",
        "not",
        "so",
        "yet",
        "both",
        "either",
        "neither",
        "each",
        "every",
        "all",
        "any",
        "few",
        "more",
        "most",
        "other",
        "some",
        "such",
        "no",
        "only",
        "own",
        "same",
        "than",
        "too",
        "very",
        "just",
        "that",
        "this",
        "these",
        "those",
        "it",
        "its",
        "they",
        "them",
        "their",
        "we",
        "us",
        "our",
        "he",
        "him",
        "his",
        "she",
        "her",
    }
)

_WORD_RE = re.compile(r"\b[a-z0-9]+(?:[-'][a-z0-9]+)*\b")


@dataclass(frozen=True)
class EntailmentResult:
    """Result of an entailment check between a claim and evidence.

    Attributes:
        entailed: Whether the evidence supports the claim.
        confidence: Confidence in the entailment decision [0.0, 1.0].
        method: The method used for verification (exact_match, keyword_overlap, nli_model).
        model_version: Version identifier for the verification model/method used.
    """

    entailed: bool
    confidence: float
    method: str
    model_version: str = "keyword_overlap_v1"


def _tokenize(text: str) -> set[str]:
    """Extract lowercased non-stopword tokens from text."""
    words = set(_WORD_RE.findall(text.lower()))
    return words - _STOPWORDS


class EntailmentVerifier:
    """Verifies whether evidence entails a claim using available methods.

    The verification strategy is:
    1. Try exact matching first (claim text appears verbatim in evidence).
    2. Fall back to keyword overlap heuristic as baseline.
    3. In production: would use a compact NLI model for higher accuracy.

    The keyword overlap heuristic computes the proportion of content words
    in the claim that also appear in the evidence. This serves as the initial
    benchmark baseline.
    """

    def __init__(self, keyword_threshold: float = 0.6) -> None:
        """Initialize the entailment verifier.

        Args:
            keyword_threshold: Minimum keyword overlap ratio to consider
                a claim entailed (default 0.6 = 60% overlap).
        """
        self._keyword_threshold = keyword_threshold

    def verify_claim(self, claim: str, evidence: str) -> EntailmentResult:
        """Verify whether evidence supports a given claim.

        Attempts exact match first, then keyword overlap. A production deployment
        would additionally run a compact NLI model for claims the heuristic cannot
        confidently classify.

        Args:
            claim: The claim to verify (e.g., "Apple reported record revenue").
            evidence: The evidence text to check against.

        Returns:
            EntailmentResult with entailment decision, confidence, and method used.
        """
        if not claim or not evidence:
            return EntailmentResult(entailed=False, confidence=0.0, method="exact_match")

        # Method 1: Exact match — claim text appears verbatim
        if claim.lower() in evidence.lower():
            return EntailmentResult(entailed=True, confidence=1.0, method="exact_match")

        # Method 2: Keyword overlap heuristic
        claim_tokens = _tokenize(claim)
        if not claim_tokens:
            return EntailmentResult(entailed=False, confidence=0.0, method="keyword_overlap")

        evidence_tokens = _tokenize(evidence)
        overlap = claim_tokens & evidence_tokens
        overlap_ratio = len(overlap) / len(claim_tokens)

        entailed = overlap_ratio >= self._keyword_threshold
        # Confidence is the overlap ratio itself (higher overlap = higher confidence)
        confidence = min(overlap_ratio, 1.0)

        return EntailmentResult(
            entailed=entailed, confidence=confidence, method="keyword_overlap"
        )

    def verify_claims_batch(
        self, claims: list[str], evidence: str
    ) -> list[EntailmentResult]:
        """Verify multiple claims against the same evidence text.

        Args:
            claims: List of claims to verify.
            evidence: The evidence text to check against.

        Returns:
            List of EntailmentResult, one per claim.
        """
        return [self.verify_claim(claim, evidence) for claim in claims]