feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,209 @@
|
||||
"""Compact entailment verifier scaffold for claims that require semantic validation.
|
||||
|
||||
In production, this would load a compact NLI model (e.g., deberta-v3-xsmall-mnli).
|
||||
In this implementation, uses a keyword overlap heuristic as baseline for benchmarking.
|
||||
|
||||
The entailment verifier handles claims that exact matching cannot validate — e.g.,
|
||||
"revenue grew significantly" should be entailed by "revenue increased 15% year-over-year".
|
||||
|
||||
Benchmark plan for production:
|
||||
- Evaluate DeBERTa-v3-xsmall-mnli-2way for CPU-efficient NLI
|
||||
- Target: >85% accuracy on financial claim-evidence pairs
|
||||
- Constraint: <50ms per claim on CPU (no GPU required)
|
||||
- Compare against keyword overlap baseline on the Gold_Corpus entailment subset
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Stopwords to exclude from keyword overlap calculation
|
||||
_STOPWORDS = frozenset(
|
||||
{
|
||||
"a",
|
||||
"an",
|
||||
"the",
|
||||
"is",
|
||||
"are",
|
||||
"was",
|
||||
"were",
|
||||
"be",
|
||||
"been",
|
||||
"being",
|
||||
"have",
|
||||
"has",
|
||||
"had",
|
||||
"do",
|
||||
"does",
|
||||
"did",
|
||||
"will",
|
||||
"would",
|
||||
"could",
|
||||
"should",
|
||||
"may",
|
||||
"might",
|
||||
"shall",
|
||||
"can",
|
||||
"to",
|
||||
"of",
|
||||
"in",
|
||||
"for",
|
||||
"on",
|
||||
"with",
|
||||
"at",
|
||||
"by",
|
||||
"from",
|
||||
"as",
|
||||
"into",
|
||||
"through",
|
||||
"during",
|
||||
"before",
|
||||
"after",
|
||||
"and",
|
||||
"but",
|
||||
"or",
|
||||
"nor",
|
||||
"not",
|
||||
"so",
|
||||
"yet",
|
||||
"both",
|
||||
"either",
|
||||
"neither",
|
||||
"each",
|
||||
"every",
|
||||
"all",
|
||||
"any",
|
||||
"few",
|
||||
"more",
|
||||
"most",
|
||||
"other",
|
||||
"some",
|
||||
"such",
|
||||
"no",
|
||||
"only",
|
||||
"own",
|
||||
"same",
|
||||
"than",
|
||||
"too",
|
||||
"very",
|
||||
"just",
|
||||
"that",
|
||||
"this",
|
||||
"these",
|
||||
"those",
|
||||
"it",
|
||||
"its",
|
||||
"they",
|
||||
"them",
|
||||
"their",
|
||||
"we",
|
||||
"us",
|
||||
"our",
|
||||
"he",
|
||||
"him",
|
||||
"his",
|
||||
"she",
|
||||
"her",
|
||||
}
|
||||
)
|
||||
|
||||
_WORD_RE = re.compile(r"\b[a-z0-9]+(?:[-'][a-z0-9]+)*\b")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EntailmentResult:
|
||||
"""Result of an entailment check between a claim and evidence.
|
||||
|
||||
Attributes:
|
||||
entailed: Whether the evidence supports the claim.
|
||||
confidence: Confidence in the entailment decision [0.0, 1.0].
|
||||
method: The method used for verification (exact_match, keyword_overlap, nli_model).
|
||||
model_version: Version identifier for the verification model/method used.
|
||||
"""
|
||||
|
||||
entailed: bool
|
||||
confidence: float
|
||||
method: str
|
||||
model_version: str = "keyword_overlap_v1"
|
||||
|
||||
|
||||
def _tokenize(text: str) -> set[str]:
|
||||
"""Extract lowercased non-stopword tokens from text."""
|
||||
words = set(_WORD_RE.findall(text.lower()))
|
||||
return words - _STOPWORDS
|
||||
|
||||
|
||||
class EntailmentVerifier:
|
||||
"""Verifies whether evidence entails a claim using available methods.
|
||||
|
||||
The verification strategy is:
|
||||
1. Try exact matching first (claim text appears verbatim in evidence).
|
||||
2. Fall back to keyword overlap heuristic as baseline.
|
||||
3. In production: would use a compact NLI model for higher accuracy.
|
||||
|
||||
The keyword overlap heuristic computes the proportion of content words
|
||||
in the claim that also appear in the evidence. This serves as the initial
|
||||
benchmark baseline.
|
||||
"""
|
||||
|
||||
def __init__(self, keyword_threshold: float = 0.6) -> None:
|
||||
"""Initialize the entailment verifier.
|
||||
|
||||
Args:
|
||||
keyword_threshold: Minimum keyword overlap ratio to consider
|
||||
a claim entailed (default 0.6 = 60% overlap).
|
||||
"""
|
||||
self._keyword_threshold = keyword_threshold
|
||||
|
||||
def verify_claim(self, claim: str, evidence: str) -> EntailmentResult:
|
||||
"""Verify whether evidence supports a given claim.
|
||||
|
||||
Attempts exact match first, then keyword overlap. A production deployment
|
||||
would additionally run a compact NLI model for claims the heuristic cannot
|
||||
confidently classify.
|
||||
|
||||
Args:
|
||||
claim: The claim to verify (e.g., "Apple reported record revenue").
|
||||
evidence: The evidence text to check against.
|
||||
|
||||
Returns:
|
||||
EntailmentResult with entailment decision, confidence, and method used.
|
||||
"""
|
||||
if not claim or not evidence:
|
||||
return EntailmentResult(entailed=False, confidence=0.0, method="exact_match")
|
||||
|
||||
# Method 1: Exact match — claim text appears verbatim
|
||||
if claim.lower() in evidence.lower():
|
||||
return EntailmentResult(entailed=True, confidence=1.0, method="exact_match")
|
||||
|
||||
# Method 2: Keyword overlap heuristic
|
||||
claim_tokens = _tokenize(claim)
|
||||
if not claim_tokens:
|
||||
return EntailmentResult(entailed=False, confidence=0.0, method="keyword_overlap")
|
||||
|
||||
evidence_tokens = _tokenize(evidence)
|
||||
overlap = claim_tokens & evidence_tokens
|
||||
overlap_ratio = len(overlap) / len(claim_tokens)
|
||||
|
||||
entailed = overlap_ratio >= self._keyword_threshold
|
||||
# Confidence is the overlap ratio itself (higher overlap = higher confidence)
|
||||
confidence = min(overlap_ratio, 1.0)
|
||||
|
||||
return EntailmentResult(
|
||||
entailed=entailed, confidence=confidence, method="keyword_overlap"
|
||||
)
|
||||
|
||||
def verify_claims_batch(
|
||||
self, claims: list[str], evidence: str
|
||||
) -> list[EntailmentResult]:
|
||||
"""Verify multiple claims against the same evidence text.
|
||||
|
||||
Args:
|
||||
claims: List of claims to verify.
|
||||
evidence: The evidence text to check against.
|
||||
|
||||
Returns:
|
||||
List of EntailmentResult, one per claim.
|
||||
"""
|
||||
return [self.verify_claim(claim, evidence) for claim in claims]
|
||||
Reference in New Issue
Block a user