feat: Intelligence Pipeline v3 — full implementation

Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
@@ -0,0 +1,94 @@
+"""Simple boilerplate detection for document chunks.
+
+Returns a 0.0-1.0 score indicating how likely a chunk is boilerplate.
+Does NOT delete chunks — only marks them for downstream filtering decisions.
+"""
+
+from __future__ import annotations
+
+import re
+
+# Common boilerplate patterns in financial documents
+_BOILERPLATE_PATTERNS: list[tuple[re.Pattern[str], float]] = [
+    # Forward-looking statements disclaimer
+    (re.compile(
+        r"forward[- ]looking\s+statements?",
+        re.IGNORECASE,
+    ), 0.4),
+    # Safe harbor language
+    (re.compile(
+        r"safe\s+harbor",
+        re.IGNORECASE,
+    ), 0.3),
+    # Copyright notices
+    (re.compile(
+        r"(?:©|\bcopyright\b)\s*\d{4}",
+        re.IGNORECASE,
+    ), 0.3),
+    # All rights reserved
+    (re.compile(
+        r"all\s+rights\s+reserved",
+        re.IGNORECASE,
+    ), 0.2),
+    # Disclaimer language
+    (re.compile(
+        r"\b(?:disclaimer|disclaims?)\b",
+        re.IGNORECASE,
+    ), 0.2),
+    # "This press release" meta-reference
+    (re.compile(
+        r"this\s+(?:press\s+release|report|document)\s+(?:contains?|includes?|may\s+contain)",
+        re.IGNORECASE,
+    ), 0.2),
+    # Not an offer/solicitation language
+    (re.compile(
+        r"(?:not|does\s+not)\s+constitute\s+(?:an?\s+)?(?:offer|solicitation|recommendation)",
+        re.IGNORECASE,
+    ), 0.3),
+    # Boilerplate risk factors intro
+    (re.compile(
+        r"(?:actual\s+results|future\s+results)\s+(?:may|could|might)\s+differ\s+materially",
+        re.IGNORECASE,
+    ), 0.3),
+    # Contact/investor relations boilerplate
+    (re.compile(
+        r"(?:investor\s+relations?|media\s+(?:contact|inquiries))\s*:",
+        re.IGNORECASE,
+    ), 0.2),
+    # Legal entity registrations
+    (re.compile(
+        r"registered\s+(?:in|under)\s+(?:the\s+)?(?:laws?\s+of|state\s+of)",
+        re.IGNORECASE,
+    ), 0.15),
+]
+
+# If the chunk is mostly short lines (like a signature block), boost score
+_SHORT_LINE_THRESHOLD = 40
+_SHORT_LINE_RATIO_THRESHOLD = 0.7
+
+
+def score_boilerplate(text: str) -> float:
+    """Score a text chunk for boilerplate content.
+
+    Returns a float between 0.0 (not boilerplate) and 1.0 (definitely boilerplate).
+    The score is the sum of matched pattern weights, capped at 1.0.
+    """
+    if not text.strip():
+        return 0.0
+
+    score = 0.0
+
+    # Check pattern matches
+    for pattern, weight in _BOILERPLATE_PATTERNS:
+        if pattern.search(text):
+            score += weight
+
+    # Check for signature-block-like structure (many short lines)
+    lines = text.split("\n")
+    if lines:
+        short_lines = sum(1 for line in lines if 0 < len(line.strip()) <= _SHORT_LINE_THRESHOLD)
+        non_empty = sum(1 for line in lines if line.strip())
+        if non_empty > 3 and short_lines / non_empty > _SHORT_LINE_RATIO_THRESHOLD:
+            score += 0.15
+
+    return min(score, 1.0)