feat: Intelligence Pipeline v3 — full implementation

Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
@@ -0,0 +1,106 @@
+"""Document-type-specific chunk strategies.
+
+Each strategy defines target/max sizes, overlap, and boundary-preservation
+rules for a particular document type.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class ChunkStrategy:
+    """Configuration for how a document type should be chunked.
+
+    Sizes are in characters (approximate 4 chars/token for English text).
+    """
+
+    target_chars: int
+    max_chars: int
+    overlap_chars: int
+    preserve_boundaries: list[str] = field(default_factory=list)
+    boundary_patterns: list[re.Pattern[str]] = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        if self.target_chars <= 0:
+            raise ValueError("target_chars must be positive")
+        if self.max_chars < self.target_chars:
+            raise ValueError("max_chars must be >= target_chars")
+        if self.overlap_chars < 0:
+            raise ValueError("overlap_chars must be non-negative")
+
+
+# ~4 chars per token approximation
+# News / press release: 700-1000 tokens target, 100 tokens overlap
+ArticleStrategy = ChunkStrategy(
+    target_chars=3200,  # ~800 tokens
+    max_chars=4000,  # ~1000 tokens
+    overlap_chars=400,  # ~100 tokens
+    preserve_boundaries=["paragraph", "heading"],
+    boundary_patterns=[
+        re.compile(r"\n\n+"),  # Paragraph breaks
+        re.compile(r"\n#{1,6}\s"),  # Markdown headings
+        re.compile(r"\n[A-Z][A-Z\s]{3,}(?:\n|$)"),  # ALL-CAPS headings
+    ],
+)
+
+# Filing: 900-1300 tokens target, 150 tokens overlap
+FilingStrategy = ChunkStrategy(
+    target_chars=4400,  # ~1100 tokens
+    max_chars=5200,  # ~1300 tokens
+    overlap_chars=600,  # ~150 tokens
+    preserve_boundaries=["section", "item", "heading"],
+    boundary_patterns=[
+        re.compile(r"\n(?:Item\s+\d+[A-Z]?[\.\:])", re.IGNORECASE),  # SEC item headers
+        re.compile(r"\n(?:PART\s+[IVX]+)", re.IGNORECASE),  # Part headers
+        re.compile(r"\n#{1,6}\s"),  # Markdown headings
+        re.compile(r"\n[A-Z][A-Z\s]{3,}(?:\n|$)"),  # ALL-CAPS headings
+        re.compile(r"\n\n+"),  # Paragraph breaks
+    ],
+)
+
+# Transcript: 700-1000 tokens target, 100 tokens overlap
+TranscriptStrategy = ChunkStrategy(
+    target_chars=3200,  # ~800 tokens
+    max_chars=4000,  # ~1000 tokens
+    overlap_chars=400,  # ~100 tokens
+    preserve_boundaries=["speaker", "paragraph"],
+    boundary_patterns=[
+        # Speaker turn patterns: "John Smith:" or "OPERATOR:" or "John Smith - CEO:"
+        re.compile(r"\n(?:[A-Z][a-zA-Z\s\-\.]+(?:\s*[-–—]\s*[A-Za-z\s,]+)?)\s*:\s*"),
+        re.compile(r"\n[A-Z][A-Z\s]{2,}:\s*"),  # ALL-CAPS speaker
+        re.compile(r"\n\n+"),  # Paragraph breaks
+    ],
+)
+
+# Macro event: 500-800 tokens target, 80 tokens overlap
+MacroEventStrategy = ChunkStrategy(
+    target_chars=2400,  # ~600 tokens
+    max_chars=3200,  # ~800 tokens
+    overlap_chars=320,  # ~80 tokens
+    preserve_boundaries=["paragraph"],
+    boundary_patterns=[
+        re.compile(r"\n\n+"),  # Paragraph breaks
+        re.compile(r"\n#{1,6}\s"),  # Markdown headings
+    ],
+)
+
+# Strategy lookup by document type
+STRATEGY_MAP: dict[str, ChunkStrategy] = {
+    "article": ArticleStrategy,
+    "news": ArticleStrategy,
+    "press_release": ArticleStrategy,
+    "filing": FilingStrategy,
+    "transcript": TranscriptStrategy,
+    "macro_event": MacroEventStrategy,
+    "macro": MacroEventStrategy,
+}
+
+DEFAULT_STRATEGY = ArticleStrategy
+
+
+def get_strategy(document_type: str) -> ChunkStrategy:
+    """Return the appropriate ChunkStrategy for a document type."""
+    return STRATEGY_MAP.get(document_type.lower(), DEFAULT_STRATEGY)