Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
107 lines
3.6 KiB
Python
107 lines
3.6 KiB
Python
"""Document-type-specific chunk strategies.
|
|
|
|
Each strategy defines target/max sizes, overlap, and boundary-preservation
|
|
rules for a particular document type.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ChunkStrategy:
|
|
"""Configuration for how a document type should be chunked.
|
|
|
|
Sizes are in characters (approximate 4 chars/token for English text).
|
|
"""
|
|
|
|
target_chars: int
|
|
max_chars: int
|
|
overlap_chars: int
|
|
preserve_boundaries: list[str] = field(default_factory=list)
|
|
boundary_patterns: list[re.Pattern[str]] = field(default_factory=list)
|
|
|
|
def __post_init__(self) -> None:
|
|
if self.target_chars <= 0:
|
|
raise ValueError("target_chars must be positive")
|
|
if self.max_chars < self.target_chars:
|
|
raise ValueError("max_chars must be >= target_chars")
|
|
if self.overlap_chars < 0:
|
|
raise ValueError("overlap_chars must be non-negative")
|
|
|
|
|
|
# ~4 chars per token approximation
|
|
# News / press release: 700-1000 tokens target, 100 tokens overlap
|
|
ArticleStrategy = ChunkStrategy(
|
|
target_chars=3200, # ~800 tokens
|
|
max_chars=4000, # ~1000 tokens
|
|
overlap_chars=400, # ~100 tokens
|
|
preserve_boundaries=["paragraph", "heading"],
|
|
boundary_patterns=[
|
|
re.compile(r"\n\n+"), # Paragraph breaks
|
|
re.compile(r"\n#{1,6}\s"), # Markdown headings
|
|
re.compile(r"\n[A-Z][A-Z\s]{3,}(?:\n|$)"), # ALL-CAPS headings
|
|
],
|
|
)
|
|
|
|
# Filing: 900-1300 tokens target, 150 tokens overlap
|
|
FilingStrategy = ChunkStrategy(
|
|
target_chars=4400, # ~1100 tokens
|
|
max_chars=5200, # ~1300 tokens
|
|
overlap_chars=600, # ~150 tokens
|
|
preserve_boundaries=["section", "item", "heading"],
|
|
boundary_patterns=[
|
|
re.compile(r"\n(?:Item\s+\d+[A-Z]?[\.\:])", re.IGNORECASE), # SEC item headers
|
|
re.compile(r"\n(?:PART\s+[IVX]+)", re.IGNORECASE), # Part headers
|
|
re.compile(r"\n#{1,6}\s"), # Markdown headings
|
|
re.compile(r"\n[A-Z][A-Z\s]{3,}(?:\n|$)"), # ALL-CAPS headings
|
|
re.compile(r"\n\n+"), # Paragraph breaks
|
|
],
|
|
)
|
|
|
|
# Transcript: 700-1000 tokens target, 100 tokens overlap
|
|
TranscriptStrategy = ChunkStrategy(
|
|
target_chars=3200, # ~800 tokens
|
|
max_chars=4000, # ~1000 tokens
|
|
overlap_chars=400, # ~100 tokens
|
|
preserve_boundaries=["speaker", "paragraph"],
|
|
boundary_patterns=[
|
|
# Speaker turn patterns: "John Smith:" or "OPERATOR:" or "John Smith - CEO:"
|
|
re.compile(r"\n(?:[A-Z][a-zA-Z\s\-\.]+(?:\s*[-–—]\s*[A-Za-z\s,]+)?)\s*:\s*"),
|
|
re.compile(r"\n[A-Z][A-Z\s]{2,}:\s*"), # ALL-CAPS speaker
|
|
re.compile(r"\n\n+"), # Paragraph breaks
|
|
],
|
|
)
|
|
|
|
# Macro event: 500-800 tokens target, 80 tokens overlap
|
|
MacroEventStrategy = ChunkStrategy(
|
|
target_chars=2400, # ~600 tokens
|
|
max_chars=3200, # ~800 tokens
|
|
overlap_chars=320, # ~80 tokens
|
|
preserve_boundaries=["paragraph"],
|
|
boundary_patterns=[
|
|
re.compile(r"\n\n+"), # Paragraph breaks
|
|
re.compile(r"\n#{1,6}\s"), # Markdown headings
|
|
],
|
|
)
|
|
|
|
# Strategy lookup by document type
|
|
STRATEGY_MAP: dict[str, ChunkStrategy] = {
|
|
"article": ArticleStrategy,
|
|
"news": ArticleStrategy,
|
|
"press_release": ArticleStrategy,
|
|
"filing": FilingStrategy,
|
|
"transcript": TranscriptStrategy,
|
|
"macro_event": MacroEventStrategy,
|
|
"macro": MacroEventStrategy,
|
|
}
|
|
|
|
DEFAULT_STRATEGY = ArticleStrategy
|
|
|
|
|
|
def get_strategy(document_type: str) -> ChunkStrategy:
|
|
"""Return the appropriate ChunkStrategy for a document type."""
|
|
return STRATEGY_MAP.get(document_type.lower(), DEFAULT_STRATEGY)
|