feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,352 @@
|
||||
"""Sentence-aware document segmenter.
|
||||
|
||||
Splits documents into chunks that respect sentence boundaries, preserve
|
||||
source character offsets, and use document-type-specific strategies.
|
||||
|
||||
Key invariant: chunk.text == source_text[chunk.start_char:chunk.end_char]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from services.intelligence_pipeline_v3.segmenter.boilerplate import score_boilerplate
|
||||
from services.intelligence_pipeline_v3.segmenter.models import DocumentChunk
|
||||
from services.intelligence_pipeline_v3.segmenter.strategies import (
|
||||
ChunkStrategy,
|
||||
get_strategy,
|
||||
)
|
||||
|
||||
# Sentence-ending patterns: period/question/exclamation followed by space or newline
|
||||
_SENTENCE_END = re.compile(r"(?<=[.!?])\s+")
|
||||
|
||||
# Speaker turn pattern for transcripts
|
||||
_SPEAKER_PATTERN = re.compile(
|
||||
r"^([A-Z][a-zA-Z\s\-\.]+(?:\s*[-–—]\s*[A-Za-z\s,]+)?)\s*:\s*",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# Filing section headers
|
||||
_FILING_SECTION_PATTERN = re.compile(
|
||||
r"^((?:Item\s+\d+[A-Z]?[\.\:]|PART\s+[IVX]+)[^\n]*)",
|
||||
re.IGNORECASE | re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
class Segmenter:
|
||||
"""Sentence-aware document segmenter with document-type-specific strategies.
|
||||
|
||||
Does NOT truncate documents — processes full text regardless of length.
|
||||
Each chunk preserves exact character offsets into the source document.
|
||||
"""
|
||||
|
||||
def segment(
|
||||
self,
|
||||
text: str,
|
||||
document_type: str,
|
||||
document_id: str = "",
|
||||
) -> list[DocumentChunk]:
|
||||
"""Segment a document into chunks.
|
||||
|
||||
Args:
|
||||
text: Full document text (no truncation applied).
|
||||
document_type: Type of document for strategy selection.
|
||||
document_id: Identifier for deterministic chunk IDs.
|
||||
|
||||
Returns:
|
||||
List of DocumentChunk with valid offsets and checksums.
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
|
||||
strategy = get_strategy(document_type)
|
||||
|
||||
# Get structural boundaries based on document type
|
||||
boundaries = self._find_boundaries(text, strategy, document_type)
|
||||
|
||||
# Build chunks respecting sentence boundaries and strategy limits
|
||||
chunks = self._build_chunks(
|
||||
text=text,
|
||||
boundaries=boundaries,
|
||||
strategy=strategy,
|
||||
document_type=document_type,
|
||||
document_id=document_id,
|
||||
)
|
||||
|
||||
# Apply overlap between adjacent chunks
|
||||
chunks = self._apply_overlap(chunks, text, strategy, document_id, document_type)
|
||||
|
||||
# Score boilerplate
|
||||
for chunk in chunks:
|
||||
chunk.boilerplate_score = score_boilerplate(chunk.text)
|
||||
|
||||
# Apply section path and speaker metadata
|
||||
self._apply_metadata(chunks, text, document_type)
|
||||
|
||||
return chunks
|
||||
|
||||
def _find_boundaries(
|
||||
self,
|
||||
text: str,
|
||||
strategy: ChunkStrategy,
|
||||
document_type: str,
|
||||
) -> list[int]:
|
||||
"""Find structural boundary positions in the text.
|
||||
|
||||
Returns sorted list of character offsets where structural breaks occur.
|
||||
"""
|
||||
boundaries: set[int] = set()
|
||||
|
||||
for pattern in strategy.boundary_patterns:
|
||||
for match in pattern.finditer(text):
|
||||
boundaries.add(match.start())
|
||||
|
||||
return sorted(boundaries)
|
||||
|
||||
def _find_sentence_boundaries(self, text: str, start: int, end: int) -> list[int]:
|
||||
"""Find sentence-ending positions within a text range.
|
||||
|
||||
Returns character offsets (relative to full document) after sentence-ending punctuation.
|
||||
"""
|
||||
segment = text[start:end]
|
||||
positions: list[int] = []
|
||||
for match in _SENTENCE_END.finditer(segment):
|
||||
positions.append(start + match.start())
|
||||
return positions
|
||||
|
||||
def _build_chunks(
|
||||
self,
|
||||
text: str,
|
||||
boundaries: list[int],
|
||||
strategy: ChunkStrategy,
|
||||
document_type: str,
|
||||
document_id: str,
|
||||
) -> list[DocumentChunk]:
|
||||
"""Build initial chunks from text using boundaries and sentence awareness."""
|
||||
chunks: list[DocumentChunk] = []
|
||||
text_len = len(text)
|
||||
pos = 0
|
||||
|
||||
while pos < text_len:
|
||||
# Determine the ideal end position
|
||||
ideal_end = min(pos + strategy.target_chars, text_len)
|
||||
max_end = min(pos + strategy.max_chars, text_len)
|
||||
|
||||
if ideal_end >= text_len:
|
||||
# Last chunk — take everything remaining
|
||||
chunk_end = text_len
|
||||
else:
|
||||
# Try to break at a structural boundary between ideal and max
|
||||
chunk_end = self._find_best_break(
|
||||
text, pos, ideal_end, max_end, boundaries, strategy
|
||||
)
|
||||
|
||||
chunk_text = text[pos:chunk_end]
|
||||
|
||||
# Skip empty chunks (shouldn't happen, but defensive)
|
||||
if not chunk_text.strip():
|
||||
pos = chunk_end
|
||||
continue
|
||||
|
||||
chunk = DocumentChunk(
|
||||
chunk_id=f"{document_id}:{pos}",
|
||||
document_id=document_id,
|
||||
document_type=document_type,
|
||||
section_path=[],
|
||||
speaker=None,
|
||||
start_char=pos,
|
||||
end_char=chunk_end,
|
||||
text=chunk_text,
|
||||
overlap_left=0,
|
||||
overlap_right=0,
|
||||
boilerplate_score=0.0,
|
||||
)
|
||||
chunks.append(chunk)
|
||||
pos = chunk_end
|
||||
|
||||
return chunks
|
||||
|
||||
def _find_best_break(
|
||||
self,
|
||||
text: str,
|
||||
start: int,
|
||||
ideal_end: int,
|
||||
max_end: int,
|
||||
boundaries: list[int],
|
||||
strategy: ChunkStrategy,
|
||||
) -> int:
|
||||
"""Find the best break point between ideal_end and max_end.
|
||||
|
||||
Priority:
|
||||
1. Structural boundary nearest to ideal_end (within target..max range)
|
||||
2. Sentence boundary nearest to ideal_end
|
||||
3. Whitespace nearest to ideal_end
|
||||
4. Hard cut at ideal_end
|
||||
"""
|
||||
# Look for structural boundaries in the window [ideal_end - target_chars/4, max_end]
|
||||
search_start = max(start, ideal_end - strategy.target_chars // 4)
|
||||
best_structural = None
|
||||
for b in boundaries:
|
||||
if search_start <= b <= max_end and b > start:
|
||||
if best_structural is None or abs(b - ideal_end) < abs(best_structural - ideal_end):
|
||||
best_structural = b
|
||||
if best_structural is not None:
|
||||
return best_structural
|
||||
|
||||
# Look for sentence boundaries near ideal_end
|
||||
sentence_breaks = self._find_sentence_boundaries(text, search_start, max_end)
|
||||
if sentence_breaks:
|
||||
# Pick the one closest to ideal_end
|
||||
best_sentence = min(sentence_breaks, key=lambda s: abs(s - ideal_end))
|
||||
# Use position after the sentence-ending whitespace
|
||||
after = best_sentence
|
||||
while after < max_end and text[after] in " \t\n\r":
|
||||
after += 1
|
||||
return after
|
||||
|
||||
# Fall back to whitespace break
|
||||
search_region = text[ideal_end:max_end]
|
||||
ws_match = re.search(r"\s+", search_region)
|
||||
if ws_match:
|
||||
return ideal_end + ws_match.end()
|
||||
|
||||
# Hard cut
|
||||
return ideal_end
|
||||
|
||||
def _apply_overlap(
|
||||
self,
|
||||
chunks: list[DocumentChunk],
|
||||
text: str,
|
||||
strategy: ChunkStrategy,
|
||||
document_id: str,
|
||||
document_type: str,
|
||||
) -> list[DocumentChunk]:
|
||||
"""Apply overlap between adjacent chunks by extending start/end.
|
||||
|
||||
Overlap is achieved by moving each chunk's start_char backward
|
||||
to include trailing content from the previous chunk.
|
||||
"""
|
||||
if len(chunks) <= 1 or strategy.overlap_chars == 0:
|
||||
return chunks
|
||||
|
||||
result: list[DocumentChunk] = []
|
||||
for i, chunk in enumerate(chunks):
|
||||
new_start = chunk.start_char
|
||||
new_end = chunk.end_char
|
||||
overlap_left = 0
|
||||
overlap_right = 0
|
||||
|
||||
if i > 0:
|
||||
# Extend start backward for overlap
|
||||
overlap_target = min(strategy.overlap_chars, chunk.start_char)
|
||||
new_start = max(0, chunk.start_char - overlap_target)
|
||||
|
||||
# Snap to sentence boundary if possible
|
||||
if new_start < chunk.start_char:
|
||||
region = text[new_start:chunk.start_char]
|
||||
sentence_matches = list(_SENTENCE_END.finditer(region))
|
||||
if sentence_matches:
|
||||
# Use the latest sentence break in the overlap region
|
||||
last_match = sentence_matches[-1]
|
||||
candidate = new_start + last_match.end()
|
||||
if candidate < chunk.start_char:
|
||||
new_start = candidate
|
||||
|
||||
overlap_left = chunk.start_char - new_start
|
||||
|
||||
if i < len(chunks) - 1:
|
||||
# Calculate how much the next chunk will overlap into this one
|
||||
next_chunk = chunks[i + 1]
|
||||
overlap_target = min(strategy.overlap_chars, len(text) - next_chunk.start_char)
|
||||
potential_overlap_start = max(0, next_chunk.start_char - overlap_target)
|
||||
|
||||
# The overlap_right for this chunk = how much the next chunk's
|
||||
# overlap will include from this chunk's content
|
||||
overlap_right = chunk.end_char - max(potential_overlap_start, chunk.start_char)
|
||||
overlap_right = max(0, overlap_right)
|
||||
|
||||
new_text = text[new_start:new_end]
|
||||
if not new_text.strip():
|
||||
result.append(chunk)
|
||||
continue
|
||||
|
||||
result.append(DocumentChunk(
|
||||
chunk_id=f"{document_id}:{new_start}",
|
||||
document_id=document_id,
|
||||
document_type=document_type,
|
||||
section_path=chunk.section_path,
|
||||
speaker=chunk.speaker,
|
||||
start_char=new_start,
|
||||
end_char=new_end,
|
||||
text=new_text,
|
||||
overlap_left=overlap_left,
|
||||
overlap_right=overlap_right,
|
||||
boilerplate_score=chunk.boilerplate_score,
|
||||
))
|
||||
|
||||
return result
|
||||
|
||||
def _apply_metadata(
|
||||
self,
|
||||
chunks: list[DocumentChunk],
|
||||
text: str,
|
||||
document_type: str,
|
||||
) -> None:
|
||||
"""Apply section_path and speaker metadata to chunks in place."""
|
||||
if document_type.lower() == "transcript":
|
||||
self._apply_speaker_metadata(chunks, text)
|
||||
elif document_type.lower() == "filing":
|
||||
self._apply_filing_sections(chunks, text)
|
||||
|
||||
def _apply_speaker_metadata(
|
||||
self,
|
||||
chunks: list[DocumentChunk],
|
||||
text: str,
|
||||
) -> None:
|
||||
"""Find speaker turns and assign speaker labels to transcript chunks."""
|
||||
speakers: list[tuple[int, str]] = []
|
||||
for match in _SPEAKER_PATTERN.finditer(text):
|
||||
speakers.append((match.start(), match.group(1).strip()))
|
||||
|
||||
if not speakers:
|
||||
return
|
||||
|
||||
for chunk in chunks:
|
||||
# Find the most recent speaker before or within this chunk
|
||||
current_speaker = None
|
||||
for pos, name in speakers:
|
||||
if pos <= chunk.end_char:
|
||||
if pos < chunk.start_char:
|
||||
current_speaker = name
|
||||
else:
|
||||
current_speaker = name
|
||||
else:
|
||||
break
|
||||
chunk.speaker = current_speaker
|
||||
|
||||
def _apply_filing_sections(
|
||||
self,
|
||||
chunks: list[DocumentChunk],
|
||||
text: str,
|
||||
) -> None:
|
||||
"""Find filing section headers and assign section_path to chunks."""
|
||||
sections: list[tuple[int, str]] = []
|
||||
for match in _FILING_SECTION_PATTERN.finditer(text):
|
||||
sections.append((match.start(), match.group(1).strip()))
|
||||
|
||||
if not sections:
|
||||
return
|
||||
|
||||
for chunk in chunks:
|
||||
# Build section path from all sections that precede or start within this chunk
|
||||
path: list[str] = []
|
||||
for pos, title in sections:
|
||||
if pos < chunk.end_char:
|
||||
# Keep track of the most recent section(s)
|
||||
if pos <= chunk.start_char:
|
||||
path = [title]
|
||||
else:
|
||||
path.append(title)
|
||||
else:
|
||||
break
|
||||
chunk.section_path = path
|
||||
Reference in New Issue
Block a user