Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
353 lines
12 KiB
Python
353 lines
12 KiB
Python
"""Sentence-aware document segmenter.
|
|
|
|
Splits documents into chunks that respect sentence boundaries, preserve
|
|
source character offsets, and use document-type-specific strategies.
|
|
|
|
Key invariant: chunk.text == source_text[chunk.start_char:chunk.end_char]
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
from services.intelligence_pipeline_v3.segmenter.boilerplate import score_boilerplate
|
|
from services.intelligence_pipeline_v3.segmenter.models import DocumentChunk
|
|
from services.intelligence_pipeline_v3.segmenter.strategies import (
|
|
ChunkStrategy,
|
|
get_strategy,
|
|
)
|
|
|
|
# Sentence-ending patterns: period/question/exclamation followed by space or newline
|
|
_SENTENCE_END = re.compile(r"(?<=[.!?])\s+")
|
|
|
|
# Speaker turn pattern for transcripts
|
|
_SPEAKER_PATTERN = re.compile(
|
|
r"^([A-Z][a-zA-Z\s\-\.]+(?:\s*[-–—]\s*[A-Za-z\s,]+)?)\s*:\s*",
|
|
re.MULTILINE,
|
|
)
|
|
|
|
# Filing section headers
|
|
_FILING_SECTION_PATTERN = re.compile(
|
|
r"^((?:Item\s+\d+[A-Z]?[\.\:]|PART\s+[IVX]+)[^\n]*)",
|
|
re.IGNORECASE | re.MULTILINE,
|
|
)
|
|
|
|
|
|
class Segmenter:
|
|
"""Sentence-aware document segmenter with document-type-specific strategies.
|
|
|
|
Does NOT truncate documents — processes full text regardless of length.
|
|
Each chunk preserves exact character offsets into the source document.
|
|
"""
|
|
|
|
def segment(
|
|
self,
|
|
text: str,
|
|
document_type: str,
|
|
document_id: str = "",
|
|
) -> list[DocumentChunk]:
|
|
"""Segment a document into chunks.
|
|
|
|
Args:
|
|
text: Full document text (no truncation applied).
|
|
document_type: Type of document for strategy selection.
|
|
document_id: Identifier for deterministic chunk IDs.
|
|
|
|
Returns:
|
|
List of DocumentChunk with valid offsets and checksums.
|
|
"""
|
|
if not text:
|
|
return []
|
|
|
|
strategy = get_strategy(document_type)
|
|
|
|
# Get structural boundaries based on document type
|
|
boundaries = self._find_boundaries(text, strategy, document_type)
|
|
|
|
# Build chunks respecting sentence boundaries and strategy limits
|
|
chunks = self._build_chunks(
|
|
text=text,
|
|
boundaries=boundaries,
|
|
strategy=strategy,
|
|
document_type=document_type,
|
|
document_id=document_id,
|
|
)
|
|
|
|
# Apply overlap between adjacent chunks
|
|
chunks = self._apply_overlap(chunks, text, strategy, document_id, document_type)
|
|
|
|
# Score boilerplate
|
|
for chunk in chunks:
|
|
chunk.boilerplate_score = score_boilerplate(chunk.text)
|
|
|
|
# Apply section path and speaker metadata
|
|
self._apply_metadata(chunks, text, document_type)
|
|
|
|
return chunks
|
|
|
|
def _find_boundaries(
|
|
self,
|
|
text: str,
|
|
strategy: ChunkStrategy,
|
|
document_type: str,
|
|
) -> list[int]:
|
|
"""Find structural boundary positions in the text.
|
|
|
|
Returns sorted list of character offsets where structural breaks occur.
|
|
"""
|
|
boundaries: set[int] = set()
|
|
|
|
for pattern in strategy.boundary_patterns:
|
|
for match in pattern.finditer(text):
|
|
boundaries.add(match.start())
|
|
|
|
return sorted(boundaries)
|
|
|
|
def _find_sentence_boundaries(self, text: str, start: int, end: int) -> list[int]:
|
|
"""Find sentence-ending positions within a text range.
|
|
|
|
Returns character offsets (relative to full document) after sentence-ending punctuation.
|
|
"""
|
|
segment = text[start:end]
|
|
positions: list[int] = []
|
|
for match in _SENTENCE_END.finditer(segment):
|
|
positions.append(start + match.start())
|
|
return positions
|
|
|
|
def _build_chunks(
|
|
self,
|
|
text: str,
|
|
boundaries: list[int],
|
|
strategy: ChunkStrategy,
|
|
document_type: str,
|
|
document_id: str,
|
|
) -> list[DocumentChunk]:
|
|
"""Build initial chunks from text using boundaries and sentence awareness."""
|
|
chunks: list[DocumentChunk] = []
|
|
text_len = len(text)
|
|
pos = 0
|
|
|
|
while pos < text_len:
|
|
# Determine the ideal end position
|
|
ideal_end = min(pos + strategy.target_chars, text_len)
|
|
max_end = min(pos + strategy.max_chars, text_len)
|
|
|
|
if ideal_end >= text_len:
|
|
# Last chunk — take everything remaining
|
|
chunk_end = text_len
|
|
else:
|
|
# Try to break at a structural boundary between ideal and max
|
|
chunk_end = self._find_best_break(
|
|
text, pos, ideal_end, max_end, boundaries, strategy
|
|
)
|
|
|
|
chunk_text = text[pos:chunk_end]
|
|
|
|
# Skip empty chunks (shouldn't happen, but defensive)
|
|
if not chunk_text.strip():
|
|
pos = chunk_end
|
|
continue
|
|
|
|
chunk = DocumentChunk(
|
|
chunk_id=f"{document_id}:{pos}",
|
|
document_id=document_id,
|
|
document_type=document_type,
|
|
section_path=[],
|
|
speaker=None,
|
|
start_char=pos,
|
|
end_char=chunk_end,
|
|
text=chunk_text,
|
|
overlap_left=0,
|
|
overlap_right=0,
|
|
boilerplate_score=0.0,
|
|
)
|
|
chunks.append(chunk)
|
|
pos = chunk_end
|
|
|
|
return chunks
|
|
|
|
def _find_best_break(
|
|
self,
|
|
text: str,
|
|
start: int,
|
|
ideal_end: int,
|
|
max_end: int,
|
|
boundaries: list[int],
|
|
strategy: ChunkStrategy,
|
|
) -> int:
|
|
"""Find the best break point between ideal_end and max_end.
|
|
|
|
Priority:
|
|
1. Structural boundary nearest to ideal_end (within target..max range)
|
|
2. Sentence boundary nearest to ideal_end
|
|
3. Whitespace nearest to ideal_end
|
|
4. Hard cut at ideal_end
|
|
"""
|
|
# Look for structural boundaries in the window [ideal_end - target_chars/4, max_end]
|
|
search_start = max(start, ideal_end - strategy.target_chars // 4)
|
|
best_structural = None
|
|
for b in boundaries:
|
|
if search_start <= b <= max_end and b > start:
|
|
if best_structural is None or abs(b - ideal_end) < abs(best_structural - ideal_end):
|
|
best_structural = b
|
|
if best_structural is not None:
|
|
return best_structural
|
|
|
|
# Look for sentence boundaries near ideal_end
|
|
sentence_breaks = self._find_sentence_boundaries(text, search_start, max_end)
|
|
if sentence_breaks:
|
|
# Pick the one closest to ideal_end
|
|
best_sentence = min(sentence_breaks, key=lambda s: abs(s - ideal_end))
|
|
# Use position after the sentence-ending whitespace
|
|
after = best_sentence
|
|
while after < max_end and text[after] in " \t\n\r":
|
|
after += 1
|
|
return after
|
|
|
|
# Fall back to whitespace break
|
|
search_region = text[ideal_end:max_end]
|
|
ws_match = re.search(r"\s+", search_region)
|
|
if ws_match:
|
|
return ideal_end + ws_match.end()
|
|
|
|
# Hard cut
|
|
return ideal_end
|
|
|
|
def _apply_overlap(
|
|
self,
|
|
chunks: list[DocumentChunk],
|
|
text: str,
|
|
strategy: ChunkStrategy,
|
|
document_id: str,
|
|
document_type: str,
|
|
) -> list[DocumentChunk]:
|
|
"""Apply overlap between adjacent chunks by extending start/end.
|
|
|
|
Overlap is achieved by moving each chunk's start_char backward
|
|
to include trailing content from the previous chunk.
|
|
"""
|
|
if len(chunks) <= 1 or strategy.overlap_chars == 0:
|
|
return chunks
|
|
|
|
result: list[DocumentChunk] = []
|
|
for i, chunk in enumerate(chunks):
|
|
new_start = chunk.start_char
|
|
new_end = chunk.end_char
|
|
overlap_left = 0
|
|
overlap_right = 0
|
|
|
|
if i > 0:
|
|
# Extend start backward for overlap
|
|
overlap_target = min(strategy.overlap_chars, chunk.start_char)
|
|
new_start = max(0, chunk.start_char - overlap_target)
|
|
|
|
# Snap to sentence boundary if possible
|
|
if new_start < chunk.start_char:
|
|
region = text[new_start:chunk.start_char]
|
|
sentence_matches = list(_SENTENCE_END.finditer(region))
|
|
if sentence_matches:
|
|
# Use the latest sentence break in the overlap region
|
|
last_match = sentence_matches[-1]
|
|
candidate = new_start + last_match.end()
|
|
if candidate < chunk.start_char:
|
|
new_start = candidate
|
|
|
|
overlap_left = chunk.start_char - new_start
|
|
|
|
if i < len(chunks) - 1:
|
|
# Calculate how much the next chunk will overlap into this one
|
|
next_chunk = chunks[i + 1]
|
|
overlap_target = min(strategy.overlap_chars, len(text) - next_chunk.start_char)
|
|
potential_overlap_start = max(0, next_chunk.start_char - overlap_target)
|
|
|
|
# The overlap_right for this chunk = how much the next chunk's
|
|
# overlap will include from this chunk's content
|
|
overlap_right = chunk.end_char - max(potential_overlap_start, chunk.start_char)
|
|
overlap_right = max(0, overlap_right)
|
|
|
|
new_text = text[new_start:new_end]
|
|
if not new_text.strip():
|
|
result.append(chunk)
|
|
continue
|
|
|
|
result.append(DocumentChunk(
|
|
chunk_id=f"{document_id}:{new_start}",
|
|
document_id=document_id,
|
|
document_type=document_type,
|
|
section_path=chunk.section_path,
|
|
speaker=chunk.speaker,
|
|
start_char=new_start,
|
|
end_char=new_end,
|
|
text=new_text,
|
|
overlap_left=overlap_left,
|
|
overlap_right=overlap_right,
|
|
boilerplate_score=chunk.boilerplate_score,
|
|
))
|
|
|
|
return result
|
|
|
|
def _apply_metadata(
|
|
self,
|
|
chunks: list[DocumentChunk],
|
|
text: str,
|
|
document_type: str,
|
|
) -> None:
|
|
"""Apply section_path and speaker metadata to chunks in place."""
|
|
if document_type.lower() == "transcript":
|
|
self._apply_speaker_metadata(chunks, text)
|
|
elif document_type.lower() == "filing":
|
|
self._apply_filing_sections(chunks, text)
|
|
|
|
def _apply_speaker_metadata(
|
|
self,
|
|
chunks: list[DocumentChunk],
|
|
text: str,
|
|
) -> None:
|
|
"""Find speaker turns and assign speaker labels to transcript chunks."""
|
|
speakers: list[tuple[int, str]] = []
|
|
for match in _SPEAKER_PATTERN.finditer(text):
|
|
speakers.append((match.start(), match.group(1).strip()))
|
|
|
|
if not speakers:
|
|
return
|
|
|
|
for chunk in chunks:
|
|
# Find the most recent speaker before or within this chunk
|
|
current_speaker = None
|
|
for pos, name in speakers:
|
|
if pos <= chunk.end_char:
|
|
if pos < chunk.start_char:
|
|
current_speaker = name
|
|
else:
|
|
current_speaker = name
|
|
else:
|
|
break
|
|
chunk.speaker = current_speaker
|
|
|
|
def _apply_filing_sections(
|
|
self,
|
|
chunks: list[DocumentChunk],
|
|
text: str,
|
|
) -> None:
|
|
"""Find filing section headers and assign section_path to chunks."""
|
|
sections: list[tuple[int, str]] = []
|
|
for match in _FILING_SECTION_PATTERN.finditer(text):
|
|
sections.append((match.start(), match.group(1).strip()))
|
|
|
|
if not sections:
|
|
return
|
|
|
|
for chunk in chunks:
|
|
# Build section path from all sections that precede or start within this chunk
|
|
path: list[str] = []
|
|
for pos, title in sections:
|
|
if pos < chunk.end_char:
|
|
# Keep track of the most recent section(s)
|
|
if pos <= chunk.start_char:
|
|
path = [title]
|
|
else:
|
|
path.append(title)
|
|
else:
|
|
break
|
|
chunk.section_path = path
|