"""Sentence-aware document segmenter. Splits documents into chunks that respect sentence boundaries, preserve source character offsets, and use document-type-specific strategies. Key invariant: chunk.text == source_text[chunk.start_char:chunk.end_char] """ from __future__ import annotations import re from services.intelligence_pipeline_v3.segmenter.boilerplate import score_boilerplate from services.intelligence_pipeline_v3.segmenter.models import DocumentChunk from services.intelligence_pipeline_v3.segmenter.strategies import ( ChunkStrategy, get_strategy, ) # Sentence-ending patterns: period/question/exclamation followed by space or newline _SENTENCE_END = re.compile(r"(?<=[.!?])\s+") # Speaker turn pattern for transcripts _SPEAKER_PATTERN = re.compile( r"^([A-Z][a-zA-Z\s\-\.]+(?:\s*[-–—]\s*[A-Za-z\s,]+)?)\s*:\s*", re.MULTILINE, ) # Filing section headers _FILING_SECTION_PATTERN = re.compile( r"^((?:Item\s+\d+[A-Z]?[\.\:]|PART\s+[IVX]+)[^\n]*)", re.IGNORECASE | re.MULTILINE, ) class Segmenter: """Sentence-aware document segmenter with document-type-specific strategies. Does NOT truncate documents — processes full text regardless of length. Each chunk preserves exact character offsets into the source document. """ def segment( self, text: str, document_type: str, document_id: str = "", ) -> list[DocumentChunk]: """Segment a document into chunks. Args: text: Full document text (no truncation applied). document_type: Type of document for strategy selection. document_id: Identifier for deterministic chunk IDs. Returns: List of DocumentChunk with valid offsets and checksums. """ if not text: return [] strategy = get_strategy(document_type) # Get structural boundaries based on document type boundaries = self._find_boundaries(text, strategy, document_type) # Build chunks respecting sentence boundaries and strategy limits chunks = self._build_chunks( text=text, boundaries=boundaries, strategy=strategy, document_type=document_type, document_id=document_id, ) # Apply overlap between adjacent chunks chunks = self._apply_overlap(chunks, text, strategy, document_id, document_type) # Score boilerplate for chunk in chunks: chunk.boilerplate_score = score_boilerplate(chunk.text) # Apply section path and speaker metadata self._apply_metadata(chunks, text, document_type) return chunks def _find_boundaries( self, text: str, strategy: ChunkStrategy, document_type: str, ) -> list[int]: """Find structural boundary positions in the text. Returns sorted list of character offsets where structural breaks occur. """ boundaries: set[int] = set() for pattern in strategy.boundary_patterns: for match in pattern.finditer(text): boundaries.add(match.start()) return sorted(boundaries) def _find_sentence_boundaries(self, text: str, start: int, end: int) -> list[int]: """Find sentence-ending positions within a text range. Returns character offsets (relative to full document) after sentence-ending punctuation. """ segment = text[start:end] positions: list[int] = [] for match in _SENTENCE_END.finditer(segment): positions.append(start + match.start()) return positions def _build_chunks( self, text: str, boundaries: list[int], strategy: ChunkStrategy, document_type: str, document_id: str, ) -> list[DocumentChunk]: """Build initial chunks from text using boundaries and sentence awareness.""" chunks: list[DocumentChunk] = [] text_len = len(text) pos = 0 while pos < text_len: # Determine the ideal end position ideal_end = min(pos + strategy.target_chars, text_len) max_end = min(pos + strategy.max_chars, text_len) if ideal_end >= text_len: # Last chunk — take everything remaining chunk_end = text_len else: # Try to break at a structural boundary between ideal and max chunk_end = self._find_best_break( text, pos, ideal_end, max_end, boundaries, strategy ) chunk_text = text[pos:chunk_end] # Skip empty chunks (shouldn't happen, but defensive) if not chunk_text.strip(): pos = chunk_end continue chunk = DocumentChunk( chunk_id=f"{document_id}:{pos}", document_id=document_id, document_type=document_type, section_path=[], speaker=None, start_char=pos, end_char=chunk_end, text=chunk_text, overlap_left=0, overlap_right=0, boilerplate_score=0.0, ) chunks.append(chunk) pos = chunk_end return chunks def _find_best_break( self, text: str, start: int, ideal_end: int, max_end: int, boundaries: list[int], strategy: ChunkStrategy, ) -> int: """Find the best break point between ideal_end and max_end. Priority: 1. Structural boundary nearest to ideal_end (within target..max range) 2. Sentence boundary nearest to ideal_end 3. Whitespace nearest to ideal_end 4. Hard cut at ideal_end """ # Look for structural boundaries in the window [ideal_end - target_chars/4, max_end] search_start = max(start, ideal_end - strategy.target_chars // 4) best_structural = None for b in boundaries: if search_start <= b <= max_end and b > start: if best_structural is None or abs(b - ideal_end) < abs(best_structural - ideal_end): best_structural = b if best_structural is not None: return best_structural # Look for sentence boundaries near ideal_end sentence_breaks = self._find_sentence_boundaries(text, search_start, max_end) if sentence_breaks: # Pick the one closest to ideal_end best_sentence = min(sentence_breaks, key=lambda s: abs(s - ideal_end)) # Use position after the sentence-ending whitespace after = best_sentence while after < max_end and text[after] in " \t\n\r": after += 1 return after # Fall back to whitespace break search_region = text[ideal_end:max_end] ws_match = re.search(r"\s+", search_region) if ws_match: return ideal_end + ws_match.end() # Hard cut return ideal_end def _apply_overlap( self, chunks: list[DocumentChunk], text: str, strategy: ChunkStrategy, document_id: str, document_type: str, ) -> list[DocumentChunk]: """Apply overlap between adjacent chunks by extending start/end. Overlap is achieved by moving each chunk's start_char backward to include trailing content from the previous chunk. """ if len(chunks) <= 1 or strategy.overlap_chars == 0: return chunks result: list[DocumentChunk] = [] for i, chunk in enumerate(chunks): new_start = chunk.start_char new_end = chunk.end_char overlap_left = 0 overlap_right = 0 if i > 0: # Extend start backward for overlap overlap_target = min(strategy.overlap_chars, chunk.start_char) new_start = max(0, chunk.start_char - overlap_target) # Snap to sentence boundary if possible if new_start < chunk.start_char: region = text[new_start:chunk.start_char] sentence_matches = list(_SENTENCE_END.finditer(region)) if sentence_matches: # Use the latest sentence break in the overlap region last_match = sentence_matches[-1] candidate = new_start + last_match.end() if candidate < chunk.start_char: new_start = candidate overlap_left = chunk.start_char - new_start if i < len(chunks) - 1: # Calculate how much the next chunk will overlap into this one next_chunk = chunks[i + 1] overlap_target = min(strategy.overlap_chars, len(text) - next_chunk.start_char) potential_overlap_start = max(0, next_chunk.start_char - overlap_target) # The overlap_right for this chunk = how much the next chunk's # overlap will include from this chunk's content overlap_right = chunk.end_char - max(potential_overlap_start, chunk.start_char) overlap_right = max(0, overlap_right) new_text = text[new_start:new_end] if not new_text.strip(): result.append(chunk) continue result.append(DocumentChunk( chunk_id=f"{document_id}:{new_start}", document_id=document_id, document_type=document_type, section_path=chunk.section_path, speaker=chunk.speaker, start_char=new_start, end_char=new_end, text=new_text, overlap_left=overlap_left, overlap_right=overlap_right, boilerplate_score=chunk.boilerplate_score, )) return result def _apply_metadata( self, chunks: list[DocumentChunk], text: str, document_type: str, ) -> None: """Apply section_path and speaker metadata to chunks in place.""" if document_type.lower() == "transcript": self._apply_speaker_metadata(chunks, text) elif document_type.lower() == "filing": self._apply_filing_sections(chunks, text) def _apply_speaker_metadata( self, chunks: list[DocumentChunk], text: str, ) -> None: """Find speaker turns and assign speaker labels to transcript chunks.""" speakers: list[tuple[int, str]] = [] for match in _SPEAKER_PATTERN.finditer(text): speakers.append((match.start(), match.group(1).strip())) if not speakers: return for chunk in chunks: # Find the most recent speaker before or within this chunk current_speaker = None for pos, name in speakers: if pos <= chunk.end_char: if pos < chunk.start_char: current_speaker = name else: current_speaker = name else: break chunk.speaker = current_speaker def _apply_filing_sections( self, chunks: list[DocumentChunk], text: str, ) -> None: """Find filing section headers and assign section_path to chunks.""" sections: list[tuple[int, str]] = [] for match in _FILING_SECTION_PATTERN.finditer(text): sections.append((match.start(), match.group(1).strip())) if not sections: return for chunk in chunks: # Build section path from all sections that precede or start within this chunk path: list[str] = [] for pos, title in sections: if pos < chunk.end_char: # Keep track of the most recent section(s) if pos <= chunk.start_char: path = [title] else: path.append(title) else: break chunk.section_path = path