Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
95 lines
2.8 KiB
Python
95 lines
2.8 KiB
Python
"""Simple boilerplate detection for document chunks.
|
|
|
|
Returns a 0.0-1.0 score indicating how likely a chunk is boilerplate.
|
|
Does NOT delete chunks — only marks them for downstream filtering decisions.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
# Common boilerplate patterns in financial documents
|
|
_BOILERPLATE_PATTERNS: list[tuple[re.Pattern[str], float]] = [
|
|
# Forward-looking statements disclaimer
|
|
(re.compile(
|
|
r"forward[- ]looking\s+statements?",
|
|
re.IGNORECASE,
|
|
), 0.4),
|
|
# Safe harbor language
|
|
(re.compile(
|
|
r"safe\s+harbor",
|
|
re.IGNORECASE,
|
|
), 0.3),
|
|
# Copyright notices
|
|
(re.compile(
|
|
r"(?:©|\bcopyright\b)\s*\d{4}",
|
|
re.IGNORECASE,
|
|
), 0.3),
|
|
# All rights reserved
|
|
(re.compile(
|
|
r"all\s+rights\s+reserved",
|
|
re.IGNORECASE,
|
|
), 0.2),
|
|
# Disclaimer language
|
|
(re.compile(
|
|
r"\b(?:disclaimer|disclaims?)\b",
|
|
re.IGNORECASE,
|
|
), 0.2),
|
|
# "This press release" meta-reference
|
|
(re.compile(
|
|
r"this\s+(?:press\s+release|report|document)\s+(?:contains?|includes?|may\s+contain)",
|
|
re.IGNORECASE,
|
|
), 0.2),
|
|
# Not an offer/solicitation language
|
|
(re.compile(
|
|
r"(?:not|does\s+not)\s+constitute\s+(?:an?\s+)?(?:offer|solicitation|recommendation)",
|
|
re.IGNORECASE,
|
|
), 0.3),
|
|
# Boilerplate risk factors intro
|
|
(re.compile(
|
|
r"(?:actual\s+results|future\s+results)\s+(?:may|could|might)\s+differ\s+materially",
|
|
re.IGNORECASE,
|
|
), 0.3),
|
|
# Contact/investor relations boilerplate
|
|
(re.compile(
|
|
r"(?:investor\s+relations?|media\s+(?:contact|inquiries))\s*:",
|
|
re.IGNORECASE,
|
|
), 0.2),
|
|
# Legal entity registrations
|
|
(re.compile(
|
|
r"registered\s+(?:in|under)\s+(?:the\s+)?(?:laws?\s+of|state\s+of)",
|
|
re.IGNORECASE,
|
|
), 0.15),
|
|
]
|
|
|
|
# If the chunk is mostly short lines (like a signature block), boost score
|
|
_SHORT_LINE_THRESHOLD = 40
|
|
_SHORT_LINE_RATIO_THRESHOLD = 0.7
|
|
|
|
|
|
def score_boilerplate(text: str) -> float:
|
|
"""Score a text chunk for boilerplate content.
|
|
|
|
Returns a float between 0.0 (not boilerplate) and 1.0 (definitely boilerplate).
|
|
The score is the sum of matched pattern weights, capped at 1.0.
|
|
"""
|
|
if not text.strip():
|
|
return 0.0
|
|
|
|
score = 0.0
|
|
|
|
# Check pattern matches
|
|
for pattern, weight in _BOILERPLATE_PATTERNS:
|
|
if pattern.search(text):
|
|
score += weight
|
|
|
|
# Check for signature-block-like structure (many short lines)
|
|
lines = text.split("\n")
|
|
if lines:
|
|
short_lines = sum(1 for line in lines if 0 < len(line.strip()) <= _SHORT_LINE_THRESHOLD)
|
|
non_empty = sum(1 for line in lines if line.strip())
|
|
if non_empty > 3 and short_lines / non_empty > _SHORT_LINE_RATIO_THRESHOLD:
|
|
score += 0.15
|
|
|
|
return min(score, 1.0)
|