feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,26 @@
|
||||
"""Deterministic financial parsing for the v3 intelligence pipeline.
|
||||
|
||||
This package provides regex-based detection of financial entities:
|
||||
- Ticker symbols ($AAPL, AAPL)
|
||||
- Currencies and money amounts ($123.45, €99, $94.9 billion)
|
||||
- Percentages (4%, -2.5%)
|
||||
- Basis points (25 basis points, 25bps)
|
||||
- Ranges ($10-$12)
|
||||
- EPS values ($1.52 per share)
|
||||
- Revenue figures
|
||||
- Dates and fiscal periods (Q1 2024, FY2025)
|
||||
|
||||
Each match returns exact character offsets, literal text, and a normalized numeric value.
|
||||
"""
|
||||
|
||||
from services.intelligence_pipeline_v3.parsing.financial_parser import FinancialParser
|
||||
from services.intelligence_pipeline_v3.parsing.models import CandidateType, ParsedCandidate, PeriodAnnotation
|
||||
from services.intelligence_pipeline_v3.parsing.normalizer import normalize_value
|
||||
|
||||
__all__ = [
|
||||
"CandidateType",
|
||||
"FinancialParser",
|
||||
"ParsedCandidate",
|
||||
"PeriodAnnotation",
|
||||
"normalize_value",
|
||||
]
|
||||
@@ -0,0 +1,426 @@
|
||||
"""Deterministic financial parser using regex-based detection.
|
||||
|
||||
Detects tickers, currencies, money amounts, percentages, basis points,
|
||||
ranges, EPS, revenue, dates, and fiscal periods from source text.
|
||||
Each match returns exact character offsets into the source text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from services.intelligence_pipeline_v3.parsing.models import (
|
||||
CandidateType,
|
||||
ParsedCandidate,
|
||||
PeriodAnnotation,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.parsing.normalizer import (
|
||||
normalize_basis_points,
|
||||
normalize_money,
|
||||
normalize_percentage,
|
||||
normalize_range,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Regex patterns
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Ticker: $AAPL or standalone AAPL-like (1-5 uppercase letters)
|
||||
_TICKER_DOLLAR_RE = re.compile(r"\$([A-Z]{1,5})\b")
|
||||
_TICKER_BARE_RE = re.compile(r"\b([A-Z]{1,5})\b")
|
||||
|
||||
# Common English words that look like tickers but aren't
|
||||
_TICKER_STOPWORDS = frozenset({
|
||||
"A", "I", "AM", "AN", "AS", "AT", "BE", "BY", "DO", "GO", "HE", "IF",
|
||||
"IN", "IS", "IT", "ME", "MY", "NO", "OF", "OK", "ON", "OR", "OUR", "SO",
|
||||
"THE", "TO", "UP", "US", "WE", "CEO", "CFO", "COO", "CTO", "EPS", "ETF",
|
||||
"GDP", "IPO", "LLC", "LTD", "NYSE", "SEC", "USA", "AND", "ARE", "BUT",
|
||||
"CAN", "DID", "FOR", "GET", "GOT", "HAD", "HAS", "HER", "HIS", "HOW",
|
||||
"ITS", "LET", "MAY", "NEW", "NOT", "NOW", "OLD", "OUR", "OWN", "PUT",
|
||||
"RAN", "SAY", "SHE", "TOO", "TWO", "USE", "WAS", "WAY", "WHO", "WHY",
|
||||
"WIN", "WON", "YET", "YOU", "ALL", "ANY", "BIG", "DAY", "END", "FEW",
|
||||
"FAR", "HIT", "LOW", "MET", "NET", "OUT", "RUN", "SET", "TOP", "TRY",
|
||||
"ALSO", "BACK", "BEEN", "BEST", "BOTH", "CAME", "COME", "DOWN", "EACH",
|
||||
"FROM", "GAVE", "GOOD", "HAVE", "HERE", "HIGH", "INTO", "JUST", "KEEP",
|
||||
"LAST", "LONG", "MADE", "MAKE", "MANY", "MORE", "MOST", "MUCH", "MUST",
|
||||
"NEED", "NEXT", "ONLY", "OVER", "SAID", "SAME", "SOME", "SUCH", "TAKE",
|
||||
"THAN", "THAT", "THEM", "THEN", "THEY", "THIS", "VERY", "WANT", "WELL",
|
||||
"WENT", "WERE", "WHAT", "WHEN", "WILL", "WITH", "WORK", "YEAR", "YOUR",
|
||||
"ITEM", "CASH", "FLOW", "FREE", "FULL", "HALF", "RISE", "ROSE", "FELL",
|
||||
"BEAT", "MISS", "GREW", "GROW", "LOST", "LOSS", "GAIN", "HOLD", "SELL",
|
||||
"CALL", "BUY", "FUND", "BOND", "RATE", "DEBT", "DEAL", "RISK",
|
||||
"Q", "H", "FY", "YOY", "QOQ", "AI", "R", "D",
|
||||
})
|
||||
|
||||
# Fiscal period: Q1 2024, Q4'24, FY2025, FY25, H1 2024
|
||||
_FISCAL_PERIOD_RE = re.compile(
|
||||
r"\b(Q[1-4]|H[12]|FY)\s*['\u2019]?\s*(\d{4}|\d{2})\b"
|
||||
)
|
||||
|
||||
# Date patterns: January 15, 2024 / Jan 15, 2024 / 2024-01-15
|
||||
_MONTH_NAMES = (
|
||||
r"(?:January|February|March|April|May|June|July|August|September|October|November|December"
|
||||
r"|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
|
||||
)
|
||||
_DATE_NAMED_RE = re.compile(
|
||||
rf"\b({_MONTH_NAMES})\s+(\d{{1,2}})(?:,?\s+(\d{{4}}))?\b"
|
||||
)
|
||||
_DATE_ISO_RE = re.compile(r"\b(\d{4})-(\d{2})-(\d{2})\b")
|
||||
|
||||
# Basis points: "25 basis points", "50bps", "25 bps"
|
||||
_BASIS_POINTS_RE = re.compile(
|
||||
r"[+-]?\d[\d,]*\.?\d*\s*(?:basis\s+points?|bps)\b", re.IGNORECASE
|
||||
)
|
||||
|
||||
# Percentage: 4%, -2.5%, +1.2 percent
|
||||
_PERCENTAGE_RE = re.compile(
|
||||
r"[+-]?\d[\d,]*\.?\d*\s*(?:%|percent(?:age)?(?:\s+points?)?\b)", re.IGNORECASE
|
||||
)
|
||||
|
||||
# EPS: "$1.52 per share", "earnings per share of $1.52"
|
||||
_EPS_PER_SHARE_RE = re.compile(
|
||||
r"\$\s*\d[\d,]*\.?\d*\s+per\s+share\b", re.IGNORECASE
|
||||
)
|
||||
_EPS_PREFIX_RE = re.compile(
|
||||
r"\b(?:EPS|earnings\s+per\s+share)\s+(?:of\s+)?\$\s*\d[\d,]*\.?\d*", re.IGNORECASE
|
||||
)
|
||||
|
||||
# Revenue: "$94.9 billion in revenue", "revenue of $94.9 billion"
|
||||
_REVENUE_AMOUNT_RE = re.compile(
|
||||
r"\$\s*\d[\d,]*\.?\d*\s*(?:trillion|billion|million|bn|mn)\s+(?:in\s+)?revenue\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_REVENUE_PREFIX_RE = re.compile(
|
||||
r"\brevenue\s+(?:of|was|reached|hit|grew\s+to|increased\s+to|totaled)\s+\$\s*\d[\d,]*\.?\d*\s*(?:trillion|billion|million|bn|mn)?",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Range: "$10-$12", "$10 to $12", "$1.50-$2.00"
|
||||
_RANGE_RE = re.compile(
|
||||
r"\$\s*\d[\d,]*\.?\d*\s*(?:-|to|–|—)\s*\$?\s*\d[\d,]*\.?\d*",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Money with multiplier: "$94.9 billion", "$1.2 million", "€5 billion"
|
||||
_MONEY_MULT_RE = re.compile(
|
||||
r"[€£¥$]\s*\d[\d,]*\.?\d*\s*(?:trillion|billion|million|thousand|bn|mn|tn|[kmbt])\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Simple currency: $123.45, €99, £1,234.56
|
||||
# Note: requires digits after decimal point to avoid matching trailing periods
|
||||
_CURRENCY_RE = re.compile(
|
||||
r"[€£¥$]\s*\d[\d,]*(?:\.\d+)?"
|
||||
)
|
||||
|
||||
# Currency codes: USD, EUR, GBP, JPY
|
||||
_CURRENCY_SYMBOLS = {"$": "USD", "€": "EUR", "£": "GBP", "¥": "JPY"}
|
||||
|
||||
|
||||
def _detect_currency_unit(text: str) -> str:
|
||||
"""Detect currency unit from symbol in text."""
|
||||
for symbol, code in _CURRENCY_SYMBOLS.items():
|
||||
if symbol in text:
|
||||
return code
|
||||
return "USD"
|
||||
|
||||
|
||||
class FinancialParser:
|
||||
"""Deterministic regex-based financial entity parser.
|
||||
|
||||
Detects financial entities in text and returns ParsedCandidate instances
|
||||
with exact character offsets, literal text, and normalized values.
|
||||
"""
|
||||
|
||||
def parse(self, text: str) -> list[ParsedCandidate]:
|
||||
"""Parse text for financial entities.
|
||||
|
||||
Returns a list of ParsedCandidate sorted by start_char offset.
|
||||
Overlapping matches are resolved by priority (more specific wins).
|
||||
"""
|
||||
if not text or not text.strip():
|
||||
return []
|
||||
|
||||
candidates: list[ParsedCandidate] = []
|
||||
|
||||
# Order matters: more specific patterns first to claim offsets
|
||||
candidates.extend(self._parse_fiscal_periods(text))
|
||||
candidates.extend(self._parse_dates(text))
|
||||
candidates.extend(self._parse_basis_points(text))
|
||||
candidates.extend(self._parse_eps(text))
|
||||
candidates.extend(self._parse_revenue(text))
|
||||
candidates.extend(self._parse_ranges(text))
|
||||
candidates.extend(self._parse_percentages(text))
|
||||
candidates.extend(self._parse_money_with_multiplier(text))
|
||||
candidates.extend(self._parse_tickers(text))
|
||||
candidates.extend(self._parse_currency(text))
|
||||
|
||||
# Resolve overlaps: keep higher-priority (earlier in list) matches
|
||||
candidates = self._resolve_overlaps(candidates)
|
||||
|
||||
# Sort by position
|
||||
candidates.sort(key=lambda c: (c.start_char, -c.end_char))
|
||||
return candidates
|
||||
|
||||
def _resolve_overlaps(self, candidates: list[ParsedCandidate]) -> list[ParsedCandidate]:
|
||||
"""Remove overlapping candidates, keeping earlier ones (higher priority)."""
|
||||
if not candidates:
|
||||
return []
|
||||
|
||||
# Sort by start position for greedy non-overlap resolution
|
||||
sorted_candidates = sorted(candidates, key=lambda c: (c.start_char, -c.end_char))
|
||||
result: list[ParsedCandidate] = []
|
||||
claimed: list[tuple[int, int]] = []
|
||||
|
||||
for cand in sorted_candidates:
|
||||
overlaps = False
|
||||
for start, end in claimed:
|
||||
# Check if this candidate overlaps with any claimed range
|
||||
if cand.start_char < end and cand.end_char > start:
|
||||
overlaps = True
|
||||
break
|
||||
if not overlaps:
|
||||
result.append(cand)
|
||||
claimed.append((cand.start_char, cand.end_char))
|
||||
|
||||
return result
|
||||
|
||||
def _parse_tickers(self, text: str) -> list[ParsedCandidate]:
|
||||
"""Parse ticker symbols: $AAPL style."""
|
||||
results: list[ParsedCandidate] = []
|
||||
|
||||
# Dollar-prefixed tickers: $AAPL
|
||||
for match in _TICKER_DOLLAR_RE.finditer(text):
|
||||
ticker = match.group(1)
|
||||
if ticker not in _TICKER_STOPWORDS:
|
||||
results.append(ParsedCandidate(
|
||||
candidate_type=CandidateType.TICKER,
|
||||
literal_value=match.group(0),
|
||||
normalized_value=None,
|
||||
unit=None,
|
||||
start_char=match.start(),
|
||||
end_char=match.end(),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
def _parse_fiscal_periods(self, text: str) -> list[ParsedCandidate]:
|
||||
"""Parse fiscal periods: Q1 2024, FY2025, H1 2024."""
|
||||
results: list[ParsedCandidate] = []
|
||||
|
||||
for match in _FISCAL_PERIOD_RE.finditer(text):
|
||||
period_prefix = match.group(1)
|
||||
year_str = match.group(2)
|
||||
year = int(year_str)
|
||||
if len(year_str) == 2:
|
||||
year = 2000 + year if year < 80 else 1900 + year
|
||||
|
||||
if period_prefix.startswith("Q"):
|
||||
period_type = "quarter"
|
||||
period_value = period_prefix
|
||||
elif period_prefix.startswith("H"):
|
||||
period_type = "half"
|
||||
period_value = period_prefix
|
||||
else: # FY
|
||||
period_type = "fiscal_year"
|
||||
period_value = "FY"
|
||||
|
||||
results.append(ParsedCandidate(
|
||||
candidate_type=CandidateType.FISCAL_PERIOD,
|
||||
literal_value=match.group(0),
|
||||
normalized_value=None,
|
||||
unit=None,
|
||||
start_char=match.start(),
|
||||
end_char=match.end(),
|
||||
period=PeriodAnnotation(
|
||||
period_type=period_type,
|
||||
period_value=period_value,
|
||||
year=year,
|
||||
),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
def _parse_dates(self, text: str) -> list[ParsedCandidate]:
|
||||
"""Parse dates: January 15, 2024 / 2024-01-15."""
|
||||
results: list[ParsedCandidate] = []
|
||||
|
||||
for match in _DATE_NAMED_RE.finditer(text):
|
||||
results.append(ParsedCandidate(
|
||||
candidate_type=CandidateType.DATE,
|
||||
literal_value=match.group(0),
|
||||
normalized_value=None,
|
||||
unit=None,
|
||||
start_char=match.start(),
|
||||
end_char=match.end(),
|
||||
))
|
||||
|
||||
for match in _DATE_ISO_RE.finditer(text):
|
||||
results.append(ParsedCandidate(
|
||||
candidate_type=CandidateType.DATE,
|
||||
literal_value=match.group(0),
|
||||
normalized_value=None,
|
||||
unit=None,
|
||||
start_char=match.start(),
|
||||
end_char=match.end(),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
def _parse_basis_points(self, text: str) -> list[ParsedCandidate]:
|
||||
"""Parse basis points: 25 basis points, 50bps."""
|
||||
results: list[ParsedCandidate] = []
|
||||
|
||||
for match in _BASIS_POINTS_RE.finditer(text):
|
||||
literal = match.group(0)
|
||||
normalized = normalize_basis_points(literal)
|
||||
results.append(ParsedCandidate(
|
||||
candidate_type=CandidateType.BASIS_POINTS,
|
||||
literal_value=literal,
|
||||
normalized_value=normalized,
|
||||
unit="bps",
|
||||
start_char=match.start(),
|
||||
end_char=match.end(),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
def _parse_percentages(self, text: str) -> list[ParsedCandidate]:
|
||||
"""Parse percentages: 4%, -2.5%, +1.2 percent."""
|
||||
results: list[ParsedCandidate] = []
|
||||
|
||||
for match in _PERCENTAGE_RE.finditer(text):
|
||||
literal = match.group(0)
|
||||
normalized = normalize_percentage(literal)
|
||||
results.append(ParsedCandidate(
|
||||
candidate_type=CandidateType.PERCENTAGE,
|
||||
literal_value=literal,
|
||||
normalized_value=normalized,
|
||||
unit="%",
|
||||
start_char=match.start(),
|
||||
end_char=match.end(),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
def _parse_eps(self, text: str) -> list[ParsedCandidate]:
|
||||
"""Parse EPS values: $1.52 per share, EPS of $1.52."""
|
||||
results: list[ParsedCandidate] = []
|
||||
|
||||
for match in _EPS_PER_SHARE_RE.finditer(text):
|
||||
literal = match.group(0)
|
||||
normalized = normalize_money(literal)
|
||||
results.append(ParsedCandidate(
|
||||
candidate_type=CandidateType.EPS,
|
||||
literal_value=literal,
|
||||
normalized_value=normalized,
|
||||
unit="USD",
|
||||
start_char=match.start(),
|
||||
end_char=match.end(),
|
||||
))
|
||||
|
||||
for match in _EPS_PREFIX_RE.finditer(text):
|
||||
literal = match.group(0)
|
||||
normalized = normalize_money(literal)
|
||||
results.append(ParsedCandidate(
|
||||
candidate_type=CandidateType.EPS,
|
||||
literal_value=literal,
|
||||
normalized_value=normalized,
|
||||
unit="USD",
|
||||
start_char=match.start(),
|
||||
end_char=match.end(),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
def _parse_revenue(self, text: str) -> list[ParsedCandidate]:
|
||||
"""Parse revenue figures: $94.9 billion in revenue, revenue of $50 billion."""
|
||||
results: list[ParsedCandidate] = []
|
||||
|
||||
for match in _REVENUE_AMOUNT_RE.finditer(text):
|
||||
literal = match.group(0)
|
||||
normalized = normalize_money(literal)
|
||||
results.append(ParsedCandidate(
|
||||
candidate_type=CandidateType.REVENUE,
|
||||
literal_value=literal,
|
||||
normalized_value=normalized,
|
||||
unit="USD",
|
||||
start_char=match.start(),
|
||||
end_char=match.end(),
|
||||
))
|
||||
|
||||
for match in _REVENUE_PREFIX_RE.finditer(text):
|
||||
literal = match.group(0)
|
||||
normalized = normalize_money(literal)
|
||||
results.append(ParsedCandidate(
|
||||
candidate_type=CandidateType.REVENUE,
|
||||
literal_value=literal,
|
||||
normalized_value=normalized,
|
||||
unit="USD",
|
||||
start_char=match.start(),
|
||||
end_char=match.end(),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
def _parse_ranges(self, text: str) -> list[ParsedCandidate]:
|
||||
"""Parse ranges: $10-$12, $1.50 to $2.00."""
|
||||
results: list[ParsedCandidate] = []
|
||||
|
||||
for match in _RANGE_RE.finditer(text):
|
||||
literal = match.group(0)
|
||||
low, high = normalize_range(literal)
|
||||
# Store midpoint as normalized value
|
||||
normalized = None
|
||||
if low is not None and high is not None:
|
||||
normalized = (low + high) / 2.0
|
||||
|
||||
unit = _detect_currency_unit(literal)
|
||||
results.append(ParsedCandidate(
|
||||
candidate_type=CandidateType.RANGE,
|
||||
literal_value=literal,
|
||||
normalized_value=normalized,
|
||||
unit=unit,
|
||||
start_char=match.start(),
|
||||
end_char=match.end(),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
def _parse_money_with_multiplier(self, text: str) -> list[ParsedCandidate]:
|
||||
"""Parse money with multiplier: $94.9 billion, €5 million."""
|
||||
results: list[ParsedCandidate] = []
|
||||
|
||||
for match in _MONEY_MULT_RE.finditer(text):
|
||||
literal = match.group(0)
|
||||
normalized = normalize_money(literal)
|
||||
unit = _detect_currency_unit(literal)
|
||||
results.append(ParsedCandidate(
|
||||
candidate_type=CandidateType.MONEY,
|
||||
literal_value=literal,
|
||||
normalized_value=normalized,
|
||||
unit=unit,
|
||||
start_char=match.start(),
|
||||
end_char=match.end(),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
def _parse_currency(self, text: str) -> list[ParsedCandidate]:
|
||||
"""Parse simple currency: $123.45, €99, £1,234.56."""
|
||||
results: list[ParsedCandidate] = []
|
||||
|
||||
for match in _CURRENCY_RE.finditer(text):
|
||||
literal = match.group(0)
|
||||
normalized = normalize_money(literal)
|
||||
unit = _detect_currency_unit(literal)
|
||||
results.append(ParsedCandidate(
|
||||
candidate_type=CandidateType.CURRENCY,
|
||||
literal_value=literal,
|
||||
normalized_value=normalized,
|
||||
unit=unit,
|
||||
start_char=match.start(),
|
||||
end_char=match.end(),
|
||||
))
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Pydantic models for parsed financial candidates."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class CandidateType(str, Enum):
|
||||
"""Types of financial entities detected by the deterministic parser."""
|
||||
|
||||
TICKER = "ticker"
|
||||
CURRENCY = "currency"
|
||||
MONEY = "money"
|
||||
PERCENTAGE = "percentage"
|
||||
BASIS_POINTS = "basis_points"
|
||||
RANGE = "range"
|
||||
EPS = "eps"
|
||||
REVENUE = "revenue"
|
||||
DATE = "date"
|
||||
FISCAL_PERIOD = "fiscal_period"
|
||||
|
||||
|
||||
class PeriodAnnotation(BaseModel):
|
||||
"""Optional period context for a parsed candidate (e.g., Q1, FY2025)."""
|
||||
|
||||
period_type: str = Field(description="Type: quarter, year, fiscal_year, half")
|
||||
period_value: str = Field(description="Normalized period: Q1, Q2, H1, FY")
|
||||
year: int | None = Field(default=None, description="Calendar or fiscal year")
|
||||
|
||||
|
||||
class ParsedCandidate(BaseModel):
|
||||
"""A single parsed financial entity with source offset and normalization.
|
||||
|
||||
Stores both the literal text as it appeared in the source document and the
|
||||
normalized numeric value (if applicable). Exact character offsets allow
|
||||
downstream evidence linking back to the source.
|
||||
"""
|
||||
|
||||
candidate_type: CandidateType = Field(description="Classification of the parsed entity")
|
||||
literal_value: str = Field(min_length=1, description="Exact text as it appears in source")
|
||||
normalized_value: float | None = Field(default=None, description="Normalized numeric value")
|
||||
unit: str | None = Field(default=None, description="Unit: USD, EUR, %, bps, etc.")
|
||||
start_char: int = Field(ge=0, description="Start character offset in source text")
|
||||
end_char: int = Field(gt=0, description="End character offset in source text (exclusive)")
|
||||
period: PeriodAnnotation | None = Field(default=None, description="Optional fiscal/calendar period")
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Normalization rules for financial text values.
|
||||
|
||||
Converts literal financial text into normalized numeric values:
|
||||
- "$94.9 billion" → 94_900_000_000.0
|
||||
- "25 basis points" → 0.25 (percentage points)
|
||||
- "$1.52 per share" → 1.52
|
||||
- "4%" → 4.0
|
||||
- "$123.45" → 123.45
|
||||
- "€99" → 99.0
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# Multiplier suffixes for money normalization
|
||||
_MULTIPLIERS: dict[str, float] = {
|
||||
"trillion": 1_000_000_000_000.0,
|
||||
"billion": 1_000_000_000.0,
|
||||
"million": 1_000_000.0,
|
||||
"thousand": 1_000.0,
|
||||
"k": 1_000.0,
|
||||
"m": 1_000_000.0,
|
||||
"b": 1_000_000_000.0,
|
||||
"t": 1_000_000_000_000.0,
|
||||
"bn": 1_000_000_000.0,
|
||||
"mn": 1_000_000.0,
|
||||
"tn": 1_000_000_000_000.0,
|
||||
}
|
||||
|
||||
_NUMBER_RE = re.compile(r"[+-]?\d[\d,]*\.?\d*")
|
||||
|
||||
|
||||
def _extract_number(text: str) -> float | None:
|
||||
"""Extract the first numeric value from text, stripping commas."""
|
||||
match = _NUMBER_RE.search(text)
|
||||
if not match:
|
||||
return None
|
||||
num_str = match.group().replace(",", "")
|
||||
try:
|
||||
return float(num_str)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _find_multiplier(text: str) -> float:
|
||||
"""Find a magnitude multiplier in text (billion, million, etc.)."""
|
||||
lower = text.lower()
|
||||
for suffix, mult in _MULTIPLIERS.items():
|
||||
# Match word boundaries for short suffixes to avoid false positives
|
||||
if len(suffix) <= 2:
|
||||
if re.search(rf"\b{suffix}\b", lower):
|
||||
return mult
|
||||
else:
|
||||
if suffix in lower:
|
||||
return mult
|
||||
return 1.0
|
||||
|
||||
|
||||
def normalize_money(text: str) -> float | None:
|
||||
"""Normalize a money expression to its numeric value.
|
||||
|
||||
Examples:
|
||||
"$94.9 billion" → 94_900_000_000.0
|
||||
"$1.52 per share" → 1.52
|
||||
"€123.45" → 123.45
|
||||
"$1,234" → 1234.0
|
||||
"""
|
||||
number = _extract_number(text)
|
||||
if number is None:
|
||||
return None
|
||||
|
||||
# Check for "per share" — don't apply multiplier
|
||||
lower = text.lower()
|
||||
if "per share" in lower:
|
||||
return number
|
||||
|
||||
multiplier = _find_multiplier(text)
|
||||
return number * multiplier
|
||||
|
||||
|
||||
def normalize_percentage(text: str) -> float | None:
|
||||
"""Normalize a percentage to its numeric value.
|
||||
|
||||
Examples:
|
||||
"4%" → 4.0
|
||||
"-2.5%" → -2.5
|
||||
"+1.2 percent" → 1.2
|
||||
"""
|
||||
return _extract_number(text)
|
||||
|
||||
|
||||
def normalize_basis_points(text: str) -> float | None:
|
||||
"""Normalize basis points to percentage points.
|
||||
|
||||
Examples:
|
||||
"25 basis points" → 0.25
|
||||
"50bps" → 0.50
|
||||
"100 bps" → 1.0
|
||||
"""
|
||||
number = _extract_number(text)
|
||||
if number is None:
|
||||
return None
|
||||
return number / 100.0
|
||||
|
||||
|
||||
def normalize_range(text: str) -> tuple[float | None, float | None]:
|
||||
"""Normalize a range expression to (low, high).
|
||||
|
||||
Examples:
|
||||
"$10-$12" → (10.0, 12.0)
|
||||
"$1.50 to $2.00" → (1.50, 2.00)
|
||||
"""
|
||||
numbers = _NUMBER_RE.findall(text)
|
||||
if len(numbers) < 2:
|
||||
return (None, None)
|
||||
try:
|
||||
low = float(numbers[0].replace(",", ""))
|
||||
high = float(numbers[1].replace(",", ""))
|
||||
return (low, high)
|
||||
except ValueError:
|
||||
return (None, None)
|
||||
|
||||
|
||||
def normalize_value(candidate_type: str, text: str) -> float | None:
|
||||
"""Normalize a candidate value based on its type.
|
||||
|
||||
Returns the normalized numeric value, or None if not applicable.
|
||||
For ranges, returns the midpoint.
|
||||
"""
|
||||
if candidate_type == "money" or candidate_type == "currency":
|
||||
return normalize_money(text)
|
||||
elif candidate_type == "percentage":
|
||||
return normalize_percentage(text)
|
||||
elif candidate_type == "basis_points":
|
||||
return normalize_basis_points(text)
|
||||
elif candidate_type == "eps":
|
||||
return normalize_money(text)
|
||||
elif candidate_type == "revenue":
|
||||
return normalize_money(text)
|
||||
elif candidate_type == "range":
|
||||
low, high = normalize_range(text)
|
||||
if low is not None and high is not None:
|
||||
return (low + high) / 2.0
|
||||
return low
|
||||
else:
|
||||
return None
|
||||
Reference in New Issue
Block a user