"""Unit tests and property tests for the deterministic financial parser. Tests cover: - 23.1: Parse tickers, currencies, money, percentages, basis points, ranges, EPS, revenue, dates, fiscal periods - 23.2: Store literal and normalized representations - 23.3: Link each candidate to exact offsets - 23.4: Property tests for numeric formatting and unit conversions """ from __future__ import annotations import math import pytest from hypothesis import given, settings from hypothesis import strategies as st from services.intelligence_pipeline_v3.parsing import ( CandidateType, FinancialParser, normalize_value, ) from services.intelligence_pipeline_v3.parsing.normalizer import ( normalize_basis_points, normalize_money, normalize_percentage, normalize_range, ) @pytest.fixture def parser() -> FinancialParser: return FinancialParser() # --------------------------------------------------------------------------- # 23.1 — Parse tickers, currencies, money, percentages, basis points, # ranges, EPS, revenue, dates, and fiscal periods # --------------------------------------------------------------------------- class TestTickerParsing: """Test ticker symbol detection.""" def test_dollar_ticker(self, parser: FinancialParser) -> None: """Detect $AAPL style tickers.""" results = parser.parse("Shares of $AAPL rose 3% today.") tickers = [r for r in results if r.candidate_type == CandidateType.TICKER] assert len(tickers) == 1 assert tickers[0].literal_value == "$AAPL" def test_multiple_tickers(self, parser: FinancialParser) -> None: """Detect multiple tickers in one text.""" results = parser.parse("$AAPL and $MSFT both reported earnings.") tickers = [r for r in results if r.candidate_type == CandidateType.TICKER] assert len(tickers) == 2 literals = {t.literal_value for t in tickers} assert "$AAPL" in literals assert "$MSFT" in literals def test_ticker_not_confused_with_currency(self, parser: FinancialParser) -> None: """$AAPL (letters) is a ticker, not currency.""" results = parser.parse("$AAPL hit $200.") tickers = [r for r in results if r.candidate_type == CandidateType.TICKER] currencies = [r for r in results if r.candidate_type == CandidateType.CURRENCY] assert any(t.literal_value == "$AAPL" for t in tickers) assert any(c.literal_value == "$200" for c in currencies) class TestCurrencyParsing: """Test simple currency detection.""" def test_usd_amount(self, parser: FinancialParser) -> None: results = parser.parse("The stock traded at $123.45 today.") currencies = [r for r in results if r.candidate_type == CandidateType.CURRENCY] assert len(currencies) >= 1 assert any(c.literal_value == "$123.45" for c in currencies) def test_euro_amount(self, parser: FinancialParser) -> None: results = parser.parse("Trading at €99 in Frankfurt.") currencies = [r for r in results if r.candidate_type == CandidateType.CURRENCY] assert len(currencies) == 1 assert currencies[0].literal_value == "€99" assert currencies[0].unit == "EUR" def test_gbp_amount(self, parser: FinancialParser) -> None: results = parser.parse("Shares are £1,234.56 in London.") currencies = [r for r in results if r.candidate_type == CandidateType.CURRENCY] assert len(currencies) == 1 assert currencies[0].normalized_value == 1234.56 assert currencies[0].unit == "GBP" class TestMoneyParsing: """Test money amounts with multipliers.""" def test_billion_amount(self, parser: FinancialParser) -> None: results = parser.parse("Apple reported $94.9 billion in revenue.") # Should get a revenue match (more specific than money) revenue = [r for r in results if r.candidate_type == CandidateType.REVENUE] assert len(revenue) >= 1 def test_million_amount(self, parser: FinancialParser) -> None: results = parser.parse("Operating costs were $45 million last quarter.") money = [r for r in results if r.candidate_type == CandidateType.MONEY] assert len(money) >= 1 assert any(m.normalized_value == 45_000_000.0 for m in money) class TestPercentageParsing: """Test percentage detection.""" def test_simple_percentage(self, parser: FinancialParser) -> None: results = parser.parse("The stock rose 4% today.") pcts = [r for r in results if r.candidate_type == CandidateType.PERCENTAGE] assert len(pcts) == 1 assert pcts[0].normalized_value == 4.0 assert pcts[0].unit == "%" def test_negative_percentage(self, parser: FinancialParser) -> None: results = parser.parse("Revenue declined -2.5% year-over-year.") pcts = [r for r in results if r.candidate_type == CandidateType.PERCENTAGE] assert len(pcts) == 1 assert pcts[0].normalized_value == -2.5 def test_percent_word(self, parser: FinancialParser) -> None: results = parser.parse("Margins expanded 1.2 percent this quarter.") pcts = [r for r in results if r.candidate_type == CandidateType.PERCENTAGE] assert len(pcts) == 1 assert pcts[0].normalized_value == 1.2 class TestBasisPointsParsing: """Test basis points detection.""" def test_basis_points_full(self, parser: FinancialParser) -> None: results = parser.parse("The Fed raised rates by 25 basis points.") bps = [r for r in results if r.candidate_type == CandidateType.BASIS_POINTS] assert len(bps) == 1 assert bps[0].normalized_value == pytest.approx(0.25) assert bps[0].unit == "bps" def test_bps_abbreviation(self, parser: FinancialParser) -> None: results = parser.parse("Spreads widened 50bps today.") bps = [r for r in results if r.candidate_type == CandidateType.BASIS_POINTS] assert len(bps) == 1 assert bps[0].normalized_value == pytest.approx(0.50) def test_bps_with_space(self, parser: FinancialParser) -> None: results = parser.parse("Credit spreads tightened 100 bps.") bps = [r for r in results if r.candidate_type == CandidateType.BASIS_POINTS] assert len(bps) == 1 assert bps[0].normalized_value == pytest.approx(1.0) class TestRangeParsing: """Test range detection.""" def test_dollar_range_dash(self, parser: FinancialParser) -> None: results = parser.parse("Guidance was $10-$12 per share.") ranges = [r for r in results if r.candidate_type == CandidateType.RANGE] assert len(ranges) == 1 assert ranges[0].normalized_value == pytest.approx(11.0) def test_dollar_range_to(self, parser: FinancialParser) -> None: results = parser.parse("Expected range $1.50 to $2.00.") ranges = [r for r in results if r.candidate_type == CandidateType.RANGE] assert len(ranges) == 1 assert ranges[0].normalized_value == pytest.approx(1.75) class TestEPSParsing: """Test EPS detection.""" def test_eps_per_share(self, parser: FinancialParser) -> None: results = parser.parse("The company earned $1.52 per share.") eps = [r for r in results if r.candidate_type == CandidateType.EPS] assert len(eps) == 1 assert eps[0].normalized_value == pytest.approx(1.52) assert eps[0].unit == "USD" def test_eps_prefix(self, parser: FinancialParser) -> None: results = parser.parse("EPS of $2.18 beat expectations.") eps = [r for r in results if r.candidate_type == CandidateType.EPS] assert len(eps) == 1 assert eps[0].normalized_value == pytest.approx(2.18) class TestRevenueParsing: """Test revenue figure detection.""" def test_revenue_amount(self, parser: FinancialParser) -> None: results = parser.parse("Apple reported $94.9 billion in revenue.") rev = [r for r in results if r.candidate_type == CandidateType.REVENUE] assert len(rev) == 1 assert rev[0].normalized_value == pytest.approx(94_900_000_000.0) assert rev[0].unit == "USD" def test_revenue_prefix(self, parser: FinancialParser) -> None: results = parser.parse("Revenue reached $50 billion this year.") rev = [r for r in results if r.candidate_type == CandidateType.REVENUE] assert len(rev) == 1 assert rev[0].normalized_value == pytest.approx(50_000_000_000.0) class TestDateParsing: """Test date detection.""" def test_named_month_date(self, parser: FinancialParser) -> None: results = parser.parse("The report was filed on January 15, 2024.") dates = [r for r in results if r.candidate_type == CandidateType.DATE] assert len(dates) == 1 assert "January 15" in dates[0].literal_value def test_abbreviated_month(self, parser: FinancialParser) -> None: results = parser.parse("Earnings released on Oct 28, 2024.") dates = [r for r in results if r.candidate_type == CandidateType.DATE] assert len(dates) == 1 assert "Oct 28" in dates[0].literal_value def test_iso_date(self, parser: FinancialParser) -> None: results = parser.parse("Published on 2024-01-15.") dates = [r for r in results if r.candidate_type == CandidateType.DATE] assert len(dates) == 1 assert dates[0].literal_value == "2024-01-15" class TestFiscalPeriodParsing: """Test fiscal period detection.""" def test_quarter_with_year(self, parser: FinancialParser) -> None: results = parser.parse("Results for Q1 2024 were strong.") periods = [r for r in results if r.candidate_type == CandidateType.FISCAL_PERIOD] assert len(periods) == 1 assert periods[0].period is not None assert periods[0].period.period_type == "quarter" assert periods[0].period.period_value == "Q1" assert periods[0].period.year == 2024 def test_fiscal_year(self, parser: FinancialParser) -> None: results = parser.parse("FY2025 outlook is positive.") periods = [r for r in results if r.candidate_type == CandidateType.FISCAL_PERIOD] assert len(periods) == 1 assert periods[0].period is not None assert periods[0].period.period_type == "fiscal_year" assert periods[0].period.year == 2025 def test_half_year(self, parser: FinancialParser) -> None: results = parser.parse("H1 2024 revenue grew 15%.") periods = [r for r in results if r.candidate_type == CandidateType.FISCAL_PERIOD] assert len(periods) == 1 assert periods[0].period is not None assert periods[0].period.period_type == "half" assert periods[0].period.period_value == "H1" def test_short_year(self, parser: FinancialParser) -> None: results = parser.parse("Q4'24 results beat estimates.") periods = [r for r in results if r.candidate_type == CandidateType.FISCAL_PERIOD] assert len(periods) == 1 assert periods[0].period is not None assert periods[0].period.year == 2024 # --------------------------------------------------------------------------- # 23.2 — Store literal and normalized representations # --------------------------------------------------------------------------- class TestLiteralAndNormalized: """Test that both literal text and normalized numeric values are stored.""" def test_money_stores_both(self, parser: FinancialParser) -> None: text = "$94.9 billion in revenue" results = parser.parse(text) rev = [r for r in results if r.candidate_type == CandidateType.REVENUE] assert len(rev) == 1 # Literal preserved exactly assert rev[0].literal_value == "$94.9 billion in revenue" # Normalized to numeric assert rev[0].normalized_value == pytest.approx(94_900_000_000.0) def test_percentage_stores_both(self, parser: FinancialParser) -> None: text = "Growth was 4% this quarter." results = parser.parse(text) pcts = [r for r in results if r.candidate_type == CandidateType.PERCENTAGE] assert len(pcts) == 1 assert pcts[0].literal_value == "4%" assert pcts[0].normalized_value == 4.0 def test_basis_points_stores_both(self, parser: FinancialParser) -> None: text = "Rates increased 25 basis points." results = parser.parse(text) bps = [r for r in results if r.candidate_type == CandidateType.BASIS_POINTS] assert len(bps) == 1 assert "25 basis points" in bps[0].literal_value assert bps[0].normalized_value == pytest.approx(0.25) def test_eps_stores_both(self, parser: FinancialParser) -> None: text = "EPS was $1.52 per share." results = parser.parse(text) eps = [r for r in results if r.candidate_type == CandidateType.EPS] assert len(eps) == 1 assert "$1.52 per share" in eps[0].literal_value assert eps[0].normalized_value == pytest.approx(1.52) def test_ticker_has_no_normalized_value(self, parser: FinancialParser) -> None: """Tickers don't have numeric values.""" results = parser.parse("$AAPL is up today.") tickers = [r for r in results if r.candidate_type == CandidateType.TICKER] assert len(tickers) == 1 assert tickers[0].normalized_value is None def test_date_has_no_normalized_value(self, parser: FinancialParser) -> None: """Dates don't have numeric values (they have period annotations).""" results = parser.parse("Q1 2024 results are strong.") periods = [r for r in results if r.candidate_type == CandidateType.FISCAL_PERIOD] assert len(periods) == 1 assert periods[0].normalized_value is None assert periods[0].period is not None # --------------------------------------------------------------------------- # 23.3 — Link each candidate to exact offsets # --------------------------------------------------------------------------- class TestExactOffsets: """Test that each candidate has correct start_char and end_char.""" def test_offset_matches_source(self, parser: FinancialParser) -> None: """literal_value must equal text[start_char:end_char].""" text = "Apple earned $1.52 per share in Q1 2024." results = parser.parse(text) for candidate in results: assert text[candidate.start_char:candidate.end_char] == candidate.literal_value, ( f"Offset mismatch for {candidate.candidate_type}: " f"expected '{candidate.literal_value}' but got " f"'{text[candidate.start_char:candidate.end_char]}'" ) def test_offsets_for_multiple_candidates(self, parser: FinancialParser) -> None: """Multiple candidates all have valid offsets.""" text = "$AAPL reported $94.9 billion in revenue, up 15% year-over-year." results = parser.parse(text) assert len(results) >= 3 # ticker, revenue, percentage for candidate in results: assert text[candidate.start_char:candidate.end_char] == candidate.literal_value def test_offsets_no_overlap(self, parser: FinancialParser) -> None: """Candidates should not have overlapping offsets.""" text = "$AAPL rose 3% after reporting $94.9 billion in revenue and EPS of $2.18." results = parser.parse(text) for i in range(len(results)): for j in range(i + 1, len(results)): a = results[i] b = results[j] # No overlap: one must end before the other starts assert a.end_char <= b.start_char or b.end_char <= a.start_char, ( f"Overlap between {a.candidate_type}[{a.start_char}:{a.end_char}] " f"and {b.candidate_type}[{b.start_char}:{b.end_char}]" ) def test_offsets_within_bounds(self, parser: FinancialParser) -> None: """All offsets must be within the text bounds.""" text = "The stock price was $45.67 in Q3 2024." results = parser.parse(text) for candidate in results: assert candidate.start_char >= 0 assert candidate.end_char <= len(text) assert candidate.start_char < candidate.end_char def test_empty_text_no_candidates(self, parser: FinancialParser) -> None: """Empty text produces no candidates.""" assert parser.parse("") == [] assert parser.parse(" \n\t ") == [] # --------------------------------------------------------------------------- # 23.4 — Property tests for numeric formatting and unit conversions # --------------------------------------------------------------------------- class TestPropertyBasedFinancialParser: """Property-based tests for financial parsing. **Validates: Requirements 5.1, 5.7, 5.8** """ @given( amount=st.floats(min_value=0.01, max_value=999.99, allow_nan=False, allow_infinity=False), ) @settings(max_examples=100) def test_currency_strings_parse_to_expected_value(self, amount: float) -> None: """Property: Generated currency strings parse to their expected value. **Validates: Requirements 5.1, 5.7** """ # Round to 2 decimal places for realistic currency amount = round(amount, 2) text = f"The price was ${amount:.2f} per unit." parser = FinancialParser() results = parser.parse(text) # Should find at least one currency or EPS match numeric_candidates = [ r for r in results if r.candidate_type in (CandidateType.CURRENCY, CandidateType.EPS) and r.normalized_value is not None ] assert len(numeric_candidates) >= 1 assert any( abs(c.normalized_value - amount) < 0.01 for c in numeric_candidates ), f"No candidate matched expected value {amount}" @given( pct=st.floats(min_value=-99.9, max_value=99.9, allow_nan=False, allow_infinity=False), ) @settings(max_examples=100) def test_percentage_strings_parse_to_expected_value(self, pct: float) -> None: """Property: Generated percentage strings parse to their expected value. **Validates: Requirements 5.1, 5.7** """ pct = round(pct, 1) if pct == 0.0: pct = 1.0 # Avoid edge case with sign sign = "+" if pct > 0 else "" text = f"Revenue changed {sign}{pct}% this quarter." parser = FinancialParser() results = parser.parse(text) pct_candidates = [ r for r in results if r.candidate_type == CandidateType.PERCENTAGE and r.normalized_value is not None ] assert len(pct_candidates) >= 1 assert any( abs(c.normalized_value - pct) < 0.1 for c in pct_candidates ), f"No candidate matched expected percentage {pct}" @given( bps=st.integers(min_value=1, max_value=500), ) @settings(max_examples=100) def test_basis_points_normalize_to_percentage(self, bps: int) -> None: """Property: N basis points always normalizes to N/100 percentage points. **Validates: Requirements 5.7, 5.8** """ text = f"Rates moved {bps} basis points today." parser = FinancialParser() results = parser.parse(text) bps_candidates = [ r for r in results if r.candidate_type == CandidateType.BASIS_POINTS and r.normalized_value is not None ] assert len(bps_candidates) >= 1 expected = bps / 100.0 assert any( abs(c.normalized_value - expected) < 0.001 for c in bps_candidates ), f"Expected {expected} but got {[c.normalized_value for c in bps_candidates]}" @given( text=st.text(min_size=1, max_size=5000, alphabet=st.characters( categories=("L", "N", "P", "Z", "S"), include_characters="$€£¥%.,+-/0123456789", )), ) @settings(max_examples=100) def test_normalized_values_are_always_finite(self, text: str) -> None: """Property: Normalized values are always finite floats (no inf, no NaN). **Validates: Requirements 5.7, 5.8** """ parser = FinancialParser() results = parser.parse(text) for candidate in results: if candidate.normalized_value is not None: assert math.isfinite(candidate.normalized_value), ( f"Non-finite value {candidate.normalized_value} for " f"{candidate.candidate_type}: '{candidate.literal_value}'" ) @given( text=st.text(min_size=1, max_size=5000, alphabet=st.characters( categories=("L", "N", "P", "Z", "S"), include_characters="$€£¥%.,+-/0123456789 \n", )), ) @settings(max_examples=100) def test_offsets_always_map_to_literal(self, text: str) -> None: """Property: For any text, candidate offsets always map to the literal value. **Validates: Requirements 5.1, 5.8** """ parser = FinancialParser() results = parser.parse(text) for candidate in results: extracted = text[candidate.start_char:candidate.end_char] assert extracted == candidate.literal_value, ( f"Offset mismatch: [{candidate.start_char}:{candidate.end_char}] = " f"'{extracted}' != literal '{candidate.literal_value}'" ) @given( text=st.text(min_size=1, max_size=5000, alphabet=st.characters( categories=("L", "N", "P", "Z", "S"), include_characters="$€£¥%.,+-/0123456789 \n", )), ) @settings(max_examples=100) def test_no_overlapping_candidates(self, text: str) -> None: """Property: No two candidates have overlapping offsets. **Validates: Requirements 5.1** """ parser = FinancialParser() results = parser.parse(text) for i in range(len(results)): for j in range(i + 1, len(results)): a = results[i] b = results[j] assert a.end_char <= b.start_char or b.end_char <= a.start_char, ( f"Overlap: {a.candidate_type}[{a.start_char}:{a.end_char}] " f"vs {b.candidate_type}[{b.start_char}:{b.end_char}]" ) # --------------------------------------------------------------------------- # Normalizer unit tests # --------------------------------------------------------------------------- class TestNormalizerFunctions: """Test normalizer helper functions directly.""" def test_normalize_money_billion(self) -> None: assert normalize_money("$94.9 billion") == pytest.approx(94_900_000_000.0) def test_normalize_money_million(self) -> None: assert normalize_money("$45 million") == pytest.approx(45_000_000.0) def test_normalize_money_per_share(self) -> None: assert normalize_money("$1.52 per share") == pytest.approx(1.52) def test_normalize_money_simple(self) -> None: assert normalize_money("$123.45") == pytest.approx(123.45) def test_normalize_money_with_commas(self) -> None: assert normalize_money("$1,234.56") == pytest.approx(1234.56) def test_normalize_percentage(self) -> None: assert normalize_percentage("4%") == pytest.approx(4.0) assert normalize_percentage("-2.5%") == pytest.approx(-2.5) assert normalize_percentage("+1.2 percent") == pytest.approx(1.2) def test_normalize_basis_points(self) -> None: assert normalize_basis_points("25 basis points") == pytest.approx(0.25) assert normalize_basis_points("50bps") == pytest.approx(0.50) assert normalize_basis_points("100 bps") == pytest.approx(1.0) def test_normalize_range(self) -> None: low, high = normalize_range("$10-$12") assert low == pytest.approx(10.0) assert high == pytest.approx(12.0) def test_normalize_range_to(self) -> None: low, high = normalize_range("$1.50 to $2.00") assert low == pytest.approx(1.50) assert high == pytest.approx(2.00) def test_normalize_value_dispatches(self) -> None: assert normalize_value("money", "$94.9 billion") == pytest.approx(94_900_000_000.0) assert normalize_value("percentage", "4%") == pytest.approx(4.0) assert normalize_value("basis_points", "25 basis points") == pytest.approx(0.25) assert normalize_value("eps", "$1.52 per share") == pytest.approx(1.52) assert normalize_value("ticker", "$AAPL") is None