417 lines
16 KiB
Python
417 lines
16 KiB
Python
"""Property-based tests for EvidenceUnit normalization, calibrated reliability, and LLR.
|
||
|
||
Feature: math-core-v3-engine
|
||
|
||
Uses Hypothesis to validate correctness properties of the v3 calibrated
|
||
evidence engine foundation: EvidenceUnit normalization preserves field ranges,
|
||
reliability q_i is bounded, p_correct is bounded, LLR sign matches direction,
|
||
and neutral signals produce zero LLR.
|
||
|
||
Validates: Requirements 1.1–1.8, 2.1–2.9, 3.1–3.6, 21.1–21.3
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import math
|
||
from datetime import datetime, timedelta, timezone
|
||
|
||
from hypothesis import given, settings
|
||
from hypothesis import strategies as st
|
||
|
||
from services.aggregation.scoring import (
|
||
EvidenceUnit,
|
||
SourceStats,
|
||
_clamp,
|
||
compute_llr,
|
||
compute_v3_reliability,
|
||
normalize_company_signal,
|
||
normalize_competitive_signal,
|
||
normalize_macro_signal,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Hypothesis strategies
|
||
# ---------------------------------------------------------------------------
|
||
|
||
unit_floats = st.floats(min_value=0.0, max_value=1.0, allow_nan=False, allow_infinity=False)
|
||
directions = st.sampled_from([-1, 0, 1])
|
||
horizons = st.sampled_from(["intraday", "1d", "7d", "30d", "90d"])
|
||
positive_floats = st.floats(min_value=0.0, max_value=10000.0, allow_nan=False, allow_infinity=False)
|
||
non_negative_ints = st.integers(min_value=0, max_value=100)
|
||
|
||
|
||
def _evidence_unit_strategy() -> st.SearchStrategy[EvidenceUnit]:
|
||
"""Generate valid EvidenceUnit instances with fields in valid ranges."""
|
||
return st.builds(
|
||
EvidenceUnit,
|
||
symbol=st.just("AAPL"),
|
||
layer=st.sampled_from(["company", "macro", "competitive"]),
|
||
event_type=st.sampled_from(["earnings", "product_launch", "regulatory", "unknown"]),
|
||
source_id=st.just("src-001"),
|
||
source_group=st.sampled_from(["company", "macro", "competitive"]),
|
||
timestamp=st.just(datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc)),
|
||
horizon=horizons,
|
||
direction=directions,
|
||
sentiment_strength=unit_floats,
|
||
impact=unit_floats,
|
||
extraction_conf=unit_floats,
|
||
source_cred=unit_floats,
|
||
novelty=unit_floats,
|
||
event_base_rate=st.floats(min_value=0.01, max_value=1.0, allow_nan=False, allow_infinity=False),
|
||
cluster_id=st.just("cluster-001"),
|
||
)
|
||
|
||
|
||
def _source_stats_strategy() -> st.SearchStrategy[SourceStats]:
|
||
"""Generate SourceStats with valid counts."""
|
||
return st.builds(
|
||
SourceStats,
|
||
source_id=st.just("src-001"),
|
||
hits=non_negative_ints,
|
||
misses=non_negative_ints,
|
||
alpha_0=st.floats(min_value=1.0, max_value=10.0, allow_nan=False, allow_infinity=False),
|
||
beta_0=st.floats(min_value=1.0, max_value=10.0, allow_nan=False, allow_infinity=False),
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Property 1: Reliability q_i is bounded in [0, 1]
|
||
# Feature: math-core-v3-engine, Property 1: Reliability q_i is bounded in [0, 1]
|
||
# Validates: Requirements 2.1–2.9
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@settings(max_examples=100)
|
||
@given(
|
||
extraction_conf=unit_floats,
|
||
source_cred=unit_floats,
|
||
novelty=unit_floats,
|
||
impact=unit_floats,
|
||
sentiment_strength=unit_floats,
|
||
event_base_rate=st.floats(min_value=0.01, max_value=1.0, allow_nan=False, allow_infinity=False),
|
||
age_hours=st.floats(min_value=0.0, max_value=5000.0, allow_nan=False, allow_infinity=False),
|
||
duplicate_count_before=non_negative_ints,
|
||
hits=non_negative_ints,
|
||
misses=non_negative_ints,
|
||
)
|
||
def test_property_1_reliability_qi_bounded(
|
||
extraction_conf: float,
|
||
source_cred: float,
|
||
novelty: float,
|
||
impact: float,
|
||
sentiment_strength: float,
|
||
event_base_rate: float,
|
||
age_hours: float,
|
||
duplicate_count_before: int,
|
||
hits: int,
|
||
misses: int,
|
||
) -> None:
|
||
"""Property 1: Reliability q_i is bounded in [0, 1].
|
||
|
||
For any valid EvidenceUnit with extraction_conf in [0,1], source_cred in [0,1],
|
||
novelty in [0,1], any non-negative age_hours, and any non-negative
|
||
duplicate_count_before, the computed q_i SHALL be in [0.0, 1.0].
|
||
|
||
**Validates: Requirements 2.1–2.9**
|
||
"""
|
||
reference_time = datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc)
|
||
unit_time = reference_time - timedelta(hours=age_hours)
|
||
|
||
unit = EvidenceUnit(
|
||
symbol="AAPL",
|
||
layer="company",
|
||
event_type="earnings",
|
||
source_id="src-001",
|
||
source_group="company",
|
||
timestamp=unit_time,
|
||
horizon="7d",
|
||
direction=1,
|
||
sentiment_strength=sentiment_strength,
|
||
impact=impact,
|
||
extraction_conf=extraction_conf,
|
||
source_cred=source_cred,
|
||
novelty=novelty,
|
||
event_base_rate=event_base_rate,
|
||
cluster_id="cluster-001",
|
||
)
|
||
|
||
stats = SourceStats(source_id="src-001", hits=hits, misses=misses)
|
||
result = compute_v3_reliability(unit, stats, duplicate_count_before, reference_time)
|
||
|
||
assert 0.0 <= result.q_i <= 1.0, f"q_i={result.q_i} out of [0, 1]"
|
||
assert 0.0 <= result.q_ext <= 1.0, f"q_ext={result.q_ext} out of [0, 1]"
|
||
assert 0.0 <= result.q_source <= 1.0, f"q_source={result.q_source} out of [0, 1]"
|
||
assert 0.0 <= result.q_recency <= 1.0, f"q_recency={result.q_recency} out of [0, 1]"
|
||
assert 0.0 <= result.q_uniqueness <= 1.0, f"q_uniqueness={result.q_uniqueness} out of [0, 1]"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Property 2: p_correct is bounded in [0.501, 0.85]
|
||
# Feature: math-core-v3-engine, Property 2: p_correct is bounded in [0.501, 0.85]
|
||
# Validates: Requirements 3.1–3.2
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@settings(max_examples=100)
|
||
@given(
|
||
q_i=unit_floats,
|
||
impact=unit_floats,
|
||
sentiment_strength=unit_floats,
|
||
)
|
||
def test_property_2_p_correct_bounded(
|
||
q_i: float,
|
||
impact: float,
|
||
sentiment_strength: float,
|
||
) -> None:
|
||
"""Property 2: p_correct is bounded in [0.501, 0.85].
|
||
|
||
For any valid q_i in [0,1], impact in [0,1], and sentiment_strength in [0,1],
|
||
the computed p_correct SHALL be in [0.501, 0.85].
|
||
|
||
**Validates: Requirements 3.1–3.2**
|
||
"""
|
||
# Replicate the p_correct computation from compute_llr
|
||
p_correct = _clamp(
|
||
0.50 + 0.35 * q_i * impact * sentiment_strength,
|
||
0.501,
|
||
0.85,
|
||
)
|
||
|
||
assert 0.501 <= p_correct <= 0.85, f"p_correct={p_correct} out of [0.501, 0.85]"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Property 3: LLR sign matches direction and magnitude is bounded
|
||
# Feature: math-core-v3-engine, Property 3: LLR sign matches direction and magnitude is bounded
|
||
# Validates: Requirements 3.3–3.4, 3.6
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@settings(max_examples=100)
|
||
@given(
|
||
direction=st.sampled_from([-1, 1]),
|
||
q_i=unit_floats,
|
||
impact=unit_floats,
|
||
sentiment_strength=unit_floats,
|
||
)
|
||
def test_property_3_llr_sign_matches_direction(
|
||
direction: int,
|
||
q_i: float,
|
||
impact: float,
|
||
sentiment_strength: float,
|
||
) -> None:
|
||
"""Property 3: LLR sign matches direction and magnitude is bounded.
|
||
|
||
For any valid signal with direction in {-1, +1}, the LLR SHALL have the same
|
||
sign as direction, with abs magnitude in [~0.004, ~1.735].
|
||
|
||
**Validates: Requirements 3.3–3.4, 3.6**
|
||
"""
|
||
unit = EvidenceUnit(
|
||
symbol="AAPL",
|
||
layer="company",
|
||
event_type="earnings",
|
||
source_id="src-001",
|
||
source_group="company",
|
||
timestamp=datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc),
|
||
horizon="7d",
|
||
direction=direction,
|
||
sentiment_strength=sentiment_strength,
|
||
impact=impact,
|
||
extraction_conf=0.8,
|
||
source_cred=0.8,
|
||
novelty=0.8,
|
||
event_base_rate=0.25,
|
||
cluster_id="cluster-001",
|
||
)
|
||
|
||
llr = compute_llr(unit, q_i)
|
||
|
||
# LLR sign must match direction
|
||
if direction == 1:
|
||
assert llr > 0.0, f"LLR={llr} should be positive for direction=+1"
|
||
else:
|
||
assert llr < 0.0, f"LLR={llr} should be negative for direction=-1"
|
||
|
||
# Magnitude bounds: ln(0.501/0.499) ≈ 0.004, ln(0.85/0.15) ≈ 1.735
|
||
min_magnitude = math.log(0.501 / 0.499) # ~0.004
|
||
max_magnitude = math.log(0.85 / 0.15) # ~1.735
|
||
|
||
assert abs(llr) >= min_magnitude - 1e-9, f"|LLR|={abs(llr)} below min ~0.004"
|
||
assert abs(llr) <= max_magnitude + 1e-9, f"|LLR|={abs(llr)} above max ~1.735"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Property 4: Neutral signals produce zero LLR
|
||
# Feature: math-core-v3-engine, Property 4: Neutral signals produce zero LLR
|
||
# Validates: Requirements 3.5
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@settings(max_examples=100)
|
||
@given(
|
||
q_i=unit_floats,
|
||
impact=unit_floats,
|
||
sentiment_strength=unit_floats,
|
||
extraction_conf=unit_floats,
|
||
source_cred=unit_floats,
|
||
novelty=unit_floats,
|
||
)
|
||
def test_property_4_neutral_produces_zero_llr(
|
||
q_i: float,
|
||
impact: float,
|
||
sentiment_strength: float,
|
||
extraction_conf: float,
|
||
source_cred: float,
|
||
novelty: float,
|
||
) -> None:
|
||
"""Property 4: Neutral signals produce zero LLR.
|
||
|
||
For any EvidenceUnit with direction=0, LLR SHALL be exactly 0.0.
|
||
|
||
**Validates: Requirements 3.5**
|
||
"""
|
||
unit = EvidenceUnit(
|
||
symbol="AAPL",
|
||
layer="company",
|
||
event_type="earnings",
|
||
source_id="src-001",
|
||
source_group="company",
|
||
timestamp=datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc),
|
||
horizon="7d",
|
||
direction=0,
|
||
sentiment_strength=sentiment_strength,
|
||
impact=impact,
|
||
extraction_conf=extraction_conf,
|
||
source_cred=source_cred,
|
||
novelty=novelty,
|
||
event_base_rate=0.25,
|
||
cluster_id="cluster-001",
|
||
)
|
||
|
||
llr = compute_llr(unit, q_i)
|
||
assert llr == 0.0, f"LLR={llr} should be exactly 0.0 for neutral direction"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Property 21: EvidenceUnit normalization preserves field ranges
|
||
# Feature: math-core-v3-engine, Property 21: EvidenceUnit normalization preserves field ranges
|
||
# Validates: Requirements 1.1–1.8, 21.1–21.3
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _company_signal_strategy() -> st.SearchStrategy[dict]:
|
||
"""Generate raw company signal dicts with arbitrary values."""
|
||
return st.fixed_dictionaries({
|
||
"symbol": st.just("TSLA"),
|
||
"timestamp": st.just(datetime(2024, 6, 15, 10, 0, 0, tzinfo=timezone.utc)),
|
||
"source_id": st.just("doc-123"),
|
||
"event_type": st.sampled_from(["earnings", "product_launch", "regulatory", "unknown", None]),
|
||
"source_group": st.sampled_from(["company", "reuters", None]),
|
||
"horizon": st.sampled_from(["intraday", "1d", "7d", "30d", "90d", "invalid", None]),
|
||
"sentiment": st.sampled_from(["positive", "negative", "neutral", "mixed", "bullish", "bearish", None]),
|
||
"sentiment_strength": st.one_of(unit_floats, st.just(None)),
|
||
"impact": st.one_of(unit_floats, st.just(None)),
|
||
"extraction_conf": st.one_of(unit_floats, st.just(None)),
|
||
"source_cred": st.one_of(unit_floats, st.just(None)),
|
||
"novelty": st.one_of(unit_floats, st.just(None)),
|
||
})
|
||
|
||
|
||
def _macro_signal_strategy() -> st.SearchStrategy[dict]:
|
||
"""Generate raw macro signal dicts with arbitrary values."""
|
||
return st.fixed_dictionaries({
|
||
"symbol": st.just("AAPL"),
|
||
"timestamp": st.just(datetime(2024, 6, 15, 10, 0, 0, tzinfo=timezone.utc)),
|
||
"source_id": st.just("event-456"),
|
||
"event_type": st.sampled_from(["earnings", "regulatory", "market_data", None]),
|
||
"estimated_duration": st.sampled_from(["short_term", "medium_term", "long_term", "unknown", None]),
|
||
"impact_direction": st.sampled_from(["positive", "negative", "neutral", "bullish", "bearish", None]),
|
||
"macro_impact_score": st.one_of(unit_floats, st.just(None)),
|
||
"event_confidence": st.one_of(unit_floats, st.just(None)),
|
||
"novelty": st.one_of(unit_floats, st.just(None)),
|
||
"sentiment_strength": st.one_of(unit_floats, st.just(None)),
|
||
})
|
||
|
||
|
||
def _competitive_signal_strategy() -> st.SearchStrategy[dict]:
|
||
"""Generate raw competitive signal dicts with arbitrary values."""
|
||
return st.fixed_dictionaries({
|
||
"symbol": st.just("MSFT"),
|
||
"timestamp": st.just(datetime(2024, 6, 15, 10, 0, 0, tzinfo=timezone.utc)),
|
||
"source_id": st.just("comp-789"),
|
||
"event_type": st.sampled_from(["earnings", "product_launch", None]),
|
||
"signal_direction": st.sampled_from(["bullish", "bearish", "neutral", None]),
|
||
"signal_strength": st.one_of(unit_floats, st.just(None)),
|
||
"relationship_strength": st.one_of(unit_floats, st.just(None)),
|
||
"pattern_confidence": st.one_of(unit_floats, st.just(None)),
|
||
"time_horizon": st.sampled_from(["intraday", "1d", "7d", "30d", "90d", "short_term", None]),
|
||
"novelty": st.one_of(unit_floats, st.just(None)),
|
||
"sentiment_strength": st.one_of(unit_floats, st.just(None)),
|
||
})
|
||
|
||
|
||
@settings(max_examples=100)
|
||
@given(signal=_company_signal_strategy())
|
||
def test_property_21_company_normalization_preserves_ranges(signal: dict) -> None:
|
||
"""Property 21: EvidenceUnit normalization preserves field ranges (company).
|
||
|
||
For any valid company signal input, normalized EvidenceUnit SHALL have
|
||
direction in {-1,0,+1}, all float fields in [0,1], event_base_rate in (0,1].
|
||
|
||
**Validates: Requirements 1.1–1.8, 21.1–21.3**
|
||
"""
|
||
unit = normalize_company_signal(signal)
|
||
assert unit is not None, "Expected valid EvidenceUnit from company signal"
|
||
|
||
_assert_evidence_unit_ranges(unit)
|
||
|
||
|
||
@settings(max_examples=100)
|
||
@given(signal=_macro_signal_strategy())
|
||
def test_property_21_macro_normalization_preserves_ranges(signal: dict) -> None:
|
||
"""Property 21: EvidenceUnit normalization preserves field ranges (macro).
|
||
|
||
For any valid macro signal input, normalized EvidenceUnit SHALL have
|
||
direction in {-1,0,+1}, all float fields in [0,1], event_base_rate in (0,1].
|
||
|
||
**Validates: Requirements 1.1–1.8, 21.1–21.3**
|
||
"""
|
||
unit = normalize_macro_signal(signal)
|
||
assert unit is not None, "Expected valid EvidenceUnit from macro signal"
|
||
|
||
_assert_evidence_unit_ranges(unit)
|
||
|
||
|
||
@settings(max_examples=100)
|
||
@given(signal=_competitive_signal_strategy())
|
||
def test_property_21_competitive_normalization_preserves_ranges(signal: dict) -> None:
|
||
"""Property 21: EvidenceUnit normalization preserves field ranges (competitive).
|
||
|
||
For any valid competitive signal input, normalized EvidenceUnit SHALL have
|
||
direction in {-1,0,+1}, all float fields in [0,1], event_base_rate in (0,1].
|
||
|
||
**Validates: Requirements 1.1–1.8, 21.1–21.3**
|
||
"""
|
||
unit = normalize_competitive_signal(signal)
|
||
assert unit is not None, "Expected valid EvidenceUnit from competitive signal"
|
||
|
||
_assert_evidence_unit_ranges(unit)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Shared assertion helper
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _assert_evidence_unit_ranges(unit: EvidenceUnit) -> None:
|
||
"""Assert all EvidenceUnit fields are within their valid ranges."""
|
||
assert unit.direction in (-1, 0, 1), f"direction={unit.direction} not in {{-1, 0, +1}}"
|
||
assert 0.0 <= unit.sentiment_strength <= 1.0, f"sentiment_strength={unit.sentiment_strength} out of [0, 1]"
|
||
assert 0.0 <= unit.impact <= 1.0, f"impact={unit.impact} out of [0, 1]"
|
||
assert 0.0 <= unit.extraction_conf <= 1.0, f"extraction_conf={unit.extraction_conf} out of [0, 1]"
|
||
assert 0.0 <= unit.source_cred <= 1.0, f"source_cred={unit.source_cred} out of [0, 1]"
|
||
assert 0.0 <= unit.novelty <= 1.0, f"novelty={unit.novelty} out of [0, 1]"
|
||
assert 0.0 < unit.event_base_rate <= 1.0, f"event_base_rate={unit.event_base_rate} out of (0, 1]"
|
||
assert unit.horizon in ("intraday", "1d", "7d", "30d", "90d"), f"horizon={unit.horizon} invalid"
|
||
assert unit.layer in ("company", "macro", "competitive"), f"layer={unit.layer} invalid"
|