Files
stonks-oracle/tests/test_pbt_v3_posterior.py

611 lines
21 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Property-based tests for v3 posterior assembly and regime classification.
Feature: math-core-v3-engine
Uses Hypothesis to validate correctness properties of the v3 posterior
and regime detection layer.
Validates: Requirements 5.3, 6.26.5, 21.5
"""
from __future__ import annotations
import math
from hypothesis import given, settings
from hypothesis import strategies as st
from services.aggregation.bayesian import V3Posterior, compute_v3_posterior
from services.aggregation.regime import (
MarketRegime,
V3RegimeClassification,
classify_regime_v3,
)
from services.aggregation.scoring import EvidenceUnit
from services.aggregation.worker import EvidenceCluster
# ---------------------------------------------------------------------------
# Hypothesis strategies
# ---------------------------------------------------------------------------
# Cluster LLR values in [-2.5, 2.5] as per spec
cluster_llrs = st.floats(
min_value=-2.5, max_value=2.5, allow_nan=False, allow_infinity=False
)
# Lists of cluster LLR values (0 to 10 clusters)
cluster_llr_lists = st.lists(cluster_llrs, min_size=0, max_size=10)
# Regime evidence multiplier gamma from the defined set
gammas = st.sampled_from([0.70, 0.80, 0.90, 1.10])
# Prior probability in [0.40, 0.60]
priors = st.floats(
min_value=0.40, max_value=0.60, allow_nan=False, allow_infinity=False
)
# Random walk closing prices for regime classification
# Generate 120+ prices as a random walk starting from a base price
_base_price = st.floats(min_value=10.0, max_value=500.0, allow_nan=False, allow_infinity=False)
_price_step = st.floats(min_value=-2.0, max_value=2.0, allow_nan=False, allow_infinity=False)
@st.composite
def closing_prices_strategy(draw: st.DrawFn) -> list[float]:
"""Generate 120+ closing prices as a random walk."""
base = draw(_base_price)
n = draw(st.integers(min_value=120, max_value=200))
steps = draw(st.lists(_price_step, min_size=n, max_size=n))
prices = [base]
for step in steps:
prices.append(max(0.01, prices[-1] + step)) # keep prices positive
return prices
@st.composite
def daily_returns_strategy(draw: st.DrawFn) -> list[float]:
"""Generate 120+ daily returns (small floats)."""
n = draw(st.integers(min_value=120, max_value=200))
returns = draw(
st.lists(
st.floats(min_value=-0.10, max_value=0.10, allow_nan=False, allow_infinity=False),
min_size=n,
max_size=n,
)
)
return returns
# ATR_20 > 0
atr_20_strategy = st.floats(
min_value=0.01, max_value=50.0, allow_nan=False, allow_infinity=False
)
# ---------------------------------------------------------------------------
# Helper: build mock EvidenceCluster from a cluster_llr value
# ---------------------------------------------------------------------------
def _make_cluster(cluster_llr: float) -> EvidenceCluster:
"""Create a minimal EvidenceCluster with the given cluster_llr."""
return EvidenceCluster(
cluster_id=f"test_cluster_{id(cluster_llr)}",
units=[],
llrs=[cluster_llr],
n_eff=1.0,
cluster_llr=cluster_llr,
)
def _make_regime(gamma: float) -> V3RegimeClassification:
"""Create a minimal V3RegimeClassification with the given gamma."""
return V3RegimeClassification(
regime=MarketRegime.UNCERTAINTY,
trend_z=0.0,
vol_ratio=1.0,
evidence_multiplier=gamma,
confidence_multiplier=0.85,
phi_decay=0.50,
atr_multiplier=2.0,
)
# ---------------------------------------------------------------------------
# Property 7: Posterior P_up is in open interval (0, 1)
# ---------------------------------------------------------------------------
@given(
llr_values=cluster_llr_lists,
gamma=gammas,
p_prior=priors,
)
@settings(max_examples=100)
def test_property_7_posterior_p_up_in_open_interval(
llr_values: list[float],
gamma: float,
p_prior: float,
) -> None:
"""**Validates: Requirements 5.3**
Property 7: For any set of cluster LLRs (each in [-2.5, 2.5]), any regime
evidence multiplier gamma in {0.70, 0.80, 0.90, 1.10}, and any prior
P_prior in [0.40, 0.60], the computed P_up SHALL be in (1e-10, 1 - 1e-10).
"""
# Build mock clusters from the LLR values
clusters = [_make_cluster(llr) for llr in llr_values]
# Build mock regime with the given gamma
regime = _make_regime(gamma)
# Compute posterior
result = compute_v3_posterior(clusters, regime, p_prior=p_prior)
# P_up must be in the open interval (1e-10, 1 - 1e-10)
assert result.p_up >= 1e-10, (
f"P_up {result.p_up} is below lower bound 1e-10"
)
assert result.p_up <= 1 - 1e-10, (
f"P_up {result.p_up} is above upper bound 1 - 1e-10"
)
# p_down should be 1 - p_up
assert math.isclose(result.p_up + result.p_down, 1.0, abs_tol=1e-12), (
f"p_up + p_down = {result.p_up + result.p_down}, expected 1.0"
)
# Strength should be abs(2 * P_up - 1)
expected_strength = abs(2.0 * result.p_up - 1.0)
assert math.isclose(result.strength, expected_strength, abs_tol=1e-12), (
f"strength {result.strength} != expected {expected_strength}"
)
# ---------------------------------------------------------------------------
# Property 20: Regime classification is exhaustive and deterministic
# ---------------------------------------------------------------------------
_VALID_REGIMES = {"panic", "trend_following", "mean_reversion", "uncertainty"}
@given(
closing_prices=closing_prices_strategy(),
daily_returns=daily_returns_strategy(),
atr_20=atr_20_strategy,
)
@settings(max_examples=100)
def test_property_20_regime_classification_exhaustive_and_deterministic(
closing_prices: list[float],
daily_returns: list[float],
atr_20: float,
) -> None:
"""**Validates: Requirements 6.26.5, 21.5**
Property 20: For any valid market data inputs (closing_prices of sufficient
length, daily_returns, ATR_20 > 0), the regime classification SHALL produce
exactly one of {panic, trend_following, mean_reversion, uncertainty} and the
same inputs SHALL always produce the same classification.
"""
# First classification
result_1 = classify_regime_v3(closing_prices, daily_returns, atr_20)
# Result must be exactly one of the four valid regimes
assert result_1.regime.value in _VALID_REGIMES, (
f"Regime '{result_1.regime.value}' not in valid set {_VALID_REGIMES}"
)
# Determinism: second call with same inputs must produce same result
result_2 = classify_regime_v3(closing_prices, daily_returns, atr_20)
assert result_1.regime == result_2.regime, (
f"Non-deterministic: first call gave {result_1.regime.value}, "
f"second gave {result_2.regime.value}"
)
assert result_1.trend_z == result_2.trend_z, (
f"Non-deterministic trend_z: {result_1.trend_z} vs {result_2.trend_z}"
)
assert result_1.vol_ratio == result_2.vol_ratio, (
f"Non-deterministic vol_ratio: {result_1.vol_ratio} vs {result_2.vol_ratio}"
)
assert result_1.evidence_multiplier == result_2.evidence_multiplier, (
f"Non-deterministic evidence_multiplier: "
f"{result_1.evidence_multiplier} vs {result_2.evidence_multiplier}"
)
# ---------------------------------------------------------------------------
# Property 8: Contradiction is zero when evidence is unidirectional
# Feature: math-core-v3-engine, Property 8: Contradiction is zero when evidence is unidirectional
# ---------------------------------------------------------------------------
from services.aggregation.contradiction import compute_v3_contradiction
# Strategies for unidirectional cluster LLRs
all_positive_llrs = st.lists(
st.floats(min_value=0.001, max_value=2.5, allow_nan=False, allow_infinity=False),
min_size=1, max_size=10,
)
all_negative_llrs = st.lists(
st.floats(min_value=-2.5, max_value=-0.001, allow_nan=False, allow_infinity=False),
min_size=1, max_size=10,
)
@given(llr_values=all_positive_llrs)
@settings(max_examples=100)
def test_property_8_contradiction_zero_all_positive(
llr_values: list[float],
) -> None:
"""**Validates: Requirements 7.6, 7.7**
Property 8: For any set of cluster LLRs where all clusters have the same
sign (all positive), the computed contradiction score SHALL be 0.0.
"""
clusters = [_make_cluster(llr) for llr in llr_values]
score = compute_v3_contradiction(clusters)
assert score == 0.0, (
f"Expected contradiction 0.0 for all-positive LLRs {llr_values}, got {score}"
)
@given(llr_values=all_negative_llrs)
@settings(max_examples=100)
def test_property_8_contradiction_zero_all_negative(
llr_values: list[float],
) -> None:
"""**Validates: Requirements 7.6, 7.7**
Property 8: For any set of cluster LLRs where all clusters have the same
sign (all negative), the computed contradiction score SHALL be 0.0.
"""
clusters = [_make_cluster(llr) for llr in llr_values]
score = compute_v3_contradiction(clusters)
assert score == 0.0, (
f"Expected contradiction 0.0 for all-negative LLRs {llr_values}, got {score}"
)
# ---------------------------------------------------------------------------
# Property 9: Contradiction score is bounded in [0, 1]
# Feature: math-core-v3-engine, Property 9: Contradiction score is bounded in [0, 1]
# ---------------------------------------------------------------------------
# Mix of positive and negative cluster LLRs
mixed_llrs = st.lists(
st.floats(min_value=-2.5, max_value=2.5, allow_nan=False, allow_infinity=False),
min_size=0, max_size=15,
)
@given(llr_values=mixed_llrs)
@settings(max_examples=100)
def test_property_9_contradiction_bounded_zero_one(
llr_values: list[float],
) -> None:
"""**Validates: Requirements 7.6, 7.7, 21.7**
Property 9: For any set of cluster LLRs (including mixed positive and
negative), the computed contradiction score SHALL be in [0.0, 1.0].
"""
clusters = [_make_cluster(llr) for llr in llr_values]
score = compute_v3_contradiction(clusters)
assert 0.0 <= score <= 1.0, (
f"Contradiction score {score} out of bounds [0, 1] for LLRs {llr_values}"
)
# ---------------------------------------------------------------------------
# Property 10: Multiplicative confidence is bounded in [0, 1] and suppressed
# by weak dimensions
# Feature: math-core-v3-engine
# ---------------------------------------------------------------------------
from services.aggregation.worker import compute_v3_confidence, compute_v3_data_quality
# Strategies for Property 10
n_eff_totals = st.floats(min_value=0.0, max_value=20.0, allow_nan=False, allow_infinity=False)
unit_floats = st.floats(min_value=0.0, max_value=1.0, allow_nan=False, allow_infinity=False)
q_value_lists = st.lists(unit_floats, min_size=1, max_size=15)
llr_lists_conf = st.lists(
st.floats(min_value=-2.5, max_value=2.5, allow_nan=False, allow_infinity=False),
min_size=1, max_size=15,
)
regime_conf_mults = st.floats(min_value=0.01, max_value=1.0, allow_nan=False, allow_infinity=False)
@given(
n_eff_total=n_eff_totals,
q_values=q_value_lists,
llrs=llr_lists_conf,
strength=unit_floats,
regime_confidence_mult=regime_conf_mults,
contradiction=unit_floats,
data_quality=unit_floats,
)
@settings(max_examples=100)
def test_property_10_confidence_bounded_and_suppressed(
n_eff_total: float,
q_values: list[float],
llrs: list[float],
strength: float,
regime_confidence_mult: float,
contradiction: float,
data_quality: float,
) -> None:
"""**Validates: Requirements 8.3, 8.5**
Property 10: For any valid inputs (n_eff_total >= 0, q_values in [0,1],
strength in [0,1], regime_confidence_mult in (0,1], contradiction in [0,1],
data_quality in [0,1]), the computed confidence SHALL be in [0, 1].
Furthermore, if any single dimension (data_quality, 1-contradiction, or
C_quality) is below 0.01, the resulting confidence SHALL be below 0.10.
"""
# Align list lengths: q_values and llrs must be parallel
min_len = min(len(q_values), len(llrs))
q_values = q_values[:min_len]
llrs = llrs[:min_len]
confidence = compute_v3_confidence(
n_eff_total=n_eff_total,
q_values=q_values,
llrs=llrs,
strength=strength,
regime_confidence_mult=regime_confidence_mult,
contradiction=contradiction,
data_quality=data_quality,
)
# Confidence must be in [0, 1]
assert 0.0 <= confidence <= 1.0, (
f"Confidence {confidence} out of bounds [0, 1]"
)
# Suppression check: if data_quality < 0.01, confidence < 0.10
if data_quality < 0.01:
assert confidence < 0.10, (
f"Confidence {confidence} not suppressed by data_quality={data_quality}"
)
# Suppression check: if (1 - contradiction) < 0.01, confidence < 0.10
if (1.0 - contradiction) < 0.01:
assert confidence < 0.10, (
f"Confidence {confidence} not suppressed by contradiction={contradiction}"
)
# Suppression check: if C_quality < 0.01, confidence < 0.10
# C_quality = weighted_mean(q_i, |LLR_i|) = sum(q_i * |LLR_i|) / sum(|LLR_i|)
abs_llrs = [abs(llr) for llr in llrs]
sum_abs_llrs = sum(abs_llrs)
if sum_abs_llrs == 0.0:
c_quality = 0.0
else:
c_quality = sum(q * w for q, w in zip(q_values, abs_llrs)) / sum_abs_llrs
if c_quality < 0.01:
assert confidence < 0.10, (
f"Confidence {confidence} not suppressed by C_quality={c_quality}"
)
# ---------------------------------------------------------------------------
# Property 10 suppression sub-test: force one dimension below 0.01
# ---------------------------------------------------------------------------
@given(
n_eff_total=st.floats(min_value=1.0, max_value=20.0, allow_nan=False, allow_infinity=False),
q_values=st.lists(
st.floats(min_value=0.5, max_value=1.0, allow_nan=False, allow_infinity=False),
min_size=1, max_size=10,
),
llrs=st.lists(
st.floats(min_value=0.5, max_value=2.5, allow_nan=False, allow_infinity=False),
min_size=1, max_size=10,
),
strength=st.floats(min_value=0.3, max_value=1.0, allow_nan=False, allow_infinity=False),
regime_confidence_mult=regime_conf_mults,
dimension=st.sampled_from(["data_quality", "contradiction", "c_quality"]),
)
@settings(max_examples=100)
def test_property_10_suppression_by_weak_dimension(
n_eff_total: float,
q_values: list[float],
llrs: list[float],
strength: float,
regime_confidence_mult: float,
dimension: str,
) -> None:
"""**Validates: Requirements 8.3, 8.5**
Property 10 (suppression): If any single dimension (data_quality,
1-contradiction, or C_quality) is below 0.01, the resulting confidence
SHALL be below 0.10.
"""
# Align list lengths
min_len = min(len(q_values), len(llrs))
q_values = q_values[:min_len]
llrs = llrs[:min_len]
# Force one dimension to be weak
if dimension == "data_quality":
data_quality = 0.005 # below 0.01
contradiction = 0.0
elif dimension == "contradiction":
data_quality = 0.9
contradiction = 0.995 # (1 - contradiction) = 0.005 < 0.01
else: # c_quality
# Force all q_values near zero so C_quality < 0.01
q_values = [0.001] * min_len
data_quality = 0.9
contradiction = 0.0
confidence = compute_v3_confidence(
n_eff_total=n_eff_total,
q_values=q_values,
llrs=llrs,
strength=strength,
regime_confidence_mult=regime_confidence_mult,
contradiction=contradiction,
data_quality=data_quality,
)
assert 0.0 <= confidence <= 1.0, (
f"Confidence {confidence} out of bounds [0, 1]"
)
assert confidence < 0.10, (
f"Confidence {confidence} not suppressed when {dimension} is weak"
)
# ---------------------------------------------------------------------------
# Property 17: Data quality score is bounded in [0, 1]
# Feature: math-core-v3-engine
# ---------------------------------------------------------------------------
from datetime import datetime
from datetime import timezone as _tz
# Strategies for Property 17
extraction_failure_rates = st.floats(
min_value=0.0, max_value=1.0, allow_nan=False, allow_infinity=False
)
extraction_confs = st.floats(
min_value=0.0, max_value=1.0, allow_nan=False, allow_infinity=False
)
impacts = st.floats(
min_value=0.0, max_value=1.0, allow_nan=False, allow_infinity=False
)
age_newest_hours_strat = st.floats(
min_value=0.0, max_value=10000.0, allow_nan=False, allow_infinity=False
)
n_source_types_strat = st.integers(min_value=0, max_value=20)
@st.composite
def evidence_unit_list_strategy(draw: st.DrawFn) -> list[EvidenceUnit]:
"""Generate a list of EvidenceUnit-like objects for data quality testing."""
n = draw(st.integers(min_value=0, max_value=15))
units = []
_ts = datetime(2024, 1, 1, tzinfo=_tz.utc)
for i in range(n):
ext_conf = draw(extraction_confs)
impact = draw(impacts)
unit = EvidenceUnit(
symbol="TEST",
layer="company",
event_type="earnings",
source_id=f"src_{i}",
source_group="news",
timestamp=_ts,
horizon="7d",
direction=1,
sentiment_strength=0.5,
impact=impact,
extraction_conf=ext_conf,
source_cred=0.7,
novelty=0.5,
event_base_rate=0.25,
cluster_id=f"cluster_{i}",
)
units.append(unit)
return units
@given(
units=evidence_unit_list_strategy(),
extraction_failure_rate=extraction_failure_rates,
age_newest_hours=age_newest_hours_strat,
n_source_types=n_source_types_strat,
)
@settings(max_examples=100)
def test_property_17_data_quality_bounded_zero_one(
units: list[EvidenceUnit],
extraction_failure_rate: float,
age_newest_hours: float,
n_source_types: int,
) -> None:
"""**Validates: Requirements 17.6, 21.6**
Property 17: For any valid inputs (extraction_failure_rate in [0,1],
extraction_conf_i in [0,1], impact_i in [0,1], age_newest_hours >= 0,
N_valid >= 0, N_source_types >= 0), the computed data_quality_score
SHALL be in [0, 1].
"""
data_quality = compute_v3_data_quality(
units=units,
extraction_failure_rate=extraction_failure_rate,
age_newest_hours=age_newest_hours,
n_source_types=n_source_types,
)
assert 0.0 <= data_quality <= 1.0, (
f"Data quality score {data_quality} out of bounds [0, 1] "
f"for extraction_failure_rate={extraction_failure_rate}, "
f"age_newest_hours={age_newest_hours}, n_source_types={n_source_types}, "
f"n_units={len(units)}"
)
# ---------------------------------------------------------------------------
# Property 12: Posterior state JSON round-trip
# Feature: math-core-v3-engine
# ---------------------------------------------------------------------------
import json
from dataclasses import asdict
@given(
llr_values=st.lists(
st.floats(min_value=-2.5, max_value=2.5, allow_nan=False, allow_infinity=False),
min_size=1, max_size=10,
),
gamma=gammas,
p_prior=priors,
)
@settings(max_examples=100)
def test_property_12_posterior_json_round_trip(
llr_values: list[float],
gamma: float,
p_prior: float,
) -> None:
"""**Validates: Requirements 20.1, 21.10**
Property 12: Serialize V3Posterior to JSON via dataclasses.asdict() and
json.dumps(), then deserialize via json.loads() and V3Posterior(**data).
All numeric fields SHALL be equivalent within 1e-10, and string fields
SHALL be exactly equal.
"""
# Build clusters and regime, compute posterior
clusters = [_make_cluster(llr) for llr in llr_values]
regime = _make_regime(gamma)
posterior = compute_v3_posterior(clusters, regime, p_prior=p_prior)
# Serialize to JSON
serialized = json.dumps(asdict(posterior))
# Deserialize back to V3Posterior
restored = V3Posterior(**json.loads(serialized))
# Verify numeric fields within 1e-10
assert math.isclose(posterior.p_up, restored.p_up, abs_tol=1e-10), (
f"p_up mismatch: {posterior.p_up} vs {restored.p_up}"
)
assert math.isclose(posterior.p_down, restored.p_down, abs_tol=1e-10), (
f"p_down mismatch: {posterior.p_down} vs {restored.p_down}"
)
assert math.isclose(posterior.log_odds, restored.log_odds, abs_tol=1e-10), (
f"log_odds mismatch: {posterior.log_odds} vs {restored.log_odds}"
)
assert math.isclose(posterior.strength, restored.strength, abs_tol=1e-10), (
f"strength mismatch: {posterior.strength} vs {restored.strength}"
)
assert math.isclose(posterior.n_eff_total, restored.n_eff_total, abs_tol=1e-10), (
f"n_eff_total mismatch: {posterior.n_eff_total} vs {restored.n_eff_total}"
)
# Verify string fields are exactly equal
assert posterior.direction == restored.direction, (
f"direction mismatch: {posterior.direction!r} vs {restored.direction!r}"
)
assert posterior.regime == restored.regime, (
f"regime mismatch: {posterior.regime!r} vs {restored.regime!r}"
)