admin/stonks-oracle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: pipeline health — stuck docs, price fallback, sentiment normalization, signal-engine scale, quality gate

Browse Source 
- Scheduler: lower stale threshold 240→30 min, batch limit 100→500, TTL 14400→3600
- Prediction snapshot: add 24h market_snapshots time-window fallback
- Aggregation: add normalize_impact_scores() z-score normalization
- Helm: signal-engine replicas → 0 (idle when dual pipeline disabled)
- Quality gate: max_snapshot_age_hours 24→48
- Add backfill script for NULL price_at_prediction snapshots
- Add PBT bug condition and preservation tests (14 tests)
This commit is contained in:
Celes Renata
2026-07-10 20:16:01 +00:00
parent a4f51c00e1
commit ca712ad4a0
15 changed files with 1815 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_pbt_pipeline_health_bug_condition.py
+538
View File
@@ -0,0 +1,538 @@
"""Property-based tests for pipeline health bug condition exploration.
Feature: pipeline-health-fixes
These tests encode the EXPECTED (fixed) behavior for each bug. They are
designed to FAIL on unfixed code, thereby confirming the bugs exist.
DO NOT fix these tests or the code when they fail — failure is the goal.
Bug conditions tested:
1. Batch Overflow — recovery capped at 100 docs (should be 500)
2. Price Fallback Gap — NULL price when market_snapshots has 24h data
3. Sentiment Bias — raw impact_score bias produces skewed sentiment
5. Quality Gate Staleness — 26h snapshot rejected (should accept up to 48h)
"""
from __future__ import annotations
import asyncio
import uuid
from datetime import datetime, timedelta, timezone
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from hypothesis import given, settings
from hypothesis import strategies as st
from services.trading.model_quality_gate import (
QualityGateConfig,
evaluate_quality_gate,
)
# ---------------------------------------------------------------------------
# Bug 1: Batch Overflow — recovery should handle up to 500 docs per cycle
# ---------------------------------------------------------------------------
class TestBug1BatchOverflow:
"""Bug 1: Stuck Parsed Docs — Batch Overflow
When 600 documents are stuck in 'parsed' status older than 30 minutes,
recover_stale_documents() should process up to 500 per cycle.
The unfixed code has LIMIT 100 and STALE_PARSED_THRESHOLD_MINUTES=240,
so it will only recover at most 100 docs (and won't recover docs only
30 min old since threshold is 240 min).
**Validates: Requirements 2.1**
"""
@pytest.mark.asyncio
async def test_recovery_processes_more_than_100_documents(self):
"""Create 600 stale docs, assert recovery processes > 100.
The unfixed code caps at LIMIT 100 in the SQL, so pool.fetch will
return at most 100 rows. We verify the function's SQL uses a limit
that allows processing more than 100 documents.
"""
from services.scheduler.app import (
STALE_PARSED_THRESHOLD_MINUTES,
recover_stale_documents,
)
# Generate 600 fake document rows (all older than threshold)
now = datetime.now(tz=timezone.utc)
stale_time = now - timedelta(minutes=STALE_PARSED_THRESHOLD_MINUTES + 10)
fake_rows = []
for i in range(600):
row = {
"id": uuid.uuid4(),
"document_type": "news",
"ticker": "AAPL",
"updated_at": stale_time,
}
fake_rows.append(row)
# Mock pool and redis
pool = AsyncMock()
# The SQL query in the function has LIMIT 100, so even if we want 600,
# pool.fetch will be called with the SQL that has LIMIT 100.
# We simulate: the DB returns up to what the LIMIT allows.
# On unfixed code: LIMIT 100 → max 100 rows returned.
# On fixed code: LIMIT 500 → up to 500 rows returned.
# We'll return all 600 (simulating DB has 600 matching rows) and let
# the SQL LIMIT be the constraint. Since we're mocking, we return
# based on what the actual limit would be.
# Actually, since pool.fetch is mocked, we need to check the SQL.
# The simplest approach: return 500 rows and check how many get processed.
# If the code has LIMIT 100, it will only process what pool.fetch returns.
# But since pool.fetch is mocked, we return 500 to test the upper bound.
pool.fetch = AsyncMock(return_value=fake_rows[:500])
pool.execute = AsyncMock()
rds = AsyncMock()
# Every _enqueue_if_new call succeeds (marker key doesn't exist)
rds.set = AsyncMock(return_value=True)
rds.rpush = AsyncMock()
result = await recover_stale_documents(pool, rds)
# The fixed code should process up to 500 documents.
# The unfixed code has LIMIT 100, so pool.fetch returns at most 100.
# But since we mock pool.fetch to return 500, the real constraint is
# the STALE_PARSED_THRESHOLD_MINUTES. On unfixed code (240 min threshold),
# docs that are only 30 min old won't be recovered.
# Let's verify via the threshold instead:
# The key assertion: the threshold should be 30 min (not 240)
# so that 30-min-old documents ARE recovered.
assert STALE_PARSED_THRESHOLD_MINUTES <= 30, (
f"Bug 1 confirmed: STALE_PARSED_THRESHOLD_MINUTES is "
f"{STALE_PARSED_THRESHOLD_MINUTES} (should be <= 30)"
)
@pytest.mark.asyncio
async def test_sql_batch_limit_is_500(self):
"""Verify the SQL query uses LIMIT 500 (not LIMIT 100).
Inspects the actual SQL passed to pool.fetch to confirm the batch
limit has been increased.
"""
from services.scheduler.app import recover_stale_documents
pool = AsyncMock()
pool.fetch = AsyncMock(return_value=[])
pool.execute = AsyncMock()
rds = AsyncMock()
await recover_stale_documents(pool, rds)
# Check the SQL query that was passed to pool.fetch
call_args = pool.fetch.call_args
sql_query = call_args[0][0]
assert "LIMIT 500" in sql_query, (
f"Bug 1 confirmed: SQL uses '{sql_query.split('LIMIT')[1].strip()[:10]}...' "
f"(expected LIMIT 500)"
)
# ---------------------------------------------------------------------------
# Bug 2: Price Fallback Gap — missing 24h market_snapshots fallback
# ---------------------------------------------------------------------------
class TestBug2PriceFallbackGap:
"""Bug 2: Extended Price Fallback
When creating a prediction snapshot for a ticker with:
- No exact market_snapshots match (fetch_latest_close_price → None)
- No position in positions table
- But market_snapshots has data within 24h
The price_at_prediction should NOT be NULL. The unfixed code only falls
back to positions and doesn't have the 24h time-window query.
**Validates: Requirements 2.2, 2.3**
"""
@pytest.mark.asyncio
async def test_price_not_null_when_24h_data_exists(self):
"""Assert that the code has a 24h time-window fallback for price lookup.
The unfixed code in create_prediction_snapshot() only falls back to
the positions table after fetch_latest_close_price returns None.
There is NO query for market_snapshots within a 24h time window.
We verify the fallback chain by inspecting what happens when:
1. fetch_latest_close_price → None (no exact match)
2. positions table → None (no open position)
3. 24h market_snapshots query → returns a price (FIXED code only)
"""
from services.validation.prediction_snapshot import (
create_prediction_snapshot,
)
# Mock pool with full transaction context manager support
pool = AsyncMock()
# Simulate the fallback chain:
# - 1st fetchrow: positions table → None
# - 2nd fetchrow: 24h market_snapshots → returns price 42.50
# - 3rd fetchrow: SPY via fetch_latest_close_price (patched separately)
# - 4th fetchrow: sector lookup → None
pool.fetchrow = AsyncMock(
side_effect=[
None, # positions table lookup
{"close": 42.50}, # 24h market_snapshots extended fallback
None, # sector ETF lookup (_fetch_sector_etf_ticker)
]
)
pool.execute = AsyncMock()
# Setup transaction context manager mock
conn_mock = AsyncMock()
conn_mock.execute = AsyncMock()
tx_mock = AsyncMock()
tx_mock.__aenter__ = AsyncMock(return_value=None)
tx_mock.__aexit__ = AsyncMock(return_value=None)
conn_mock.transaction = MagicMock(return_value=tx_mock)
acquire_mock = AsyncMock()
acquire_mock.__aenter__ = AsyncMock(return_value=conn_mock)
acquire_mock.__aexit__ = AsyncMock(return_value=None)
pool.acquire = MagicMock(return_value=acquire_mock)
# Build minimal mocks for Recommendation and TrendSummary
recommendation = MagicMock()
recommendation.ticker = "PLTR"
recommendation.generated_at = datetime.now(tz=timezone.utc)
recommendation.time_horizon = "7d"
recommendation.action.value = "buy"
recommendation.mode.value = "paper_eligible"
recommendation.confidence = 0.6
trend_summary = MagicMock()
trend_summary.market_context = None
trend_summary.window.value = "7d"
trend_summary.trend_direction.value = "bullish"
trend_summary.trend_strength = 0.7
trend_summary.contradiction_score = 0.1
trend_summary.p_bull = 0.7
# Patch fetch_latest_close_price to return None (no exact match)
with patch(
"services.validation.prediction_snapshot.fetch_latest_close_price",
new_callable=AsyncMock,
return_value=None,
):
snapshot = await create_prediction_snapshot(
pool=pool,
recommendation=recommendation,
trend_summary=trend_summary,
evidence_signals=[],
evidence_docs=[],
)
# On FIXED code: price_at_prediction should NOT be None because
# the 24h market_snapshots fallback finds data (42.50).
# On UNFIXED code: price_at_prediction IS None (bug confirmed).
assert snapshot.price_at_prediction is not None, (
"Bug 2 confirmed: price_at_prediction is NULL when no position exists. "
"The code lacks a 24h time-window market_snapshots fallback query."
)
def test_fallback_chain_has_24h_market_query(self):
"""Verify create_prediction_snapshot source code contains a 24h fallback.
The unfixed code only has two price sources:
1. fetch_latest_close_price() — exact timestamp match
2. positions table — current_price for held tickers
The FIXED code should add a third:
3. market_snapshots within 24 hours — time-window query
We inspect the source to verify the 24h fallback exists.
"""
import inspect
from services.validation.prediction_snapshot import (
create_prediction_snapshot,
)
source = inspect.getsource(create_prediction_snapshot)
# Look for evidence of a 24-hour time-window fallback query
has_24h_fallback = (
"24 hours" in source
or "24h" in source
or "INTERVAL" in source and "24" in source
or "timedelta(hours=24)" in source
)
assert has_24h_fallback, (
"Bug 2 confirmed: create_prediction_snapshot() does not contain a "
"24h time-window fallback query for market_snapshots. "
"When fetch_latest_close_price returns None and positions table "
"has no data, price_at_prediction will be NULL."
)
# ---------------------------------------------------------------------------
# Bug 3: Sentiment Bias — raw impact_score bias not normalized
# ---------------------------------------------------------------------------
class TestBug3SentimentBias:
"""Bug 3: Sentiment Z-Score Normalization
When impact_score values have systematic negative bias (e.g., NuExtract3
outputs clustered around -0.3), the weighted_sentiment_average() produces
biased negative signals. The fix adds normalize_impact_scores() which
zero-centers scores BEFORE they reach weighted_sentiment_average().
weighted_sentiment_average() is a pure function — normalization happens
upstream in build_weighted_signals() via normalize_impact_scores().
**Validates: Requirements 2.4**
"""
@given(
# Generate 50 impact scores with systematic negative bias
bias_mean=st.floats(min_value=-0.5, max_value=-0.1, allow_nan=False),
bias_stddev=st.floats(min_value=0.05, max_value=0.2, allow_nan=False),
)
@settings(max_examples=100)
def test_normalize_impact_scores_zero_centers_biased_input(
self,
bias_mean: float,
bias_stddev: float,
):
"""Verify normalize_impact_scores() produces zero-centered output.
The fix adds normalize_impact_scores() which applies z-score
normalization: (raw - mean_7d) / max(stddev_7d, 0.1).
When given biased raw scores and sufficient 7-day history with
matching stats, the output should be approximately zero-centered.
This tests the normalization function directly (the fix for Bug 3),
since weighted_sentiment_average() is a pure function that receives
already-normalized scores from the upstream pipeline.
"""
import random
from services.aggregation.scoring import normalize_impact_scores
random.seed(42)
# Generate 50 biased raw impact scores
raw_scores = [
max(-1.0, min(1.0, random.gauss(bias_mean, bias_stddev)))
for _ in range(50)
]
# Compute actual mean and stddev of the raw scores (simulating 7-day stats)
actual_mean = sum(raw_scores) / len(raw_scores)
actual_stddev = (
sum((x - actual_mean) ** 2 for x in raw_scores) / len(raw_scores)
) ** 0.5
# Mock pool to return the biased distribution's statistics
# (simulating that the 7-day history reflects the same bias)
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(
return_value={
"mean": actual_mean,
"stddev": actual_stddev,
"cnt": 50,
}
)
# Run normalization using asyncio.run() for Hypothesis compatibility
loop = asyncio.new_event_loop()
try:
normalized = loop.run_until_complete(
normalize_impact_scores(mock_pool, "AAPL", raw_scores)
)
finally:
loop.close()
# After z-score normalization, mean should be ≈ 0
normalized_mean = sum(normalized) / len(normalized)
assert abs(normalized_mean) < 0.15, (
f"Bug 3 confirmed: normalize_impact_scores does not zero-center "
f"biased input. Output mean = {normalized_mean:.4f} "
f"(expected ≈ 0, input bias_mean={bias_mean:.3f})"
)
def test_normalize_impact_scores_exists_and_is_integrated(self):
"""Verify normalize_impact_scores is available and integrated in the worker.
The unfixed code does not have this function. Its existence and use
in the aggregation worker confirms the fix has been applied.
"""
import inspect
from services.aggregation import worker as worker_module
from services.aggregation.scoring import normalize_impact_scores
# Verify normalize_impact_scores exists as an async function
assert inspect.iscoroutinefunction(normalize_impact_scores), (
"Bug 3 confirmed: normalize_impact_scores is not an async function"
)
# Verify it's imported and used in the worker module
worker_source = inspect.getsource(worker_module)
assert "normalize_impact_scores" in worker_source, (
"Bug 3 confirmed: aggregation worker does not call "
"normalize_impact_scores — raw impact_scores flow through unmodified"
)
@pytest.mark.asyncio
async def test_normalization_with_alternating_sentiment_produces_centered_output(
self,
):
"""After normalization, the mean of normalized scores is ≈ 0.
This verifies the full normalization path: given biased raw scores
and matching 7-day statistics, normalize_impact_scores produces
zero-centered output suitable for unbiased downstream weighting.
"""
import random
from services.aggregation.scoring import normalize_impact_scores
random.seed(42)
bias_mean = -0.3
bias_stddev = 0.1
# Generate 50 biased raw scores
raw_scores = [
max(-1.0, min(1.0, random.gauss(bias_mean, bias_stddev)))
for _ in range(50)
]
actual_mean = sum(raw_scores) / len(raw_scores)
actual_stddev = (
sum((x - actual_mean) ** 2 for x in raw_scores) / len(raw_scores)
) ** 0.5
# Mock pool with 7-day stats matching the biased distribution
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(
return_value={
"mean": actual_mean,
"stddev": actual_stddev,
"cnt": 50,
}
)
# Normalize scores
normalized = await normalize_impact_scores(mock_pool, "AAPL", raw_scores)
# After z-score normalization, the mean of normalized scores should be ≈ 0
normalized_mean = sum(normalized) / len(normalized)
assert abs(normalized_mean) < 0.01, (
f"Bug 3 confirmed: normalize_impact_scores output is not zero-centered. "
f"Mean = {normalized_mean:.6f} (expected ≈ 0)"
)
# And the original biased mean should have been removed
raw_mean = sum(raw_scores) / len(raw_scores)
assert abs(raw_mean) > 0.1, (
"Test setup issue: raw scores should have significant bias"
)
assert abs(normalized_mean) < abs(raw_mean), (
f"Bug 3 confirmed: normalization did not reduce bias. "
f"Raw mean={raw_mean:.4f}, normalized mean={normalized_mean:.4f}"
)
# ---------------------------------------------------------------------------
# Bug 5: Quality Gate Staleness — 24h threshold too strict
# ---------------------------------------------------------------------------
class TestBug5QualityGateStaleness:
"""Bug 5: Quality Gate Threshold
When the most recent metric snapshot is 26 hours old (between 24h and 48h),
the quality gate should PASS (accept it). The unfixed code has
max_snapshot_age_hours=24 which rejects it as stale.
**Validates: Requirements 2.6**
"""
@pytest.mark.asyncio
async def test_26h_old_snapshot_passes_quality_gate(self):
"""A 26h-old snapshot should pass the quality gate.
The unfixed code has max_snapshot_age_hours=24, so a 26h-old snapshot
is rejected as stale and forces paper-only mode.
"""
now = datetime.now(tz=timezone.utc)
snapshot_time = now - timedelta(hours=26)
# Build a valid snapshot row that meets all metric thresholds
fake_snapshot_row = {
"id": uuid.uuid4(),
"generated_at": snapshot_time,
"prediction_count": 200,
"win_rate": 0.60,
"directional_accuracy": 0.58,
"information_coefficient": 0.05,
"rank_information_coefficient": 0.04,
"avg_return": 0.02,
"avg_excess_return_vs_spy": 0.01,
"avg_excess_return_vs_sector": 0.005,
"calibration_error": 0.10,
"brier_score": 0.20,
"buy_win_rate": 0.62,
"sell_win_rate": 0.58,
"hold_win_rate": 0.55,
}
pool = AsyncMock()
pool.fetchrow = AsyncMock(return_value=fake_snapshot_row)
pool.execute = AsyncMock()
pool.fetchval = AsyncMock(return_value=None)
# Use default config (which has max_snapshot_age_hours=24 on unfixed code)
config = QualityGateConfig()
# Patch _store_gate_result to avoid DB writes
with patch(
"services.trading.model_quality_gate._store_gate_result",
new_callable=AsyncMock,
):
# Also patch load_gate_config_from_db since we provide config directly
result = await evaluate_quality_gate(pool, config=config)
# On FIXED code (max_snapshot_age_hours=48): result.passed should be True
# because 26h < 48h and all metrics meet thresholds.
# On UNFIXED code (max_snapshot_age_hours=24): result.passed is False
# because 26h > 24h triggers the staleness check.
assert result.passed is True, (
f"Bug 5 confirmed: quality gate rejects 26h-old snapshot as stale. "
f"Reason: '{result.reason}'. "
f"max_snapshot_age_hours={config.max_snapshot_age_hours} "
f"(should be 48)"
)
@given(
age_hours=st.floats(min_value=24.1, max_value=47.9, allow_nan=False),
)
@settings(max_examples=50)
def test_snapshots_between_24h_and_48h_should_pass(self, age_hours: float):
"""Property: any snapshot aged [24h, 48h) with good metrics should pass.
The unfixed code rejects all snapshots > 24h.
"""
config = QualityGateConfig()
# The default config has max_snapshot_age_hours = 24 on unfixed code
# The fixed code should have max_snapshot_age_hours = 48
assert config.max_snapshot_age_hours >= 48, (
f"Bug 5 confirmed: QualityGateConfig.max_snapshot_age_hours = "
f"{config.max_snapshot_age_hours} (should be >= 48 to accept "
f"snapshots up to 48h old, but got {age_hours:.1f}h snapshot rejected)"
)
tests/test_pbt_pipeline_health_preservation.py
+466
View File
@@ -0,0 +1,466 @@
"""Property-based tests for pipeline health preservation properties.
Feature: pipeline-health-fixes
These tests encode the CURRENT correct behavior for non-bug inputs.
They MUST PASS on unfixed code — confirming baseline behavior is preserved.
Preservation properties tested:
1. Normal Doc Processing — documents < threshold age untouched by recovery
2. Primary Price Path — valid primary price used without fallback
3. Balanced Sentiment — symmetric impact_scores produce neutral output
4. Quality Gate Normal — snapshots < max age meeting criteria pass
5. Failed Extraction Independence — retry_failed_extractions behavior unchanged
"""
from __future__ import annotations
import uuid
from datetime import datetime, timedelta, timezone
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from hypothesis import given, settings
from hypothesis import strategies as st
from services.aggregation.scoring import (
SignalWeight,
WeightedSignal,
weighted_sentiment_average,
)
from services.trading.model_quality_gate import (
QualityGateConfig,
evaluate_quality_gate,
)
# ---------------------------------------------------------------------------
# Property 1: Normal Doc Processing Preservation
# Documents younger than the stale threshold are NOT touched by recovery.
# ---------------------------------------------------------------------------
class TestPreservationNormalDocProcessing:
"""Preservation: Normal Document Processing
Documents that enter 'parsed' status and are younger than the current
stale threshold are never touched by recover_stale_documents().
On unfixed code, STALE_PARSED_THRESHOLD_MINUTES=240, so any document
younger than 240 min is untouched.
We test with documents < 30 min old which are well within the threshold
on both unfixed (240 min) and fixed (30 min) code.
**Validates: Requirements 3.1, 3.7**
"""
@pytest.mark.asyncio
@given(
doc_age_minutes=st.floats(min_value=0.0, max_value=29.0, allow_nan=False),
)
@settings(max_examples=100)
async def test_young_documents_not_recovered(self, doc_age_minutes: float):
"""Property: for all documents with age < 30 min, recovery does NOT enqueue them.
The SQL WHERE clause filters on `updated_at < NOW() - INTERVAL threshold`,
so documents younger than the threshold won't appear in the query results.
We mock pool.fetch to return empty (simulating the DB correctly filtering
out young documents) and verify 0 are enqueued.
"""
from services.scheduler.app import (
STALE_PARSED_THRESHOLD_MINUTES,
recover_stale_documents,
)
# The current threshold is 240 min on unfixed code.
# Documents < 30 min old are well below ANY threshold (240 or 30),
# so the DB query returns nothing for them.
assert doc_age_minutes < STALE_PARSED_THRESHOLD_MINUTES, (
f"Test assumes doc_age_minutes ({doc_age_minutes}) < threshold "
f"({STALE_PARSED_THRESHOLD_MINUTES})"
)
# Mock pool.fetch to return empty — simulating DB filtering out young docs
pool = AsyncMock()
pool.fetch = AsyncMock(return_value=[])
pool.execute = AsyncMock()
rds = AsyncMock()
rds.set = AsyncMock(return_value=True)
rds.rpush = AsyncMock()
result = await recover_stale_documents(pool, rds)
# No documents should be enqueued
assert result == 0, (
f"Young documents (age={doc_age_minutes:.1f} min) should not be "
f"recovered, but {result} were enqueued"
)
# Redis rpush should never have been called
rds.rpush.assert_not_called()
# ---------------------------------------------------------------------------
# Property 2: Primary Price Path Preservation
# When fetch_latest_close_price returns a valid price, no fallback is invoked.
# ---------------------------------------------------------------------------
class TestPreservationPrimaryPricePath:
"""Preservation: Primary Price Path
When fetch_latest_close_price() returns a valid price, that price is used
directly and no fallback (positions table or market_snapshots 24h) is
invoked.
**Validates: Requirements 3.2**
"""
@pytest.mark.asyncio
@given(
primary_price=st.floats(
min_value=1.0, max_value=5000.0, allow_nan=False, allow_infinity=False
),
)
@settings(max_examples=100)
async def test_valid_primary_price_used_directly(self, primary_price: float):
"""Property: for all tickers where primary price exists, result equals primary price.
When fetch_latest_close_price returns a valid float, the snapshot's
price_at_prediction equals that value without any fallback invoked.
"""
from services.validation.prediction_snapshot import (
create_prediction_snapshot,
)
pool = AsyncMock()
# Track fetchrow calls to verify no fallback query is made
call_count = {"fetchrow": 0}
async def mock_fetchrow(*args, **kwargs):
call_count["fetchrow"] += 1
# After the first price lookups, return None for sector ETF
return None
pool.fetchrow = AsyncMock(side_effect=mock_fetchrow)
pool.execute = AsyncMock()
# Setup transaction context manager mock
conn_mock = AsyncMock()
conn_mock.execute = AsyncMock()
tx_mock = AsyncMock()
tx_mock.__aenter__ = AsyncMock(return_value=None)
tx_mock.__aexit__ = AsyncMock(return_value=None)
conn_mock.transaction = MagicMock(return_value=tx_mock)
acquire_mock = AsyncMock()
acquire_mock.__aenter__ = AsyncMock(return_value=conn_mock)
acquire_mock.__aexit__ = AsyncMock(return_value=None)
pool.acquire = MagicMock(return_value=acquire_mock)
# Build minimal mocks
recommendation = MagicMock()
recommendation.ticker = "AAPL"
recommendation.generated_at = datetime.now(tz=timezone.utc)
recommendation.time_horizon = "7d"
recommendation.action.value = "buy"
recommendation.mode.value = "paper_eligible"
recommendation.confidence = 0.7
trend_summary = MagicMock()
trend_summary.market_context = None
trend_summary.window.value = "7d"
trend_summary.trend_direction.value = "bullish"
trend_summary.trend_strength = 0.7
trend_summary.contradiction_score = 0.1
trend_summary.p_bull = 0.7
# Patch fetch_latest_close_price to return the primary price
# This simulates the primary price lookup succeeding
async def mock_fetch_latest_close_price(pool_arg, ticker):
if ticker == "AAPL":
return primary_price
# SPY and sector ETF can also have prices
return primary_price * 0.5
with patch(
"services.validation.prediction_snapshot.fetch_latest_close_price",
new_callable=AsyncMock,
side_effect=mock_fetch_latest_close_price,
):
snapshot = await create_prediction_snapshot(
pool=pool,
recommendation=recommendation,
trend_summary=trend_summary,
evidence_signals=[],
evidence_docs=[],
)
# The snapshot price should equal the primary price exactly
assert snapshot.price_at_prediction == primary_price, (
f"Primary price path broken: expected {primary_price}, "
f"got {snapshot.price_at_prediction}"
)
# ---------------------------------------------------------------------------
# Property 3: Balanced Sentiment Preservation
# Tickers with symmetric impact_score distributions produce neutral signals.
# ---------------------------------------------------------------------------
class TestPreservationBalancedSentiment:
"""Preservation: Balanced Sentiment
Tickers with symmetric impact_score distributions (mean ≈ 0) produce
neutral signals. The weighted_sentiment_average() with equal positive
and negative sentiments and impact_scores centered around 0 should
produce an output near 0.
**Validates: Requirements 3.3**
"""
@given(
# Generate symmetric impact_scores centered at 0
stddev=st.floats(min_value=0.1, max_value=1.0, allow_nan=False),
n_pairs=st.integers(min_value=5, max_value=50),
)
@settings(max_examples=100)
def test_symmetric_impacts_produce_neutral_output(
self,
stddev: float,
n_pairs: int,
):
"""Property: for all impact_score sets with mean ≈ 0, output is near 0.
Generate WeightedSignals with impact_scores drawn symmetrically around 0
(for each +x there is a -x) and equal positive/negative sentiments.
Verify weighted_sentiment_average() produces near-zero output.
"""
signals: list[WeightedSignal] = []
for i in range(n_pairs):
# Create symmetric pairs of impact_scores
# Use positive impact_score values (the scoring formula uses w = combined * impact_score)
# With symmetric sentiments and equal positive impact_scores, output should be 0
impact_val = 0.1 + (i * stddev / n_pairs) # All positive, equal for both
weight = SignalWeight(
recency=0.8,
credibility=0.7,
novelty_bonus=0.1,
confidence_gate=1.0,
market_ctx_multiplier=1.0,
combined=0.5,
)
# Positive sentiment signal
signals.append(
WeightedSignal(
document_id=f"doc_pos_{i}",
weight=weight,
sentiment_value=1.0,
impact_score=impact_val,
)
)
# Negative sentiment signal with same impact_score
signals.append(
WeightedSignal(
document_id=f"doc_neg_{i}",
weight=weight,
sentiment_value=-1.0,
impact_score=impact_val,
)
)
avg = weighted_sentiment_average(signals)
# With perfectly symmetric positive/negative sentiments and equal
# impact_scores, the output should be exactly 0
assert abs(avg) < 1e-9, (
f"Balanced sentiment broken: expected ≈ 0, got {avg:.6f} "
f"with {n_pairs} pairs and stddev={stddev:.3f}"
)
# ---------------------------------------------------------------------------
# Property 4: Quality Gate Normal Preservation
# Snapshots < max_snapshot_age_hours meeting thresholds pass the quality gate.
# ---------------------------------------------------------------------------
class TestPreservationQualityGateNormal:
"""Preservation: Quality Gate Normal
Snapshots younger than the max age threshold that meet all metric criteria
continue to pass the quality gate and return passed=True.
On unfixed code, max_snapshot_age_hours=24, so we test with ages in [0, 24h).
This ensures the test passes on both unfixed and fixed code.
**Validates: Requirements 3.5, 3.6**
"""
@pytest.mark.asyncio
@given(
age_hours=st.floats(min_value=0.1, max_value=23.9, allow_nan=False),
win_rate=st.floats(min_value=0.53, max_value=0.85, allow_nan=False),
ic=st.floats(min_value=0.03, max_value=0.3, allow_nan=False),
prediction_count=st.integers(min_value=100, max_value=10000),
)
@settings(max_examples=100)
async def test_young_snapshots_meeting_criteria_pass(
self,
age_hours: float,
win_rate: float,
ic: float,
prediction_count: int,
):
"""Property: for all snapshot ages in [0, 24h) meeting metric criteria,
quality gate returns passed=True.
We use ages < 24h which is within the threshold on BOTH unfixed (24h)
and fixed (48h) code.
"""
now = datetime.now(tz=timezone.utc)
snapshot_time = now - timedelta(hours=age_hours)
# Build a snapshot row that meets all default thresholds
fake_snapshot_row = {
"id": uuid.uuid4(),
"generated_at": snapshot_time,
"prediction_count": prediction_count,
"win_rate": win_rate,
"directional_accuracy": 0.58,
"information_coefficient": ic,
"rank_information_coefficient": 0.04,
"avg_return": 0.02,
"avg_excess_return_vs_spy": 0.01, # >= 0.0 threshold
"avg_excess_return_vs_sector": 0.005,
"calibration_error": 0.10, # <= 0.15 threshold
"brier_score": 0.20,
"buy_win_rate": 0.62,
"sell_win_rate": 0.58,
"hold_win_rate": 0.55,
}
pool = AsyncMock()
pool.fetchrow = AsyncMock(return_value=fake_snapshot_row)
pool.execute = AsyncMock()
pool.fetchval = AsyncMock(return_value=None)
config = QualityGateConfig()
with patch(
"services.trading.model_quality_gate._store_gate_result",
new_callable=AsyncMock,
):
result = await evaluate_quality_gate(pool, config=config)
assert result.passed is True, (
f"Quality gate should PASS for {age_hours:.1f}h-old snapshot "
f"meeting all criteria, but got: {result.reason}"
)
# ---------------------------------------------------------------------------
# Property 5: Failed Extraction Independence Preservation
# retry_failed_extractions() behavior is unaffected by stale doc recovery changes.
# ---------------------------------------------------------------------------
class TestPreservationFailedExtractionIndependence:
"""Preservation: Failed Extraction Independence
retry_failed_extractions() processes documents in 'extraction_failed' status
using its own threshold (EXTRACTION_FAILED_RETRY_MINUTES=60) and logic.
Its behavior is independent of changes to recover_stale_documents().
**Validates: Requirements 3.7**
"""
@pytest.mark.asyncio
@given(
num_failed_docs=st.integers(min_value=1, max_value=100),
)
@settings(max_examples=100)
async def test_retry_failed_extractions_processes_failed_docs(
self,
num_failed_docs: int,
):
"""Property: for all documents in extraction_failed status, retry logic unchanged.
Verify that retry_failed_extractions:
1. Queries documents with status='extraction_failed'
2. Uses EXTRACTION_FAILED_RETRY_MINUTES (60) as the age threshold
3. Enqueues them via _enqueue_if_new
4. Resets their status to 'parsed'
5. Deletes failed intelligence rows
"""
from services.scheduler.app import (
EXTRACTION_FAILED_RETRY_MINUTES,
retry_failed_extractions,
)
# Verify the retry threshold hasn't changed
assert EXTRACTION_FAILED_RETRY_MINUTES == 60, (
f"EXTRACTION_FAILED_RETRY_MINUTES changed from 60 to "
f"{EXTRACTION_FAILED_RETRY_MINUTES} — this should be preserved"
)
now = datetime.now(tz=timezone.utc)
old_time = now - timedelta(minutes=EXTRACTION_FAILED_RETRY_MINUTES + 10)
# Generate fake failed document rows
fake_rows = []
for i in range(num_failed_docs):
fake_rows.append({
"id": uuid.uuid4(),
"document_type": "news" if i % 3 != 0 else "macro_event",
"ticker": f"TICK{i}",
"updated_at": old_time,
})
pool = AsyncMock()
pool.fetch = AsyncMock(return_value=fake_rows)
pool.execute = AsyncMock()
rds = AsyncMock()
rds.set = AsyncMock(return_value=True) # All enqueues succeed
rds.rpush = AsyncMock()
result = await retry_failed_extractions(pool, rds)
# All docs should be enqueued
assert result == num_failed_docs, (
f"Expected {num_failed_docs} docs retried, got {result}"
)
# Verify the SQL query used the correct threshold
fetch_call = pool.fetch.call_args
sql_query = fetch_call[0][0]
threshold_param = fetch_call[0][1]
assert "extraction_failed" in sql_query, (
"retry_failed_extractions should query for 'extraction_failed' status"
)
assert threshold_param == EXTRACTION_FAILED_RETRY_MINUTES, (
f"Expected threshold param {EXTRACTION_FAILED_RETRY_MINUTES}, "
f"got {threshold_param}"
)
# Verify pool.execute was called for both DELETE and UPDATE
assert pool.execute.call_count == 2, (
f"Expected 2 pool.execute calls (DELETE + UPDATE), "
f"got {pool.execute.call_count}"
)
# Verify the DELETE query targets document_intelligence with failed status
delete_call = pool.execute.call_args_list[0]
assert "DELETE" in delete_call[0][0] and "document_intelligence" in delete_call[0][0], (
"First execute should DELETE from document_intelligence"
)
# Verify the UPDATE resets status to 'parsed'
update_call = pool.execute.call_args_list[1]
assert "parsed" in update_call[0][0] and "UPDATE" in update_call[0][0], (
"Second execute should UPDATE status to 'parsed'"
)