fix: pipeline health — stuck docs, price fallback, sentiment normalization, signal-engine scale, quality gate
- Scheduler: lower stale threshold 240→30 min, batch limit 100→500, TTL 14400→3600 - Prediction snapshot: add 24h market_snapshots time-window fallback - Aggregation: add normalize_impact_scores() z-score normalization - Helm: signal-engine replicas → 0 (idle when dual pipeline disabled) - Quality gate: max_snapshot_age_hours 24→48 - Add backfill script for NULL price_at_prediction snapshots - Add PBT bug condition and preservation tests (14 tests)
This commit is contained in:
@@ -15,6 +15,8 @@ from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import asyncpg
|
||||
|
||||
from services.shared.schemas import MarketContext
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -577,6 +579,57 @@ def sentiment_to_numeric(sentiment: str) -> float:
|
||||
return mapping.get(sentiment.lower(), 0.0)
|
||||
|
||||
|
||||
async def normalize_impact_scores(
|
||||
pool: asyncpg.Pool,
|
||||
ticker: str,
|
||||
raw_scores: list[float],
|
||||
) -> list[float]:
|
||||
"""Normalize impact scores using 7-day rolling z-score per ticker.
|
||||
|
||||
Computes the 7-day mean and stddev of impact_score values from
|
||||
document_impact_records for the given ticker, then normalizes
|
||||
each raw score: normalized = (raw - mean_7d) / max(stddev_7d, 0.1)
|
||||
|
||||
The 0.1 floor prevents division by near-zero stddev for low-activity tickers.
|
||||
|
||||
Fallback: if fewer than 5 records in the 7-day window, returns raw scores
|
||||
unchanged (insufficient data for meaningful normalization).
|
||||
|
||||
Args:
|
||||
pool: asyncpg connection pool.
|
||||
ticker: The ticker symbol to query distribution for.
|
||||
raw_scores: List of raw impact_score values to normalize.
|
||||
|
||||
Returns:
|
||||
List of normalized impact scores (same length as input).
|
||||
"""
|
||||
if not raw_scores:
|
||||
return []
|
||||
|
||||
row = await pool.fetchrow(
|
||||
"""
|
||||
SELECT AVG(impact_score) as mean,
|
||||
STDDEV(impact_score) as stddev,
|
||||
COUNT(*) as cnt
|
||||
FROM document_impact_records
|
||||
WHERE ticker = $1 AND created_at >= NOW() - INTERVAL '7 days'
|
||||
""",
|
||||
ticker,
|
||||
)
|
||||
|
||||
# Fallback: insufficient data for meaningful normalization
|
||||
if row is None or row["cnt"] < 5:
|
||||
return list(raw_scores)
|
||||
|
||||
mean_7d: float = float(row["mean"])
|
||||
stddev_7d: float = float(row["stddev"]) if row["stddev"] is not None and row["stddev"] > 0 else 0.0
|
||||
|
||||
# Apply 0.1 floor to prevent division by near-zero stddev
|
||||
effective_stddev = max(stddev_7d, 0.1)
|
||||
|
||||
return [(raw - mean_7d) / effective_stddev for raw in raw_scores]
|
||||
|
||||
|
||||
def weighted_sentiment_average(signals: list[WeightedSignal]) -> float:
|
||||
"""Compute a weight-adjusted average sentiment across signals.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user