feat: math core v3 engine upgrade

This commit is contained in:
Celes Renata
2026-06-27 12:21:41 +00:00
parent 365bc5d4b7
commit b4bf0f2361
34 changed files with 11693 additions and 3 deletions
+129
View File
@@ -5,14 +5,20 @@ log-likelihood accumulation, Beta distribution parameters, and
Shannon entropy for mixed-signal detection.
Requirements: 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 9.1, 9.7
V3 posterior assembly: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7
"""
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING
from services.aggregation.scoring import WeightedSignal
if TYPE_CHECKING:
from services.aggregation.regime import V3RegimeClassification
from services.aggregation.worker import EvidenceCluster
@dataclass(frozen=True)
class BayesianPosterior:
@@ -125,3 +131,126 @@ def compute_bayesian_posterior(
entropy=entropy,
signal_count=count,
)
# ---------------------------------------------------------------------------
# V3 Posterior Assembly — Calibrated Evidence Engine
# Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7
# ---------------------------------------------------------------------------
@dataclass(frozen=True)
class V3Posterior:
"""V3 posterior result from log-odds Bayesian assembly.
Attributes:
p_up: Posterior probability of upward move, (0, 1).
p_down: 1 - p_up.
log_odds: Raw log-odds (logit) of P_up.
strength: abs(2 × P_up - 1), signal conviction [0, 1].
direction: Classified direction string ('bullish', 'bearish', 'neutral').
n_eff_total: Total effective evidence count across all clusters.
regime: Market regime string used for this computation.
"""
p_up: float
p_down: float
log_odds: float
strength: float
direction: str
n_eff_total: float
regime: str
# Regime-specific direction thresholds: (bullish_threshold, bearish_threshold)
# P_up >= bullish → "bullish"; P_up <= bearish → "bearish"; else "neutral"
_V3_DIRECTION_THRESHOLDS: dict[str, tuple[float, float]] = {
"panic": (0.68, 0.32),
"trend_following": (0.60, 0.40),
"mean_reversion": (0.63, 0.37),
"uncertainty": (0.65, 0.35),
}
def _logit(p: float) -> float:
"""Compute logit = ln(p / (1-p)) with boundary guard."""
p = max(1e-10, min(1 - 1e-10, p))
return math.log(p / (1 - p))
def _sigmoid(x: float) -> float:
"""Compute sigmoid = 1 / (1 + exp(-x)) with overflow guard."""
if x > 500:
return 1.0
if x < -500:
return 0.0
return 1.0 / (1.0 + math.exp(-x))
def compute_v3_posterior(
clusters: list[EvidenceCluster],
regime: V3RegimeClassification,
p_prior: float = 0.50,
) -> V3Posterior:
"""Assemble v3 posterior via log-odds accumulation.
Computes:
logit(P_up) = logit(P_prior) + sum(gamma_regime × LLR_c)
P_up = sigmoid(log_odds), clamped to [1e-10, 1 - 1e-10]
strength = abs(2 × P_up - 1)
direction via regime-specific thresholds
Args:
clusters: List of EvidenceCluster objects with computed cluster_llr.
regime: V3RegimeClassification providing evidence_multiplier and regime.
p_prior: Calibrated prior probability, clamped to [0.40, 0.60].
Returns:
V3Posterior with computed posterior fields.
Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7
"""
# Clamp prior to [0.40, 0.60] (Req 5.6)
p_prior = max(0.40, min(0.60, p_prior))
# Gamma: regime-specific evidence multiplier (Req 5.2)
gamma = regime.evidence_multiplier
# Compute log-odds: logit(P_prior) + sum(gamma × LLR_c) (Req 5.1, 5.2)
log_odds = _logit(p_prior) + sum(gamma * c.cluster_llr for c in clusters)
# Compute P_up via sigmoid (Req 5.3)
p_up = _sigmoid(log_odds)
# Clamp to open interval (Req 5.3)
p_up = max(1e-10, min(1 - 1e-10, p_up))
p_down = 1.0 - p_up
# Strength = |2 × P_up - 1| (Req 5.4)
strength = abs(2.0 * p_up - 1.0)
# n_eff_total = sum of cluster n_eff (Req 5.5)
n_eff_total = sum(c.n_eff for c in clusters)
# Classify direction using regime-specific thresholds (Req 5.5)
regime_key = regime.regime.value # MarketRegime enum → string
bull_thresh, bear_thresh = _V3_DIRECTION_THRESHOLDS.get(
regime_key, (0.65, 0.35)
)
if p_up >= bull_thresh:
direction = "bullish"
elif p_up <= bear_thresh:
direction = "bearish"
else:
direction = "neutral"
return V3Posterior(
p_up=p_up,
p_down=p_down,
log_odds=log_odds,
strength=strength,
direction=direction,
n_eff_total=n_eff_total,
regime=regime_key,
)
+68
View File
@@ -10,10 +10,14 @@ from __future__ import annotations
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING
from services.aggregation.scoring import WeightedSignal
from services.shared.schemas import DisagreementDetail
if TYPE_CHECKING:
from services.aggregation.worker import EvidenceCluster
@dataclass
class CatalystEntry:
@@ -236,3 +240,67 @@ def _detect_catalyst_disagreement(
))
return details
# ---------------------------------------------------------------------------
# V3 LLR Entropy Contradiction
# ---------------------------------------------------------------------------
def compute_v3_contradiction(clusters: list[EvidenceCluster]) -> float:
"""Compute LLR entropy contradiction score.
Uses Shannon entropy over positive/negative cluster LLR magnitudes,
weighted by a volume factor that grows with total evidence mass.
Formula:
E_pos = sum(max(LLR_c, 0))
E_neg = sum(max(-LLR_c, 0))
E_total = E_pos + E_neg
f_pos = E_pos / E_total, f_neg = E_neg / E_total
H_conflict = -f_pos × log2(f_pos) - f_neg × log2(f_neg)
volume_factor = 1 - exp(-E_total / 3.0)
result = H_conflict × volume_factor, bounded in [0.0, 1.0]
Returns 0.0 when:
- clusters is empty
- E_total == 0 (all cluster LLRs are zero)
- Only one direction exists (E_pos == 0 or E_neg == 0)
Requirements: 7.17.7
"""
if not clusters:
return 0.0
e_pos = 0.0
e_neg = 0.0
for cluster in clusters:
llr_c = cluster.cluster_llr
if llr_c > 0.0:
e_pos += llr_c
elif llr_c < 0.0:
e_neg += -llr_c # max(-LLR_c, 0) when LLR_c < 0
e_total = e_pos + e_neg
# No evidence or unidirectional → no contradiction
if e_total == 0.0:
return 0.0
if e_pos == 0.0 or e_neg == 0.0:
return 0.0
# Compute fractions
f_pos = e_pos / e_total
f_neg = e_neg / e_total
# Shannon entropy H_conflict = -f_pos × log2(f_pos) - f_neg × log2(f_neg)
# 0 × log2(0) is treated as 0, but the early returns above guarantee
# both f_pos and f_neg are positive here.
h_conflict = -f_pos * math.log2(f_pos) - f_neg * math.log2(f_neg)
# Volume factor: suppresses score when total evidence mass is small
volume_factor = 1.0 - math.exp(-e_total / 3.0)
# Final score bounded to [0.0, 1.0]
result = h_conflict * volume_factor
return max(0.0, min(1.0, result))
+105
View File
@@ -11,6 +11,7 @@ from __future__ import annotations
import json
import logging
import math
from dataclasses import dataclass, field
from datetime import datetime, timezone
@@ -955,3 +956,107 @@ def apply_accelerated_decay(
return accelerated
return standard_decay
# ---------------------------------------------------------------------------
# V3 Macro Layer — Noisy-OR Exposure & LLR Emission (Requirements: 9.19.5)
# ---------------------------------------------------------------------------
# Noisy-OR weights per dimension
_V3_MACRO_WEIGHTS: dict[str, float] = {
"geo": 0.35,
"supply": 0.25,
"commodity": 0.25,
"sector": 0.15,
}
# Resilience dampener per tier
_V3_RESILIENCE_DAMPENER: dict[str, float] = {
"global_leader": 0.70,
"multinational": 0.85,
"regional": 1.00,
"domestic": 1.20,
}
def compute_normalized_macro_exposure(
overlaps: dict[str, float],
tier: str = "regional",
) -> float:
"""Compute normalized macro exposure via noisy-OR.
E_raw = 1 - product(1 - w_k × O_k)
E_max = 1 - product(1 - w_k)
E_macro = E_raw / E_max × resilience_dampener
Args:
overlaps: Dimension overlap values keyed by 'geo', 'supply',
'commodity', 'sector'. Missing keys treated as 0.0.
tier: Market position tier for resilience dampening.
Returns:
Normalized macro exposure in [0, ∞) (can exceed 1.0 for domestic
tier due to 1.20 dampener, but typically in [0, ~1.2]).
Requirements: 9.1, 9.2, 9.3
"""
# E_raw = 1 - product(1 - w_k × O_k)
product_raw = 1.0
for dim, weight in _V3_MACRO_WEIGHTS.items():
o_k = max(0.0, min(1.0, overlaps.get(dim, 0.0)))
product_raw *= (1.0 - weight * o_k)
e_raw = 1.0 - product_raw
# E_max = 1 - product(1 - w_k) — theoretical max when all overlaps = 1.0
product_max = 1.0
for weight in _V3_MACRO_WEIGHTS.values():
product_max *= (1.0 - weight)
e_max = 1.0 - product_max
# Guard against zero (should never happen with default weights)
if e_max <= 0.0:
return 0.0
# Normalize to [0, 1]
e_macro = e_raw / e_max
# Apply resilience dampener per tier
dampener = _V3_RESILIENCE_DAMPENER.get(tier, 1.0)
return e_macro * dampener
def compute_macro_llr(
macro_impact: float,
event_confidence: float,
q_recency: float,
macro_direction: int,
) -> float:
"""Compute macro LLR for shared posterior.
p_macro = clamp(0.50 + 0.30 × macro_impact × event_confidence × q_recency, 0.501, 0.80)
LLR_macro = macro_direction × ln(p_macro / (1 - p_macro))
When macro_direction == 0, returns 0.0 (neutral — no directional signal).
Args:
macro_impact: Normalized macro impact score (typically [0, 1]).
event_confidence: Event classification confidence [0, 1].
q_recency: Recency quality factor [0, 1].
macro_direction: +1 for positive, -1 for negative, 0 for neutral.
Returns:
Log-likelihood ratio for the macro signal. Feeds directly into the
shared posterior without separate post-hoc modifier.
Requirements: 9.4, 9.5
"""
if macro_direction == 0:
return 0.0
# p_macro = clamp(0.50 + 0.30 × macro_impact × event_confidence × q_recency, 0.501, 0.80)
p_macro = 0.50 + 0.30 * macro_impact * event_confidence * q_recency
p_macro = max(0.501, min(0.80, p_macro))
# LLR_macro = macro_direction × ln(p_macro / (1 - p_macro))
llr = macro_direction * math.log(p_macro / (1.0 - p_macro))
return llr
+102
View File
@@ -13,11 +13,15 @@ import logging
import math
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import TYPE_CHECKING
import asyncpg
from services.shared.schemas import TrendSummary
if TYPE_CHECKING:
from services.aggregation.regime import V3RegimeClassification
logger = logging.getLogger("projection")
# ---------------------------------------------------------------------------
@@ -493,3 +497,101 @@ async def persist_trend_projection(
projection.diverges_from_current,
)
return str(row_id)
# ---------------------------------------------------------------------------
# V3 Posterior State Projection (Requirements: 11.111.7)
# ---------------------------------------------------------------------------
# Regime decay factors (phi) — Req 11.3
_V3_PHI_DECAY: dict[str, float] = {
"panic": 0.35,
"trend_following": 0.80,
"mean_reversion": 0.55,
"uncertainty": 0.50,
}
def _logit(p: float) -> float:
"""Compute logit = ln(p / (1-p)) with boundary guard."""
p = max(1e-10, min(1 - 1e-10, p))
return math.log(p / (1 - p))
def _sigmoid(x: float) -> float:
"""Compute sigmoid = 1 / (1 + exp(-x)) with overflow guard."""
if x > 500:
return 1.0
if x < -500:
return 0.0
return 1.0 / (1.0 + math.exp(-x))
@dataclass
class V3ProjectionState:
"""V3 posterior state projection result.
Attributes:
a_t: Accumulated evidence state A_t.
p_up_projected: Projected probability sigmoid(logit(P_prior) + phi^h * A_t).
projected_strength: abs(2 * P_up_projected - 1).
diverges: True when sign(P_up_projected - 0.5) != sign(P_up_t - 0.5).
phi_regime: Regime-specific decay factor used.
"""
a_t: float
p_up_projected: float
projected_strength: float
diverges: bool
phi_regime: float
def compute_v3_projection(
a_prev: float,
cluster_llrs: list[float],
regime: V3RegimeClassification,
p_prior: float,
projection_horizon: int,
known_catalyst_llr: float = 0.0,
) -> V3ProjectionState:
"""Compute posterior state projection with regime-aware decay.
Evidence state: A_t = phi_regime * A_{t-1} + sum(LLR_c), init A_0 = 0.0
Projected alpha: A_projected = phi^h * A_t + known_catalyst_LLR
P_up_projected = sigmoid(logit(P_prior) + A_projected)
Projected strength = abs(2 * P_up_projected - 1)
Divergence flagged when sign(P_up_projected - 0.5) != sign(P_up_t - 0.5)
Requirements: 11.111.7
"""
# Resolve phi from regime; default to uncertainty (0.50) if unavailable (Req 11.7)
phi = _V3_PHI_DECAY.get(regime.regime.value, 0.50) if regime else 0.50
# Evidence state update: A_t = phi * A_{t-1} + sum(LLR_c) — Req 11.1, 11.2
a_t = phi * a_prev + sum(cluster_llrs)
# Projected alpha: A_projected = phi^h * A_t + known_catalyst_LLR — Req 11.4
a_projected = (phi ** projection_horizon) * a_t + known_catalyst_llr
# P_up_projected = sigmoid(logit(P_prior) + A_projected) — Req 11.5
p_up_projected = _sigmoid(_logit(p_prior) + a_projected)
# Projected strength = abs(2 * P_up_projected - 1) — Req 11.6
projected_strength = abs(2.0 * p_up_projected - 1.0)
# Compute current P_up_t for divergence check (not projected)
p_up_t = _sigmoid(_logit(p_prior) + a_t)
# Flag divergence when projected direction differs from current — Req 11.6
sign_projected = (p_up_projected - 0.5) >= 0
sign_current = (p_up_t - 0.5) >= 0
diverges = sign_projected != sign_current
return V3ProjectionState(
a_t=a_t,
p_up_projected=p_up_projected,
projected_strength=projected_strength,
diverges=diverges,
phi_regime=phi,
)
+148
View File
@@ -168,3 +168,151 @@ def classify_regime(
bearish_threshold=-threshold,
contradiction_penalty_multiplier=contradiction_mult,
)
# ---------------------------------------------------------------------------
# V3 Regime Detection — Calibrated Evidence Engine
# Requirements: 6.1, 6.2, 6.3, 6.4, 6.5, 6.6, 6.7, 6.8
# ---------------------------------------------------------------------------
@dataclass(frozen=True)
class V3RegimeClassification:
"""V3 regime classification result with calibrated parameters.
Attributes:
regime: Market regime category.
trend_z: ATR-normalized trend indicator (EMA_20 - EMA_100) / ATR_20.
vol_ratio: Volatility ratio sigma_20 / sigma_100.
evidence_multiplier: Regime-specific gamma for posterior LLR scaling.
confidence_multiplier: Regime-specific confidence scaling factor.
phi_decay: Evidence state decay factor for projection.
atr_multiplier: Regime-specific ATR multiplier for stop computation.
"""
regime: MarketRegime
trend_z: float
vol_ratio: float
evidence_multiplier: float
confidence_multiplier: float
phi_decay: float
atr_multiplier: float
# Regime parameter lookup: (gamma, confidence_mult, phi, ATR_mult, min_edge)
_V3_REGIME_PARAMS: dict[MarketRegime, tuple[float, float, float, float, float]] = {
MarketRegime.PANIC: (0.70, 0.70, 0.35, 2.5, 0.0100),
MarketRegime.TREND_FOLLOWING: (1.10, 1.00, 0.80, 1.8, 0.0035),
MarketRegime.MEAN_REVERSION: (0.90, 0.95, 0.55, 1.4, 0.0050),
MarketRegime.UNCERTAINTY: (0.80, 0.85, 0.50, 2.0, 0.0075),
}
# Default uncertainty classification for v3 when data is insufficient (Req 6.8)
_DEFAULT_V3_UNCERTAINTY = V3RegimeClassification(
regime=MarketRegime.UNCERTAINTY,
trend_z=0.0,
vol_ratio=1.0,
evidence_multiplier=0.80,
confidence_multiplier=0.85,
phi_decay=0.50,
atr_multiplier=2.0,
)
def _compute_ema_full(values: list[float], span: int) -> float:
"""Compute EMA over the full values list with given span.
Uses standard EMA formula: alpha = 2 / (span + 1), iterating from the
beginning of the list. Seeds EMA with the first value.
This differs from ``compute_ema`` which only uses the last ``period``
values. V3 requires iterating over the full history to produce a stable
EMA_100.
"""
if not values or span < 1:
raise ValueError("values must be non-empty and span must be >= 1")
alpha = 2.0 / (span + 1)
ema = values[0]
for value in values[1:]:
ema = alpha * value + (1.0 - alpha) * ema
return ema
def classify_regime_v3(
closing_prices: list[float],
daily_returns: list[float],
atr_20: float,
) -> V3RegimeClassification:
"""Classify market regime using v3 ATR-normalized indicators.
Computes trend_z = (EMA_20 - EMA_100) / ATR_20 and
vol_ratio = sigma_20 / sigma_100 to determine the market regime.
Classification priority (Req 6.26.5):
1. Panic: vol_ratio > 1.5 OR |trend_z| > 2.5
2. Trend following: |trend_z| >= 0.75 AND vol_ratio < 1.3
3. Mean reversion: |trend_z| < 0.50 AND vol_ratio < 1.0
4. Uncertainty: all other cases
Falls back to uncertainty when data is insufficient (Req 6.8):
- Fewer than 100 closing prices for EMA_100
- ATR_20 <= 0 (insufficient bars for ATR)
- Fewer than 100 daily returns for sigma_100
Requirements: 6.1, 6.2, 6.3, 6.4, 6.5, 6.6, 6.7, 6.8
"""
# --- Data sufficiency check (Req 6.8) ---
if len(closing_prices) < 100:
return _DEFAULT_V3_UNCERTAINTY
if atr_20 <= 0.0:
return _DEFAULT_V3_UNCERTAINTY
if len(daily_returns) < 100:
return _DEFAULT_V3_UNCERTAINTY
# --- Compute trend_z (Req 6.1) ---
ema_20 = _compute_ema_full(closing_prices, span=20)
ema_100 = _compute_ema_full(closing_prices, span=100)
trend_z = (ema_20 - ema_100) / atr_20
# --- Compute vol_ratio (Req 6.1) ---
sigma_20 = statistics.stdev(daily_returns[-20:]) if len(daily_returns) >= 20 else 0.0
sigma_100 = statistics.stdev(daily_returns[-100:])
# Guard against zero sigma_100
if sigma_100 <= 0.0 or math.isnan(sigma_100):
return _DEFAULT_V3_UNCERTAINTY
if math.isnan(sigma_20):
return _DEFAULT_V3_UNCERTAINTY
vol_ratio = sigma_20 / sigma_100
# --- Classification rules (Req 6.26.5) ---
# Priority 1: Panic (Req 6.2)
if vol_ratio > 1.5 or abs(trend_z) > 2.5:
regime = MarketRegime.PANIC
# Priority 2: Trend following (Req 6.3)
elif abs(trend_z) >= 0.75 and vol_ratio < 1.3:
regime = MarketRegime.TREND_FOLLOWING
# Priority 3: Mean reversion (Req 6.4)
elif abs(trend_z) < 0.50 and vol_ratio < 1.0:
regime = MarketRegime.MEAN_REVERSION
# Priority 4: Uncertainty (Req 6.5)
else:
regime = MarketRegime.UNCERTAINTY
# --- Assign regime parameters (Req 6.6, 6.7) ---
gamma, conf_mult, phi, atr_mult, _min_edge = _V3_REGIME_PARAMS[regime]
return V3RegimeClassification(
regime=regime,
trend_z=trend_z,
vol_ratio=vol_ratio,
evidence_multiplier=gamma,
confidence_multiplier=conf_mult,
phi_decay=phi,
atr_multiplier=atr_mult,
)
+625
View File
@@ -8,9 +8,12 @@ Requirements: 2.12.6, 3.13.5, 4.24.3, 5.15.7, 6.16.5, 16.416.5
"""
from __future__ import annotations
import hashlib
import logging
import math
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any
from services.shared.schemas import MarketContext
@@ -588,3 +591,625 @@ def weighted_sentiment_average(signals: list[WeightedSignal]) -> float:
if total_weight == 0.0:
return 0.0
return weighted_sum / total_weight
# ===========================================================================
# V3 Calibrated Evidence Engine — EvidenceUnit and Normalization
# ===========================================================================
# All code below this line implements the v3 pipeline. It is gated behind
# the `v3_engine_enabled` feature flag at the worker/orchestration layer.
# ===========================================================================
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# V3 Event type base rates (expanded for v3 pipeline)
# ---------------------------------------------------------------------------
V3_EVENT_TYPE_BASE_RATES: dict[str, float] = {
"earnings": 0.25,
"guidance": 0.20,
"merger_acquisition": 0.05,
"product_launch": 0.15,
"regulatory": 0.10,
"management_change": 0.08,
"partnership": 0.12,
"legal": 0.07,
"analyst_rating": 0.30,
"market_data": 0.40,
}
V3_DEFAULT_BASE_RATE: float = 0.10
# Direction mapping constants
_POSITIVE_DIRECTIONS: frozenset[str] = frozenset({"positive", "bullish"})
_NEGATIVE_DIRECTIONS: frozenset[str] = frozenset({"negative", "bearish"})
_NEUTRAL_DIRECTIONS: frozenset[str] = frozenset({"neutral", "mixed"})
# Macro horizon mapping
_MACRO_HORIZON_MAP: dict[str, str] = {
"short_term": "7d",
"medium_term": "30d",
"long_term": "90d",
}
# Valid horizons
_VALID_HORIZONS: frozenset[str] = frozenset({"intraday", "1d", "7d", "30d", "90d"})
# ---------------------------------------------------------------------------
# EvidenceUnit dataclass
# ---------------------------------------------------------------------------
@dataclass(frozen=True)
class EvidenceUnit:
"""Canonical normalized signal representation for the v3 pipeline.
Every signal — company, macro, or competitive — is normalized into this
shape before entering the calibrated reliability / LLR pipeline.
"""
symbol: str
layer: str # "company" | "macro" | "competitive"
event_type: str
source_id: str
source_group: str
timestamp: datetime
horizon: str # "intraday" | "1d" | "7d" | "30d" | "90d"
direction: int # -1, 0, +1
sentiment_strength: float # [0, 1]
impact: float # [0, 1]
extraction_conf: float # [0, 1]
source_cred: float # [0, 1]
novelty: float # [0, 1]
event_base_rate: float # (0, 1]
cluster_id: str
# ---------------------------------------------------------------------------
# Helper functions
# ---------------------------------------------------------------------------
def _map_direction(direction_str: str | None) -> int:
"""Map a sentiment/impact_direction string to a numeric direction.
Returns:
+1 for positive/bullish, -1 for negative/bearish, 0 for neutral/mixed/unknown.
"""
if direction_str is None:
return 0
lowered = direction_str.lower().strip()
if lowered in _POSITIVE_DIRECTIONS:
return 1
if lowered in _NEGATIVE_DIRECTIONS:
return -1
return 0
def _get_event_base_rate(event_type: str | None) -> float:
"""Look up base rate for an event type, defaulting to 0.10."""
if event_type is None:
return V3_DEFAULT_BASE_RATE
return V3_EVENT_TYPE_BASE_RATES.get(event_type, V3_DEFAULT_BASE_RATE)
def _compute_cluster_id(
symbol: str,
horizon: str,
event_type: str,
source_group: str,
time_bucket: str,
) -> str:
"""Compute a deterministic cluster_id from the grouping key."""
key = f"{symbol}|{horizon}|{event_type}|{source_group}|{time_bucket}"
return hashlib.sha256(key.encode()).hexdigest()[:16]
def _default_time_bucket(ts: datetime, horizon: str) -> str:
"""Compute a time bucket string for clustering based on horizon.
Bucket resolution per horizon:
intraday → 1h, 1d → 4h, 7d → 24h, 30d → 72h, 90d → 168h
"""
bucket_hours: dict[str, int] = {
"intraday": 1,
"1d": 4,
"7d": 24,
"30d": 72,
"90d": 168,
}
hours = bucket_hours.get(horizon, 24)
# Truncate timestamp to bucket boundary
epoch_hours = int(ts.timestamp() / 3600)
bucket_start = (epoch_hours // hours) * hours
return str(bucket_start)
def _safe_float(value: Any, default: float = 0.5) -> float:
"""Extract a float value, substituting default for missing/None."""
if value is None:
return default
try:
return float(value)
except (TypeError, ValueError):
return default
def _clamp(value: float, lo: float, hi: float) -> float:
"""Clamp a value to [lo, hi]."""
return max(lo, min(value, hi))
# ---------------------------------------------------------------------------
# Normalization functions
# ---------------------------------------------------------------------------
def normalize_company_signal(
signal: dict[str, Any],
*,
cluster_id: str | None = None,
) -> EvidenceUnit | None:
"""Normalize a company signal into an EvidenceUnit.
Args:
signal: Dict with keys from document_impact_records or similar.
Required: symbol, timestamp, source_id
Optional: event_type, source_group, horizon, sentiment,
sentiment_strength, impact, extraction_conf,
source_cred, novelty
cluster_id: If provided, use this cluster_id. Otherwise compute
from the signal's grouping key.
Returns:
EvidenceUnit or None if required fields are missing.
"""
# Validate required fields
symbol = signal.get("symbol")
timestamp = signal.get("timestamp")
source_id = signal.get("source_id")
if not symbol:
logger.warning("v3: Rejecting company signal — missing 'symbol'. source: %s", signal.get("source_id", "unknown"))
return None
if timestamp is None:
logger.warning("v3: Rejecting company signal — missing 'timestamp'. symbol=%s, source_id=%s", symbol, source_id)
return None
if not source_id:
logger.warning("v3: Rejecting company signal — missing 'source_id'. symbol=%s", symbol)
return None
# Ensure timestamp is datetime
if isinstance(timestamp, str):
timestamp = datetime.fromisoformat(timestamp)
if timestamp.tzinfo is None:
timestamp = timestamp.replace(tzinfo=timezone.utc)
# Extract and default fields
event_type = signal.get("event_type") or "unknown"
source_group = signal.get("source_group") or "company"
horizon = signal.get("horizon") or "7d"
if horizon not in _VALID_HORIZONS:
horizon = "7d"
# Direction mapping
direction = _map_direction(signal.get("sentiment") or signal.get("direction"))
# Optional numeric fields — default to 0.5 if missing
sentiment_strength = _clamp(_safe_float(signal.get("sentiment_strength")), 0.0, 1.0)
impact = _clamp(_safe_float(signal.get("impact")), 0.0, 1.0)
extraction_conf = _clamp(_safe_float(signal.get("extraction_conf") or signal.get("extraction_confidence")), 0.0, 1.0)
source_cred = _clamp(_safe_float(signal.get("source_cred") or signal.get("source_credibility")), 0.0, 1.0)
novelty = _clamp(_safe_float(signal.get("novelty") or signal.get("novelty_score")), 0.0, 1.0)
# Event base rate
event_base_rate = _get_event_base_rate(event_type)
# Cluster ID
if cluster_id is None:
time_bucket = _default_time_bucket(timestamp, horizon)
cluster_id = _compute_cluster_id(symbol, horizon, event_type, source_group, time_bucket)
return EvidenceUnit(
symbol=str(symbol),
layer="company",
event_type=event_type,
source_id=str(source_id),
source_group=source_group,
timestamp=timestamp,
horizon=horizon,
direction=direction,
sentiment_strength=sentiment_strength,
impact=impact,
extraction_conf=extraction_conf,
source_cred=source_cred,
novelty=novelty,
event_base_rate=event_base_rate,
cluster_id=cluster_id,
)
def normalize_macro_signal(
signal: dict[str, Any],
*,
cluster_id: str | None = None,
) -> EvidenceUnit | None:
"""Normalize a macro signal into an EvidenceUnit.
Macro signals come from macro_impact_records joined with global_events.
Args:
signal: Dict with keys from macro impact/global event records.
Required: symbol (or ticker), timestamp, source_id (or event_id)
Optional: event_type, impact_direction, macro_impact_score,
event_confidence, estimated_duration, novelty
cluster_id: If provided, use this cluster_id.
Returns:
EvidenceUnit or None if required fields are missing.
"""
# Validate required fields
symbol = signal.get("symbol") or signal.get("ticker")
timestamp = signal.get("timestamp")
source_id = signal.get("source_id") or signal.get("event_id")
if not symbol:
logger.warning("v3: Rejecting macro signal — missing 'symbol'/'ticker'. source: %s", signal.get("source_id", "unknown"))
return None
if timestamp is None:
logger.warning("v3: Rejecting macro signal — missing 'timestamp'. symbol=%s, source_id=%s", symbol, source_id)
return None
if not source_id:
logger.warning("v3: Rejecting macro signal — missing 'source_id'/'event_id'. symbol=%s", symbol)
return None
# Ensure timestamp is datetime
if isinstance(timestamp, str):
timestamp = datetime.fromisoformat(timestamp)
if timestamp.tzinfo is None:
timestamp = timestamp.replace(tzinfo=timezone.utc)
# Extract fields
event_type = signal.get("event_type") or "unknown"
source_group = "macro"
# Horizon from estimated_duration
estimated_duration = signal.get("estimated_duration") or "medium_term"
horizon = _MACRO_HORIZON_MAP.get(estimated_duration, "30d")
# Direction from impact_direction
direction = _map_direction(signal.get("impact_direction") or signal.get("direction"))
# Impact from macro_impact_score
impact = _clamp(_safe_float(signal.get("macro_impact_score") or signal.get("impact")), 0.0, 1.0)
# Source cred and extraction conf from event_confidence
event_confidence = _safe_float(signal.get("event_confidence") or signal.get("confidence"))
source_cred = _clamp(event_confidence, 0.0, 1.0)
extraction_conf = _clamp(event_confidence, 0.0, 1.0)
# Novelty: 1.0 for new events (as per requirement 1.2)
novelty = _clamp(_safe_float(signal.get("novelty"), default=1.0), 0.0, 1.0)
# Sentiment strength — default 0.5 for macro
sentiment_strength = _clamp(_safe_float(signal.get("sentiment_strength")), 0.0, 1.0)
# Event base rate
event_base_rate = _get_event_base_rate(event_type)
# Cluster ID
if cluster_id is None:
time_bucket = _default_time_bucket(timestamp, horizon)
cluster_id = _compute_cluster_id(str(symbol), horizon, event_type, source_group, time_bucket)
return EvidenceUnit(
symbol=str(symbol),
layer="macro",
event_type=event_type,
source_id=str(source_id),
source_group=source_group,
timestamp=timestamp,
horizon=horizon,
direction=direction,
sentiment_strength=sentiment_strength,
impact=impact,
extraction_conf=extraction_conf,
source_cred=source_cred,
novelty=novelty,
event_base_rate=event_base_rate,
cluster_id=cluster_id,
)
def normalize_competitive_signal(
signal: dict[str, Any],
*,
cluster_id: str | None = None,
) -> EvidenceUnit | None:
"""Normalize a competitive signal into an EvidenceUnit.
Competitive signals come from pattern mining and cross-company propagation.
Args:
signal: Dict with keys from competitive_signal_records.
Required: symbol (or target_ticker), timestamp, source_id (or source_document_id)
Optional: event_type, signal_direction, signal_strength,
relationship_strength, pattern_confidence, time_horizon
cluster_id: If provided, use this cluster_id.
Returns:
EvidenceUnit or None if required fields are missing.
"""
# Validate required fields
symbol = signal.get("symbol") or signal.get("target_ticker")
timestamp = signal.get("timestamp")
source_id = signal.get("source_id") or signal.get("source_document_id")
if not symbol:
logger.warning("v3: Rejecting competitive signal — missing 'symbol'/'target_ticker'. source: %s", signal.get("source_id", "unknown"))
return None
if timestamp is None:
logger.warning("v3: Rejecting competitive signal — missing 'timestamp'. symbol=%s, source_id=%s", symbol, source_id)
return None
if not source_id:
logger.warning("v3: Rejecting competitive signal — missing 'source_id'/'source_document_id'. symbol=%s", symbol)
return None
# Ensure timestamp is datetime
if isinstance(timestamp, str):
timestamp = datetime.fromisoformat(timestamp)
if timestamp.tzinfo is None:
timestamp = timestamp.replace(tzinfo=timezone.utc)
# Extract fields
event_type = signal.get("event_type") or "unknown"
source_group = "competitive"
# Horizon from time_horizon field
time_horizon = signal.get("time_horizon") or signal.get("horizon") or "7d"
if time_horizon in _MACRO_HORIZON_MAP:
horizon = _MACRO_HORIZON_MAP[time_horizon]
elif time_horizon in _VALID_HORIZONS:
horizon = time_horizon
else:
horizon = "7d"
# Direction from signal_direction (bullish/bearish/neutral)
direction = _map_direction(signal.get("signal_direction") or signal.get("direction"))
# Impact = signal_strength × relationship_strength (Req 1.3)
signal_strength = _safe_float(signal.get("signal_strength"))
relationship_strength = _safe_float(signal.get("relationship_strength"))
impact = _clamp(signal_strength * relationship_strength, 0.0, 1.0)
# Source cred from pattern_confidence (Req 1.3)
pattern_confidence = _safe_float(signal.get("pattern_confidence"))
source_cred = _clamp(pattern_confidence, 0.0, 1.0)
# Extraction conf = pattern_confidence (Req 1.3)
extraction_conf = _clamp(pattern_confidence, 0.0, 1.0)
# Novelty: 1.0 for competitive signals (Req 1.3)
novelty = _clamp(_safe_float(signal.get("novelty"), default=1.0), 0.0, 1.0)
# Sentiment strength — default 0.5 for competitive
sentiment_strength = _clamp(_safe_float(signal.get("sentiment_strength")), 0.0, 1.0)
# Event base rate
event_base_rate = _get_event_base_rate(event_type)
# Cluster ID
if cluster_id is None:
time_bucket = _default_time_bucket(timestamp, horizon)
cluster_id = _compute_cluster_id(str(symbol), horizon, event_type, source_group, time_bucket)
return EvidenceUnit(
symbol=str(symbol),
layer="competitive",
event_type=event_type,
source_id=str(source_id),
source_group=source_group,
timestamp=timestamp,
horizon=horizon,
direction=direction,
sentiment_strength=sentiment_strength,
impact=impact,
extraction_conf=extraction_conf,
source_cred=source_cred,
novelty=novelty,
event_base_rate=event_base_rate,
cluster_id=cluster_id,
)
# ===========================================================================
# V3 Calibrated Reliability Pipeline
# ===========================================================================
@dataclass(frozen=True)
class SourceStats:
"""Historical accuracy stats for a signal source (Bayesian prior).
Used to compute q_source via Beta-Binomial shrinkage.
"""
source_id: str
hits: int = 0 # correct directional predictions
misses: int = 0 # incorrect directional predictions
alpha_0: float = 3.0 # Beta prior alpha (pseudo-successes)
beta_0: float = 3.0 # Beta prior beta (pseudo-failures)
@dataclass(frozen=True)
class ReliabilityComponents:
"""Breakdown of calibrated reliability for a single signal.
Each q_* factor is in [0, 1] and represents one quality dimension.
q_i is the final combined reliability used downstream in LLR conversion.
"""
q_ext: float # extraction confidence reliability
q_source: float # source accuracy reliability (Bayesian shrinkage)
q_recency: float # temporal freshness reliability
q_uniqueness: float # novelty / de-duplication reliability
q_i: float # final combined: clamp(q_ext × q_source × source_cred × q_recency × q_uniqueness, 0, 1)
# Horizon-specific base half-lives for recency decay (hours)
_V3_TAU_BASE: dict[str, float] = {
"intraday": 2.0,
"1d": 12.0,
"7d": 72.0,
"30d": 240.0,
"90d": 720.0,
}
def _sigmoid(x: float) -> float:
"""Compute sigmoid(x) = 1 / (1 + exp(-x)) with overflow guard."""
if x < -500.0:
return 0.0
if x > 500.0:
return 1.0
return 1.0 / (1.0 + math.exp(-x))
def compute_v3_reliability(
unit: EvidenceUnit,
source_stats: SourceStats,
cluster_position: int, # duplicate_count_before
reference_time: datetime,
) -> ReliabilityComponents:
"""Compute calibrated reliability components for an EvidenceUnit.
Implements Requirements 2.12.9: extraction confidence gate, Bayesian
source accuracy, adaptive recency decay, and novelty/uniqueness penalty.
Args:
unit: The normalized evidence unit to score.
source_stats: Historical accuracy record for the signal's source.
cluster_position: Number of signals in the same cluster ingested
before this one (duplicate_count_before). 0 for first-in-cluster.
reference_time: The "now" anchor for computing age_hours.
Returns:
ReliabilityComponents with individual factors and combined q_i.
"""
# --- q_ext: extraction confidence reliability (Req 2.1) ---
# sigmoid(8.0 × (extraction_conf - 0.55))
q_ext = _sigmoid(8.0 * (unit.extraction_conf - 0.55))
# --- q_source: Bayesian shrinkage source reliability (Req 2.2, 2.3) ---
alpha = source_stats.alpha_0 + source_stats.hits
beta = source_stats.beta_0 + source_stats.misses
e_theta = alpha / (alpha + beta)
# clamp((E[theta] - 0.50) / 0.35, 0, 1)
q_source = _clamp((e_theta - 0.50) / 0.35, 0.0, 1.0)
# --- q_recency: adaptive exponential decay (Req 2.4, 2.5, 2.6) ---
# Ensure tz-aware timestamps
ts = unit.timestamp
if ts.tzinfo is None:
ts = ts.replace(tzinfo=timezone.utc)
ref = reference_time
if ref.tzinfo is None:
ref = ref.replace(tzinfo=timezone.utc)
age_hours = max((ref - ts).total_seconds() / 3600.0, 0.0)
# Adaptive half-life: tau_adaptive = tau_base × (1 + 0.75 × impact + 0.50 × surprise)
# surprise = clamp(-log2(event_base_rate) / 5, 0, 1)
event_base_rate = unit.event_base_rate
if event_base_rate <= 0.0:
event_base_rate = 0.10 # Req 2.5: default to 0.10 to prevent log(0)
surprise = _clamp(-math.log2(event_base_rate) / 5.0, 0.0, 1.0)
tau_base = _V3_TAU_BASE.get(unit.horizon, 72.0)
tau_adaptive = tau_base * (1.0 + 0.75 * unit.impact + 0.50 * surprise)
# q_recency = 2^(-age_hours / tau_adaptive)
# Guard against extreme exponents
if tau_adaptive <= 0.0:
tau_adaptive = tau_base # fallback
exponent = -age_hours / tau_adaptive
# For very large negative exponents, result is effectively 0
if exponent < -1000.0:
q_recency = 0.0
else:
q_recency = math.pow(2.0, exponent)
# --- q_uniqueness: novelty + de-duplication (Req 2.7) ---
# clamp(0.5 + 0.5 × novelty, 0.5, 1.0) × (1 / sqrt(1 + dup_count))
novelty_factor = _clamp(0.5 + 0.5 * unit.novelty, 0.5, 1.0)
dedup_factor = 1.0 / math.sqrt(1.0 + cluster_position)
q_uniqueness = novelty_factor * dedup_factor
# --- q_i: combined reliability (Req 2.8) ---
q_i = _clamp(
q_ext * q_source * unit.source_cred * q_recency * q_uniqueness,
0.0,
1.0,
)
# --- Explainability floor on q_recency (Req 2.9) ---
# Apply floor of 0.01 only for the display value; q_i uses raw q_recency
q_recency_display = max(q_recency, 0.01)
return ReliabilityComponents(
q_ext=q_ext,
q_source=q_source,
q_recency=q_recency_display,
q_uniqueness=q_uniqueness,
q_i=q_i,
)
# ---------------------------------------------------------------------------
# V3 LLR Conversion (Requirements 3.13.6)
# ---------------------------------------------------------------------------
def compute_llr(unit: EvidenceUnit, q_i: float) -> float:
"""Convert calibrated reliability to log-likelihood ratio.
Requirements: 3.13.6
Formula:
p_correct = clamp(0.50 + 0.35 × q_i × impact × sentiment_strength, 0.501, 0.85)
LLR_i = direction × ln(p_correct / (1 - p_correct))
For neutral signals (direction == 0), returns 0.0 immediately.
For directional signals, the LLR sign always matches direction.
Bounds:
- Minimum |LLR| ≈ ln(0.501/0.499) ≈ 0.004 for directional signals
- Maximum |LLR| ≈ ln(0.85/0.15) ≈ 1.735
Args:
unit: The normalized evidence unit containing direction, impact,
and sentiment_strength.
q_i: The combined calibrated reliability from compute_v3_reliability.
Returns:
Log-likelihood ratio. Positive for bullish, negative for bearish,
zero for neutral.
"""
# Req 3.5: Neutral signals produce zero LLR
if unit.direction == 0:
return 0.0
# Req 3.13.2: Compute p_correct with calibrated reliability
p_correct = _clamp(
0.50 + 0.35 * q_i * unit.impact * unit.sentiment_strength,
0.501,
0.85,
)
# Req 3.33.4: LLR_i = direction × ln(p_correct / (1 - p_correct))
llr = unit.direction * math.log(p_correct / (1.0 - p_correct))
return llr
@@ -378,3 +378,74 @@ def build_pattern_weighted_signals(
))
return signals
# ---------------------------------------------------------------------------
# v3 — Correlation-shrunk competitive propagation (Requirements: 10.110.5)
# ---------------------------------------------------------------------------
_V3_MAX_NETWORK_DISTANCE = 3
def compute_shrunk_correlation(
rho_rolling: float,
n_observations: int,
same_sector: bool,
) -> float:
"""Compute shrinkage-adjusted correlation.
Shrinks the rolling correlation toward a sector-aware prior using a
Bayesian-style weight of n / (n + 30).
rho_prior = 0.30 if same_sector else 0.10
rho_shrunk = (n/(n+30)) × rho_rolling + (30/(n+30)) × rho_prior
rho_effective = max(rho_shrunk, 0)
Args:
rho_rolling: Rolling pairwise correlation estimate.
n_observations: Number of observations used to compute rho_rolling.
same_sector: Whether the two securities are in the same sector.
Returns:
Non-negative shrinkage-adjusted correlation (rho_effective).
Requirements: 10.1, 10.2
"""
rho_prior = 0.30 if same_sector else 0.10
n = n_observations
rho_shrunk = (n / (n + 30)) * rho_rolling + (30 / (n + 30)) * rho_prior
rho_effective = max(rho_shrunk, 0.0)
return rho_effective
def compute_competitive_llr(
llr_source: float,
rho_effective: float,
d_network: int,
pattern_confidence: float,
) -> float:
"""Compute competitive LLR with graph attenuation.
attenuation = rho_effective × exp(-0.85 × d_network)
LLR_competitive = clamp(llr_source × attenuation × pattern_confidence, -1.25, 1.25)
When d_network > 3 → attenuation = 0 → LLR_competitive = 0
Args:
llr_source: Source signal LLR value.
rho_effective: Shrinkage-adjusted correlation (non-negative).
d_network: Graph distance between source and target (integer >= 1).
pattern_confidence: Confidence of the historical pattern in [0, 1].
Returns:
Competitive LLR clamped to [-1.25, 1.25]. Returns 0.0 when
d_network exceeds max distance of 3.
Requirements: 10.3, 10.4, 10.5
"""
if d_network > _V3_MAX_NETWORK_DISTANCE:
return 0.0
attenuation = rho_effective * math.exp(-0.85 * d_network)
llr_competitive = llr_source * attenuation * pattern_confidence
return max(-1.25, min(1.25, llr_competitive))
File diff suppressed because it is too large Load Diff