455 lines
16 KiB
Python
455 lines
16 KiB
Python
"""Integration tests for the v3 calibrated evidence pipeline.
|
||
|
||
Tests the full pipeline path through pure functions end-to-end:
|
||
raw signals → EvidenceUnit → q_i → LLR → cluster → posterior → recommendation
|
||
|
||
Also validates feature flag routing and v3 metadata fields.
|
||
|
||
Requirements validated: 19.1–19.6, 20.1–20.5
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from datetime import datetime, timezone
|
||
|
||
from services.aggregation.bayesian import V3Posterior, compute_v3_posterior
|
||
from services.aggregation.contradiction import compute_v3_contradiction
|
||
from services.aggregation.regime import (
|
||
_DEFAULT_V3_UNCERTAINTY,
|
||
)
|
||
from services.aggregation.scoring import (
|
||
SourceStats,
|
||
compute_llr,
|
||
compute_v3_reliability,
|
||
normalize_company_signal,
|
||
)
|
||
from services.aggregation.worker import (
|
||
_annotate_pipeline_mode,
|
||
cluster_evidence,
|
||
compute_cluster_llr,
|
||
compute_n_eff,
|
||
compute_v3_confidence,
|
||
compute_v3_data_quality,
|
||
should_force_informational_v3,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Fixtures / helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_NOW = datetime(2025, 1, 15, 12, 0, 0, tzinfo=timezone.utc)
|
||
|
||
_DEFAULT_REGIME = _DEFAULT_V3_UNCERTAINTY
|
||
|
||
|
||
def _make_signal(
|
||
sentiment: str = "positive",
|
||
impact: float = 0.7,
|
||
source_id: str = "doc1",
|
||
event_type: str = "earnings",
|
||
source_group: str = "company",
|
||
extraction_conf: float = 0.85,
|
||
source_cred: float = 0.80,
|
||
novelty: float = 0.7,
|
||
) -> dict:
|
||
"""Build a raw company signal dict for normalization."""
|
||
return {
|
||
"symbol": "AAPL",
|
||
"timestamp": _NOW,
|
||
"source_id": source_id,
|
||
"event_type": event_type,
|
||
"source_group": source_group,
|
||
"horizon": "7d",
|
||
"sentiment": sentiment,
|
||
"sentiment_strength": impact,
|
||
"impact": impact,
|
||
"extraction_conf": extraction_conf,
|
||
"source_cred": source_cred,
|
||
"novelty": novelty,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Test 1: Full pipeline path through pure functions
|
||
# Requirements: 20.1, 20.2, 20.3, 20.4, 20.5
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_full_pipeline_path():
|
||
"""End-to-end test: raw signals → EvidenceUnit → q_i → LLR → cluster → posterior."""
|
||
# 1. Create raw signal dicts with opposing sentiments
|
||
signals = [
|
||
_make_signal(
|
||
sentiment="positive",
|
||
impact=0.8,
|
||
source_id="doc1",
|
||
event_type="earnings",
|
||
extraction_conf=0.9,
|
||
source_cred=0.85,
|
||
novelty=0.8,
|
||
),
|
||
_make_signal(
|
||
sentiment="positive",
|
||
impact=0.6,
|
||
source_id="doc2",
|
||
event_type="earnings",
|
||
extraction_conf=0.75,
|
||
source_cred=0.7,
|
||
novelty=0.6,
|
||
),
|
||
_make_signal(
|
||
sentiment="negative",
|
||
impact=0.5,
|
||
source_id="doc3",
|
||
event_type="regulatory",
|
||
extraction_conf=0.7,
|
||
source_cred=0.6,
|
||
novelty=0.9,
|
||
),
|
||
]
|
||
|
||
# 2. Normalize to EvidenceUnit
|
||
units = [normalize_company_signal(s) for s in signals]
|
||
assert all(u is not None for u in units), "All signals should normalize successfully"
|
||
units = [u for u in units if u is not None] # type narrowing
|
||
assert len(units) == 3
|
||
|
||
# 3. Compute q_i (reliability) for each unit
|
||
neutral_stats = SourceStats(source_id="default", hits=0, misses=0)
|
||
q_values = []
|
||
for unit in units:
|
||
reliability = compute_v3_reliability(
|
||
unit=unit,
|
||
source_stats=neutral_stats,
|
||
cluster_position=0,
|
||
reference_time=_NOW,
|
||
)
|
||
q_values.append(reliability.q_i)
|
||
|
||
# All q_i should be in [0, 1]
|
||
for q in q_values:
|
||
assert 0.0 <= q <= 1.0, f"q_i out of bounds: {q}"
|
||
|
||
# 4. Compute LLR for each unit
|
||
llrs = [compute_llr(unit, q) for unit, q in zip(units, q_values)]
|
||
# Positive sentiment → positive LLR, negative → negative LLR
|
||
assert llrs[0] > 0.0, "Positive signal should produce positive LLR"
|
||
assert llrs[1] > 0.0, "Positive signal should produce positive LLR"
|
||
assert llrs[2] < 0.0, "Negative signal should produce negative LLR"
|
||
|
||
# 5. Cluster evidence
|
||
clusters = cluster_evidence(units, llrs)
|
||
assert len(clusters) >= 1, "Should produce at least one cluster"
|
||
|
||
# 6. Compute n_eff and cluster_llr for each cluster
|
||
for cluster in clusters:
|
||
cluster.n_eff = compute_n_eff(cluster.llrs)
|
||
cluster.cluster_llr = compute_cluster_llr(cluster.llrs, cluster.n_eff)
|
||
assert cluster.n_eff >= 1.0, "n_eff should be >= 1.0"
|
||
assert -2.5 <= cluster.cluster_llr <= 2.5, "cluster_llr should be clamped"
|
||
|
||
# 7. Compute posterior (using default uncertainty regime)
|
||
posterior = compute_v3_posterior(clusters, _DEFAULT_REGIME, p_prior=0.50)
|
||
assert isinstance(posterior, V3Posterior)
|
||
|
||
# 8. Assert posterior is valid
|
||
assert 0.0 < posterior.p_up < 1.0, f"P_up should be in (0, 1), got {posterior.p_up}"
|
||
assert 0.0 < posterior.p_down < 1.0, "P_down should be in (0, 1)"
|
||
assert abs(posterior.p_up + posterior.p_down - 1.0) < 1e-9
|
||
assert 0.0 <= posterior.strength <= 1.0
|
||
assert posterior.direction in ("bullish", "bearish", "neutral")
|
||
assert posterior.n_eff_total > 0
|
||
|
||
# 9. Compute contradiction
|
||
contradiction = compute_v3_contradiction(clusters)
|
||
# 10. Assert contradiction is bounded
|
||
assert 0.0 <= contradiction <= 1.0
|
||
|
||
# 11. Compute data quality
|
||
data_quality = compute_v3_data_quality(
|
||
units=units,
|
||
extraction_failure_rate=0.0,
|
||
age_newest_hours=0.0,
|
||
n_source_types=1,
|
||
)
|
||
assert 0.0 <= data_quality <= 1.0
|
||
|
||
# 12. Compute confidence
|
||
confidence = compute_v3_confidence(
|
||
n_eff_total=posterior.n_eff_total,
|
||
q_values=q_values,
|
||
llrs=llrs,
|
||
strength=posterior.strength,
|
||
regime_confidence_mult=_DEFAULT_REGIME.confidence_multiplier,
|
||
contradiction=contradiction,
|
||
data_quality=data_quality,
|
||
)
|
||
assert 0.0 <= confidence <= 1.0
|
||
|
||
# With real positive signals we should get a non-trivial posterior
|
||
# (not exactly 0.5 since we have net-positive evidence)
|
||
assert posterior.p_up > 0.50, (
|
||
"Net-positive evidence should push P_up above 0.50"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Test 2: Feature flag routing — _annotate_pipeline_mode
|
||
# Requirements: 19.1, 19.5
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_annotate_pipeline_mode_v3():
|
||
"""_annotate_pipeline_mode sets pipeline_mode correctly for v3."""
|
||
from services.shared.schemas import TrendDirection, TrendSummary, TrendWindow
|
||
|
||
summary = TrendSummary(
|
||
entity_type="company",
|
||
entity_id="AAPL",
|
||
window=TrendWindow.SEVEN_DAY,
|
||
trend_direction=TrendDirection.BULLISH,
|
||
trend_strength=0.6,
|
||
confidence=0.7,
|
||
top_supporting_evidence=["doc1"],
|
||
top_opposing_evidence=[],
|
||
dominant_catalysts=["earnings"],
|
||
material_risks=[],
|
||
contradiction_score=0.1,
|
||
disagreement_details=[],
|
||
generated_at=_NOW,
|
||
)
|
||
# Initially no market_context
|
||
summary.market_context = {}
|
||
|
||
_annotate_pipeline_mode(summary, "v3")
|
||
assert summary.market_context["pipeline_mode"] == "v3"
|
||
|
||
|
||
def test_annotate_pipeline_mode_heuristic():
|
||
"""_annotate_pipeline_mode sets pipeline_mode correctly for heuristic."""
|
||
from services.shared.schemas import TrendDirection, TrendSummary, TrendWindow
|
||
|
||
summary = TrendSummary(
|
||
entity_type="company",
|
||
entity_id="AAPL",
|
||
window=TrendWindow.SEVEN_DAY,
|
||
trend_direction=TrendDirection.NEUTRAL,
|
||
trend_strength=0.0,
|
||
confidence=0.0,
|
||
top_supporting_evidence=[],
|
||
top_opposing_evidence=[],
|
||
dominant_catalysts=[],
|
||
material_risks=[],
|
||
contradiction_score=0.0,
|
||
disagreement_details=[],
|
||
generated_at=_NOW,
|
||
)
|
||
summary.market_context = {}
|
||
|
||
_annotate_pipeline_mode(summary, "heuristic")
|
||
assert summary.market_context["pipeline_mode"] == "heuristic"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Test 3: v3 metadata contains expected fields
|
||
# Requirements: 20.1, 20.2, 20.3
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_v3_metadata_contains_expected_fields():
|
||
"""Run through pipeline and verify output metadata dict contains v3 fields."""
|
||
# Build a simple pipeline run
|
||
signals = [
|
||
_make_signal(sentiment="positive", impact=0.7, source_id="a1"),
|
||
_make_signal(sentiment="negative", impact=0.4, source_id="a2", event_type="regulatory"),
|
||
]
|
||
units = [normalize_company_signal(s) for s in signals]
|
||
units = [u for u in units if u is not None]
|
||
|
||
neutral_stats = SourceStats(source_id="default", hits=0, misses=0)
|
||
q_values = []
|
||
for unit in units:
|
||
rel = compute_v3_reliability(
|
||
unit=unit,
|
||
source_stats=neutral_stats,
|
||
cluster_position=0,
|
||
reference_time=_NOW,
|
||
)
|
||
q_values.append(rel.q_i)
|
||
|
||
llrs = [compute_llr(u, q) for u, q in zip(units, q_values)]
|
||
clusters = cluster_evidence(units, llrs)
|
||
for c in clusters:
|
||
c.n_eff = compute_n_eff(c.llrs)
|
||
c.cluster_llr = compute_cluster_llr(c.llrs, c.n_eff)
|
||
|
||
posterior = compute_v3_posterior(clusters, _DEFAULT_REGIME, p_prior=0.50)
|
||
contradiction = compute_v3_contradiction(clusters)
|
||
data_quality = compute_v3_data_quality(
|
||
units=units,
|
||
extraction_failure_rate=0.0,
|
||
age_newest_hours=0.5,
|
||
n_source_types=1,
|
||
)
|
||
confidence = compute_v3_confidence(
|
||
n_eff_total=posterior.n_eff_total,
|
||
q_values=q_values,
|
||
llrs=llrs,
|
||
strength=posterior.strength,
|
||
regime_confidence_mult=_DEFAULT_REGIME.confidence_multiplier,
|
||
contradiction=contradiction,
|
||
data_quality=data_quality,
|
||
)
|
||
|
||
# Build the v3 metadata dict (mirrors _run_v3_pipeline logic)
|
||
v3_metadata = {
|
||
"v3_posterior": {
|
||
"p_up": round(posterior.p_up, 6),
|
||
"p_down": round(posterior.p_down, 6),
|
||
"log_odds": round(posterior.log_odds, 6),
|
||
"strength": round(posterior.strength, 6),
|
||
"confidence": round(confidence, 6),
|
||
"contradiction": round(contradiction, 6),
|
||
"n_eff": round(posterior.n_eff_total, 4),
|
||
"data_quality": round(data_quality, 6),
|
||
"regime": posterior.regime,
|
||
},
|
||
"pipeline_mode": "v3",
|
||
"explainability": {
|
||
"top_positive_clusters": [],
|
||
"top_negative_clusters": [],
|
||
"suppression_reasons": [],
|
||
"risk_adjustments": [],
|
||
},
|
||
}
|
||
|
||
# Verify all required v3_posterior keys exist
|
||
required_posterior_keys = {
|
||
"p_up", "p_down", "log_odds", "strength",
|
||
"confidence", "contradiction", "n_eff", "data_quality", "regime",
|
||
}
|
||
assert set(v3_metadata["v3_posterior"].keys()) == required_posterior_keys
|
||
|
||
# Verify top-level metadata keys
|
||
assert "pipeline_mode" in v3_metadata
|
||
assert v3_metadata["pipeline_mode"] == "v3"
|
||
assert "explainability" in v3_metadata
|
||
|
||
# Verify explainability structure
|
||
explainability = v3_metadata["explainability"]
|
||
assert "top_positive_clusters" in explainability
|
||
assert "top_negative_clusters" in explainability
|
||
assert "suppression_reasons" in explainability
|
||
assert "risk_adjustments" in explainability
|
||
|
||
# Verify numeric ranges
|
||
p = v3_metadata["v3_posterior"]
|
||
assert 0.0 < p["p_up"] < 1.0
|
||
assert 0.0 < p["p_down"] < 1.0
|
||
assert abs(p["p_up"] + p["p_down"] - 1.0) < 1e-5
|
||
assert 0.0 <= p["strength"] <= 1.0
|
||
assert 0.0 <= p["confidence"] <= 1.0
|
||
assert 0.0 <= p["contradiction"] <= 1.0
|
||
assert 0.0 <= p["data_quality"] <= 1.0
|
||
assert p["regime"] in ("panic", "trend_following", "mean_reversion", "uncertainty")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Test 4: Heuristic fallback function exists and is callable
|
||
# Requirements: 19.2, 19.4
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_heuristic_fallback_function_exists():
|
||
"""Verify the heuristic fallback function exists and is callable."""
|
||
from services.aggregation.worker import _aggregate_company_heuristic
|
||
|
||
assert callable(_aggregate_company_heuristic)
|
||
|
||
|
||
def test_v3_read_flag_function_exists():
|
||
"""Verify the _read_v3_flag async function exists and is callable."""
|
||
from services.aggregation.worker import _read_v3_flag
|
||
|
||
assert callable(_read_v3_flag)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Test 5: should_force_informational_v3 routing
|
||
# Requirements: 19.3, 19.4
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_force_informational_low_data_quality():
|
||
"""Low data quality should force informational mode."""
|
||
units = [
|
||
normalize_company_signal(_make_signal(source_id="x1")),
|
||
normalize_company_signal(_make_signal(source_id="x2")),
|
||
]
|
||
units = [u for u in units if u is not None]
|
||
|
||
should_force, reason = should_force_informational_v3(
|
||
data_quality=0.3,
|
||
units=units,
|
||
extraction_failure_rate=0.0,
|
||
)
|
||
assert should_force is True
|
||
assert reason == "data_quality_below_threshold"
|
||
|
||
|
||
def test_no_force_informational_good_quality():
|
||
"""Good data quality with sufficient evidence should NOT force informational."""
|
||
units = [
|
||
normalize_company_signal(_make_signal(source_id="x1")),
|
||
normalize_company_signal(_make_signal(source_id="x2")),
|
||
normalize_company_signal(_make_signal(source_id="x3")),
|
||
]
|
||
units = [u for u in units if u is not None]
|
||
|
||
should_force, reason = should_force_informational_v3(
|
||
data_quality=0.75,
|
||
units=units,
|
||
extraction_failure_rate=0.1,
|
||
)
|
||
assert should_force is False
|
||
assert reason == ""
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Test 6: Pipeline with all-neutral signals produces neutral posterior
|
||
# Requirements: 20.1, 20.4
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_neutral_signals_produce_neutral_posterior():
|
||
"""All neutral signals should produce posterior at P_up ≈ 0.50."""
|
||
signals = [
|
||
_make_signal(sentiment="neutral", impact=0.5, source_id="n1"),
|
||
_make_signal(sentiment="neutral", impact=0.3, source_id="n2"),
|
||
]
|
||
units = [normalize_company_signal(s) for s in signals]
|
||
units = [u for u in units if u is not None]
|
||
|
||
neutral_stats = SourceStats(source_id="default", hits=0, misses=0)
|
||
q_values = []
|
||
for unit in units:
|
||
rel = compute_v3_reliability(
|
||
unit=unit, source_stats=neutral_stats,
|
||
cluster_position=0, reference_time=_NOW,
|
||
)
|
||
q_values.append(rel.q_i)
|
||
|
||
llrs = [compute_llr(u, q) for u, q in zip(units, q_values)]
|
||
# All neutral → all LLRs should be 0
|
||
assert all(llr == 0.0 for llr in llrs), "Neutral signals should produce zero LLR"
|
||
|
||
clusters = cluster_evidence(units, llrs)
|
||
for c in clusters:
|
||
c.n_eff = compute_n_eff(c.llrs)
|
||
c.cluster_llr = compute_cluster_llr(c.llrs, c.n_eff)
|
||
|
||
posterior = compute_v3_posterior(clusters, _DEFAULT_REGIME, p_prior=0.50)
|
||
assert abs(posterior.p_up - 0.50) < 1e-6, (
|
||
f"Neutral evidence should maintain prior, got P_up={posterior.p_up}"
|
||
)
|
||
assert posterior.direction == "neutral"
|