feat: math core v3 engine upgrade

tests/test_v3_integration.py

@@ -0,0 +1,454 @@
+"""Integration tests for the v3 calibrated evidence pipeline.
+
+Tests the full pipeline path through pure functions end-to-end:
+    raw signals → EvidenceUnit → q_i → LLR → cluster → posterior → recommendation
+
+Also validates feature flag routing and v3 metadata fields.
+
+Requirements validated: 19.1–19.6, 20.1–20.5
+"""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from services.aggregation.bayesian import V3Posterior, compute_v3_posterior
+from services.aggregation.contradiction import compute_v3_contradiction
+from services.aggregation.regime import (
+    _DEFAULT_V3_UNCERTAINTY,
+)
+from services.aggregation.scoring import (
+    SourceStats,
+    compute_llr,
+    compute_v3_reliability,
+    normalize_company_signal,
+)
+from services.aggregation.worker import (
+    _annotate_pipeline_mode,
+    cluster_evidence,
+    compute_cluster_llr,
+    compute_n_eff,
+    compute_v3_confidence,
+    compute_v3_data_quality,
+    should_force_informational_v3,
+)
+
+# ---------------------------------------------------------------------------
+# Fixtures / helpers
+# ---------------------------------------------------------------------------
+
+_NOW = datetime(2025, 1, 15, 12, 0, 0, tzinfo=timezone.utc)
+
+_DEFAULT_REGIME = _DEFAULT_V3_UNCERTAINTY
+
+
+def _make_signal(
+    sentiment: str = "positive",
+    impact: float = 0.7,
+    source_id: str = "doc1",
+    event_type: str = "earnings",
+    source_group: str = "company",
+    extraction_conf: float = 0.85,
+    source_cred: float = 0.80,
+    novelty: float = 0.7,
+) -> dict:
+    """Build a raw company signal dict for normalization."""
+    return {
+        "symbol": "AAPL",
+        "timestamp": _NOW,
+        "source_id": source_id,
+        "event_type": event_type,
+        "source_group": source_group,
+        "horizon": "7d",
+        "sentiment": sentiment,
+        "sentiment_strength": impact,
+        "impact": impact,
+        "extraction_conf": extraction_conf,
+        "source_cred": source_cred,
+        "novelty": novelty,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Test 1: Full pipeline path through pure functions
+# Requirements: 20.1, 20.2, 20.3, 20.4, 20.5
+# ---------------------------------------------------------------------------
+
+
+def test_full_pipeline_path():
+    """End-to-end test: raw signals → EvidenceUnit → q_i → LLR → cluster → posterior."""
+    # 1. Create raw signal dicts with opposing sentiments
+    signals = [
+        _make_signal(
+            sentiment="positive",
+            impact=0.8,
+            source_id="doc1",
+            event_type="earnings",
+            extraction_conf=0.9,
+            source_cred=0.85,
+            novelty=0.8,
+        ),
+        _make_signal(
+            sentiment="positive",
+            impact=0.6,
+            source_id="doc2",
+            event_type="earnings",
+            extraction_conf=0.75,
+            source_cred=0.7,
+            novelty=0.6,
+        ),
+        _make_signal(
+            sentiment="negative",
+            impact=0.5,
+            source_id="doc3",
+            event_type="regulatory",
+            extraction_conf=0.7,
+            source_cred=0.6,
+            novelty=0.9,
+        ),
+    ]
+
+    # 2. Normalize to EvidenceUnit
+    units = [normalize_company_signal(s) for s in signals]
+    assert all(u is not None for u in units), "All signals should normalize successfully"
+    units = [u for u in units if u is not None]  # type narrowing
+    assert len(units) == 3
+
+    # 3. Compute q_i (reliability) for each unit
+    neutral_stats = SourceStats(source_id="default", hits=0, misses=0)
+    q_values = []
+    for unit in units:
+        reliability = compute_v3_reliability(
+            unit=unit,
+            source_stats=neutral_stats,
+            cluster_position=0,
+            reference_time=_NOW,
+        )
+        q_values.append(reliability.q_i)
+
+    # All q_i should be in [0, 1]
+    for q in q_values:
+        assert 0.0 <= q <= 1.0, f"q_i out of bounds: {q}"
+
+    # 4. Compute LLR for each unit
+    llrs = [compute_llr(unit, q) for unit, q in zip(units, q_values)]
+    # Positive sentiment → positive LLR, negative → negative LLR
+    assert llrs[0] > 0.0, "Positive signal should produce positive LLR"
+    assert llrs[1] > 0.0, "Positive signal should produce positive LLR"
+    assert llrs[2] < 0.0, "Negative signal should produce negative LLR"
+
+    # 5. Cluster evidence
+    clusters = cluster_evidence(units, llrs)
+    assert len(clusters) >= 1, "Should produce at least one cluster"
+
+    # 6. Compute n_eff and cluster_llr for each cluster
+    for cluster in clusters:
+        cluster.n_eff = compute_n_eff(cluster.llrs)
+        cluster.cluster_llr = compute_cluster_llr(cluster.llrs, cluster.n_eff)
+        assert cluster.n_eff >= 1.0, "n_eff should be >= 1.0"
+        assert -2.5 <= cluster.cluster_llr <= 2.5, "cluster_llr should be clamped"
+
+    # 7. Compute posterior (using default uncertainty regime)
+    posterior = compute_v3_posterior(clusters, _DEFAULT_REGIME, p_prior=0.50)
+    assert isinstance(posterior, V3Posterior)
+
+    # 8. Assert posterior is valid
+    assert 0.0 < posterior.p_up < 1.0, f"P_up should be in (0, 1), got {posterior.p_up}"
+    assert 0.0 < posterior.p_down < 1.0, "P_down should be in (0, 1)"
+    assert abs(posterior.p_up + posterior.p_down - 1.0) < 1e-9
+    assert 0.0 <= posterior.strength <= 1.0
+    assert posterior.direction in ("bullish", "bearish", "neutral")
+    assert posterior.n_eff_total > 0
+
+    # 9. Compute contradiction
+    contradiction = compute_v3_contradiction(clusters)
+    # 10. Assert contradiction is bounded
+    assert 0.0 <= contradiction <= 1.0
+
+    # 11. Compute data quality
+    data_quality = compute_v3_data_quality(
+        units=units,
+        extraction_failure_rate=0.0,
+        age_newest_hours=0.0,
+        n_source_types=1,
+    )
+    assert 0.0 <= data_quality <= 1.0
+
+    # 12. Compute confidence
+    confidence = compute_v3_confidence(
+        n_eff_total=posterior.n_eff_total,
+        q_values=q_values,
+        llrs=llrs,
+        strength=posterior.strength,
+        regime_confidence_mult=_DEFAULT_REGIME.confidence_multiplier,
+        contradiction=contradiction,
+        data_quality=data_quality,
+    )
+    assert 0.0 <= confidence <= 1.0
+
+    # With real positive signals we should get a non-trivial posterior
+    # (not exactly 0.5 since we have net-positive evidence)
+    assert posterior.p_up > 0.50, (
+        "Net-positive evidence should push P_up above 0.50"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Test 2: Feature flag routing — _annotate_pipeline_mode
+# Requirements: 19.1, 19.5
+# ---------------------------------------------------------------------------
+
+
+def test_annotate_pipeline_mode_v3():
+    """_annotate_pipeline_mode sets pipeline_mode correctly for v3."""
+    from services.shared.schemas import TrendDirection, TrendSummary, TrendWindow
+
+    summary = TrendSummary(
+        entity_type="company",
+        entity_id="AAPL",
+        window=TrendWindow.SEVEN_DAY,
+        trend_direction=TrendDirection.BULLISH,
+        trend_strength=0.6,
+        confidence=0.7,
+        top_supporting_evidence=["doc1"],
+        top_opposing_evidence=[],
+        dominant_catalysts=["earnings"],
+        material_risks=[],
+        contradiction_score=0.1,
+        disagreement_details=[],
+        generated_at=_NOW,
+    )
+    # Initially no market_context
+    summary.market_context = {}
+
+    _annotate_pipeline_mode(summary, "v3")
+    assert summary.market_context["pipeline_mode"] == "v3"
+
+
+def test_annotate_pipeline_mode_heuristic():
+    """_annotate_pipeline_mode sets pipeline_mode correctly for heuristic."""
+    from services.shared.schemas import TrendDirection, TrendSummary, TrendWindow
+
+    summary = TrendSummary(
+        entity_type="company",
+        entity_id="AAPL",
+        window=TrendWindow.SEVEN_DAY,
+        trend_direction=TrendDirection.NEUTRAL,
+        trend_strength=0.0,
+        confidence=0.0,
+        top_supporting_evidence=[],
+        top_opposing_evidence=[],
+        dominant_catalysts=[],
+        material_risks=[],
+        contradiction_score=0.0,
+        disagreement_details=[],
+        generated_at=_NOW,
+    )
+    summary.market_context = {}
+
+    _annotate_pipeline_mode(summary, "heuristic")
+    assert summary.market_context["pipeline_mode"] == "heuristic"
+
+
+# ---------------------------------------------------------------------------
+# Test 3: v3 metadata contains expected fields
+# Requirements: 20.1, 20.2, 20.3
+# ---------------------------------------------------------------------------
+
+
+def test_v3_metadata_contains_expected_fields():
+    """Run through pipeline and verify output metadata dict contains v3 fields."""
+    # Build a simple pipeline run
+    signals = [
+        _make_signal(sentiment="positive", impact=0.7, source_id="a1"),
+        _make_signal(sentiment="negative", impact=0.4, source_id="a2", event_type="regulatory"),
+    ]
+    units = [normalize_company_signal(s) for s in signals]
+    units = [u for u in units if u is not None]
+
+    neutral_stats = SourceStats(source_id="default", hits=0, misses=0)
+    q_values = []
+    for unit in units:
+        rel = compute_v3_reliability(
+            unit=unit,
+            source_stats=neutral_stats,
+            cluster_position=0,
+            reference_time=_NOW,
+        )
+        q_values.append(rel.q_i)
+
+    llrs = [compute_llr(u, q) for u, q in zip(units, q_values)]
+    clusters = cluster_evidence(units, llrs)
+    for c in clusters:
+        c.n_eff = compute_n_eff(c.llrs)
+        c.cluster_llr = compute_cluster_llr(c.llrs, c.n_eff)
+
+    posterior = compute_v3_posterior(clusters, _DEFAULT_REGIME, p_prior=0.50)
+    contradiction = compute_v3_contradiction(clusters)
+    data_quality = compute_v3_data_quality(
+        units=units,
+        extraction_failure_rate=0.0,
+        age_newest_hours=0.5,
+        n_source_types=1,
+    )
+    confidence = compute_v3_confidence(
+        n_eff_total=posterior.n_eff_total,
+        q_values=q_values,
+        llrs=llrs,
+        strength=posterior.strength,
+        regime_confidence_mult=_DEFAULT_REGIME.confidence_multiplier,
+        contradiction=contradiction,
+        data_quality=data_quality,
+    )
+
+    # Build the v3 metadata dict (mirrors _run_v3_pipeline logic)
+    v3_metadata = {
+        "v3_posterior": {
+            "p_up": round(posterior.p_up, 6),
+            "p_down": round(posterior.p_down, 6),
+            "log_odds": round(posterior.log_odds, 6),
+            "strength": round(posterior.strength, 6),
+            "confidence": round(confidence, 6),
+            "contradiction": round(contradiction, 6),
+            "n_eff": round(posterior.n_eff_total, 4),
+            "data_quality": round(data_quality, 6),
+            "regime": posterior.regime,
+        },
+        "pipeline_mode": "v3",
+        "explainability": {
+            "top_positive_clusters": [],
+            "top_negative_clusters": [],
+            "suppression_reasons": [],
+            "risk_adjustments": [],
+        },
+    }
+
+    # Verify all required v3_posterior keys exist
+    required_posterior_keys = {
+        "p_up", "p_down", "log_odds", "strength",
+        "confidence", "contradiction", "n_eff", "data_quality", "regime",
+    }
+    assert set(v3_metadata["v3_posterior"].keys()) == required_posterior_keys
+
+    # Verify top-level metadata keys
+    assert "pipeline_mode" in v3_metadata
+    assert v3_metadata["pipeline_mode"] == "v3"
+    assert "explainability" in v3_metadata
+
+    # Verify explainability structure
+    explainability = v3_metadata["explainability"]
+    assert "top_positive_clusters" in explainability
+    assert "top_negative_clusters" in explainability
+    assert "suppression_reasons" in explainability
+    assert "risk_adjustments" in explainability
+
+    # Verify numeric ranges
+    p = v3_metadata["v3_posterior"]
+    assert 0.0 < p["p_up"] < 1.0
+    assert 0.0 < p["p_down"] < 1.0
+    assert abs(p["p_up"] + p["p_down"] - 1.0) < 1e-5
+    assert 0.0 <= p["strength"] <= 1.0
+    assert 0.0 <= p["confidence"] <= 1.0
+    assert 0.0 <= p["contradiction"] <= 1.0
+    assert 0.0 <= p["data_quality"] <= 1.0
+    assert p["regime"] in ("panic", "trend_following", "mean_reversion", "uncertainty")
+
+
+# ---------------------------------------------------------------------------
+# Test 4: Heuristic fallback function exists and is callable
+# Requirements: 19.2, 19.4
+# ---------------------------------------------------------------------------
+
+
+def test_heuristic_fallback_function_exists():
+    """Verify the heuristic fallback function exists and is callable."""
+    from services.aggregation.worker import _aggregate_company_heuristic
+
+    assert callable(_aggregate_company_heuristic)
+
+
+def test_v3_read_flag_function_exists():
+    """Verify the _read_v3_flag async function exists and is callable."""
+    from services.aggregation.worker import _read_v3_flag
+
+    assert callable(_read_v3_flag)
+
+
+# ---------------------------------------------------------------------------
+# Test 5: should_force_informational_v3 routing
+# Requirements: 19.3, 19.4
+# ---------------------------------------------------------------------------
+
+
+def test_force_informational_low_data_quality():
+    """Low data quality should force informational mode."""
+    units = [
+        normalize_company_signal(_make_signal(source_id="x1")),
+        normalize_company_signal(_make_signal(source_id="x2")),
+    ]
+    units = [u for u in units if u is not None]
+
+    should_force, reason = should_force_informational_v3(
+        data_quality=0.3,
+        units=units,
+        extraction_failure_rate=0.0,
+    )
+    assert should_force is True
+    assert reason == "data_quality_below_threshold"
+
+
+def test_no_force_informational_good_quality():
+    """Good data quality with sufficient evidence should NOT force informational."""
+    units = [
+        normalize_company_signal(_make_signal(source_id="x1")),
+        normalize_company_signal(_make_signal(source_id="x2")),
+        normalize_company_signal(_make_signal(source_id="x3")),
+    ]
+    units = [u for u in units if u is not None]
+
+    should_force, reason = should_force_informational_v3(
+        data_quality=0.75,
+        units=units,
+        extraction_failure_rate=0.1,
+    )
+    assert should_force is False
+    assert reason == ""
+
+
+# ---------------------------------------------------------------------------
+# Test 6: Pipeline with all-neutral signals produces neutral posterior
+# Requirements: 20.1, 20.4
+# ---------------------------------------------------------------------------
+
+
+def test_neutral_signals_produce_neutral_posterior():
+    """All neutral signals should produce posterior at P_up ≈ 0.50."""
+    signals = [
+        _make_signal(sentiment="neutral", impact=0.5, source_id="n1"),
+        _make_signal(sentiment="neutral", impact=0.3, source_id="n2"),
+    ]
+    units = [normalize_company_signal(s) for s in signals]
+    units = [u for u in units if u is not None]
+
+    neutral_stats = SourceStats(source_id="default", hits=0, misses=0)
+    q_values = []
+    for unit in units:
+        rel = compute_v3_reliability(
+            unit=unit, source_stats=neutral_stats,
+            cluster_position=0, reference_time=_NOW,
+        )
+        q_values.append(rel.q_i)
+
+    llrs = [compute_llr(u, q) for u, q in zip(units, q_values)]
+    # All neutral → all LLRs should be 0
+    assert all(llr == 0.0 for llr in llrs), "Neutral signals should produce zero LLR"
+
+    clusters = cluster_evidence(units, llrs)
+    for c in clusters:
+        c.n_eff = compute_n_eff(c.llrs)
+        c.cluster_llr = compute_cluster_llr(c.llrs, c.n_eff)
+
+    posterior = compute_v3_posterior(clusters, _DEFAULT_REGIME, p_prior=0.50)
+    assert abs(posterior.p_up - 0.50) < 1e-6, (
+        f"Neutral evidence should maintain prior, got P_up={posterior.p_up}"
+    )
+    assert posterior.direction == "neutral"