stonks-oracle/services/reporting/validator.py

"""Report validator — cross-checks computed metrics against live data.

Compares report section values against prediction_outcomes and
model_metric_snapshots, flagging discrepancies that exceed the
configured threshold.
"""

from __future__ import annotations

import logging
import math

from services.reporting.models import (
    ModelQualitySection,
    RecommendationAccuracySection,
    ReportData,
    ValidationStatus,
    ValidationWarning,
)

logger = logging.getLogger(__name__)

DISCREPANCY_THRESHOLD_PCT = 5.0


def _sanitize(value: float | None) -> float:
    """Replace None, NaN, and infinity with 0.0."""
    if value is None:
        return 0.0
    if math.isnan(value) or math.isinf(value):
        return 0.0
    return value


def _check_discrepancy(
    field_name: str,
    computed: float,
    snapshot: float,
) -> ValidationWarning | None:
    """Compare computed vs snapshot and return a warning if >5% discrepancy.

    Edge cases:
    - snapshot=0 and computed≠0 → 100% difference → warning
    - both=0 → 0% difference → no warning
    - snapshot is handled upstream (NULL → skip before calling this)
    """
    computed = _sanitize(computed)
    snapshot = _sanitize(snapshot)

    if snapshot == 0.0 and computed == 0.0:
        return None

    if snapshot == 0.0:
        # Non-zero computed with zero snapshot → 100% discrepancy
        pct_diff = 100.0
    else:
        pct_diff = abs(computed - snapshot) / abs(snapshot) * 100.0

    if pct_diff > DISCREPANCY_THRESHOLD_PCT:
        return ValidationWarning(
            field_name=field_name,
            computed_value=computed,
            snapshot_value=snapshot,
            pct_difference=round(pct_diff, 4),
        )
    return None


def validate_recommendation_accuracy(
    section: RecommendationAccuracySection,
    prediction_outcomes: list[dict],
) -> list[ValidationWarning]:
    """Cross-reference reported win rates with prediction_outcomes.

    Computes win_rate from prediction_outcomes (count profitable / total)
    and compares against section.acted_win_rate.  Returns warnings for
    discrepancies > 5%.
    """
    warnings: list[ValidationWarning] = []

    if not prediction_outcomes:
        return warnings

    total = len(prediction_outcomes)
    profitable_count = sum(
        1 for po in prediction_outcomes if po.get("profitable")
    )
    computed_win_rate = profitable_count / total if total > 0 else 0.0

    w = _check_discrepancy(
        "acted_win_rate",
        section.acted_win_rate,
        computed_win_rate,
    )
    if w is not None:
        warnings.append(w)

    return warnings


def validate_model_quality(
    section: ModelQualitySection,
    metric_snapshots: list[dict],
) -> list[ValidationWarning]:
    """Compare reported model quality metrics against model_metric_snapshots.

    For each window in the section, finds the matching snapshot by
    lookback_window and compares win_rate, directional_accuracy,
    information_coefficient, calibration_error, and brier_score.
    Flags discrepancies > 5%.
    """
    warnings: list[ValidationWarning] = []

    if not metric_snapshots:
        return warnings

    # Build lookup: lookback_window → latest snapshot (first match since
    # collector orders by generated_at DESC)
    snap_by_window: dict[str, dict] = {}
    for snap in metric_snapshots:
        window = snap.get("lookback_window", "")
        if window and window not in snap_by_window:
            snap_by_window[window] = snap

    metric_fields = [
        ("win_rate", "win_rate"),
        ("directional_accuracy", "directional_accuracy"),
        ("information_coefficient", "information_coefficient"),
        ("calibration_error", "calibration_error"),
        ("brier_score", "brier_score"),
    ]

    for mq_window in section.windows:
        snap = snap_by_window.get(mq_window.lookback)
        if snap is None:
            continue

        for section_attr, snap_key in metric_fields:
            section_value = getattr(mq_window, section_attr, None)
            snapshot_value = snap.get(snap_key)

            # NULL snapshot → skip
            if snapshot_value is None:
                continue
            # NULL section value → skip
            if section_value is None:
                continue

            snapshot_float = _sanitize(float(snapshot_value))
            section_float = _sanitize(section_value)

            w = _check_discrepancy(
                f"{mq_window.lookback}_{section_attr}",
                section_float,
                snapshot_float,
            )
            if w is not None:
                warnings.append(w)

    return warnings


def compute_validation_status(report: ReportData) -> ValidationStatus:
    """Determine overall validation status.

    Returns 'passed' if no warnings across all sections,
    'warnings' if any section has validation warnings.
    """
    if report.pnl.validation_warnings:
        return ValidationStatus.WARNINGS
    if report.recommendation_accuracy.validation_warnings:
        return ValidationStatus.WARNINGS
    if report.model_quality.validation_warnings:
        return ValidationStatus.WARNINGS
    return ValidationStatus.PASSED