"""Report validator — cross-checks computed metrics against live data. Compares report section values against prediction_outcomes and model_metric_snapshots, flagging discrepancies that exceed the configured threshold. """ from __future__ import annotations import logging import math from services.reporting.models import ( ModelQualitySection, RecommendationAccuracySection, ReportData, ValidationStatus, ValidationWarning, ) logger = logging.getLogger(__name__) DISCREPANCY_THRESHOLD_PCT = 5.0 def _sanitize(value: float | None) -> float: """Replace None, NaN, and infinity with 0.0.""" if value is None: return 0.0 if math.isnan(value) or math.isinf(value): return 0.0 return value def _check_discrepancy( field_name: str, computed: float, snapshot: float, ) -> ValidationWarning | None: """Compare computed vs snapshot and return a warning if >5% discrepancy. Edge cases: - snapshot=0 and computed≠0 → 100% difference → warning - both=0 → 0% difference → no warning - snapshot is handled upstream (NULL → skip before calling this) """ computed = _sanitize(computed) snapshot = _sanitize(snapshot) if snapshot == 0.0 and computed == 0.0: return None if snapshot == 0.0: # Non-zero computed with zero snapshot → 100% discrepancy pct_diff = 100.0 else: pct_diff = abs(computed - snapshot) / abs(snapshot) * 100.0 if pct_diff > DISCREPANCY_THRESHOLD_PCT: return ValidationWarning( field_name=field_name, computed_value=computed, snapshot_value=snapshot, pct_difference=round(pct_diff, 4), ) return None def validate_recommendation_accuracy( section: RecommendationAccuracySection, prediction_outcomes: list[dict], ) -> list[ValidationWarning]: """Cross-reference reported win rates with prediction_outcomes. Computes win_rate from prediction_outcomes (count profitable / total) and compares against section.acted_win_rate. Returns warnings for discrepancies > 5%. """ warnings: list[ValidationWarning] = [] if not prediction_outcomes: return warnings total = len(prediction_outcomes) profitable_count = sum( 1 for po in prediction_outcomes if po.get("profitable") ) computed_win_rate = profitable_count / total if total > 0 else 0.0 w = _check_discrepancy( "acted_win_rate", section.acted_win_rate, computed_win_rate, ) if w is not None: warnings.append(w) return warnings def validate_model_quality( section: ModelQualitySection, metric_snapshots: list[dict], ) -> list[ValidationWarning]: """Compare reported model quality metrics against model_metric_snapshots. For each window in the section, finds the matching snapshot by lookback_window and compares win_rate, directional_accuracy, information_coefficient, calibration_error, and brier_score. Flags discrepancies > 5%. """ warnings: list[ValidationWarning] = [] if not metric_snapshots: return warnings # Build lookup: lookback_window → latest snapshot (first match since # collector orders by generated_at DESC) snap_by_window: dict[str, dict] = {} for snap in metric_snapshots: window = snap.get("lookback_window", "") if window and window not in snap_by_window: snap_by_window[window] = snap metric_fields = [ ("win_rate", "win_rate"), ("directional_accuracy", "directional_accuracy"), ("information_coefficient", "information_coefficient"), ("calibration_error", "calibration_error"), ("brier_score", "brier_score"), ] for mq_window in section.windows: snap = snap_by_window.get(mq_window.lookback) if snap is None: continue for section_attr, snap_key in metric_fields: section_value = getattr(mq_window, section_attr, None) snapshot_value = snap.get(snap_key) # NULL snapshot → skip if snapshot_value is None: continue # NULL section value → skip if section_value is None: continue snapshot_float = _sanitize(float(snapshot_value)) section_float = _sanitize(section_value) w = _check_discrepancy( f"{mq_window.lookback}_{section_attr}", section_float, snapshot_float, ) if w is not None: warnings.append(w) return warnings def compute_validation_status(report: ReportData) -> ValidationStatus: """Determine overall validation status. Returns 'passed' if no warnings across all sections, 'warnings' if any section has validation warnings. """ if report.pnl.validation_warnings: return ValidationStatus.WARNINGS if report.recommendation_accuracy.validation_warnings: return ValidationStatus.WARNINGS if report.model_quality.validation_warnings: return ValidationStatus.WARNINGS return ValidationStatus.PASSED