feat: model validation, calibration, and signal quality layer

- Migration 035: prediction_snapshots, prediction_outcomes, signal_evidence_links, model_metric_snapshots tables + SQL views
- Prediction snapshot writer with canonical evidence keys, duplicate detection, contribution scores
- Outcome evaluator across 5 horizons (1h, 6h, 1d, 7d, 30d)
- Metrics engine: ECE, Brier score, IC, Rank IC, benchmark comparison
- Attribution engine: per-source, per-catalyst, per-layer performance
- Calibration engine: Bayesian shrinkage source reliability
- Quality gate for live trading eligibility with configurable thresholds
- 7 new /api/validation/* endpoints
- Upgraded OpsModel dashboard with validation tab
- Enhanced recommendation display with calibration context
- Backtest replay validation mode
- 86 Python tests (unit + property-based), 179 frontend tests passing
This commit is contained in:
Celes Renata
2026-05-01 03:04:58 +00:00
parent 5d2ffd9163
commit 7fcc8a6c07
23 changed files with 7554 additions and 9 deletions
+100
View File
@@ -73,6 +73,97 @@ export const mockVariantPerfHistory = [
{ hour: '2026-04-10T11:00:00Z', invocations: 12, successes: 11, avg_duration_ms: 1300, avg_confidence: 0.82 },
];
// Validation: Model Quality & Calibration mock data
export const mockValidationSummary = {
snapshot: {
id: 'ms-1',
generated_at: '2026-04-11T12:00:00Z',
lookback_window: '30d',
horizon: '7d',
prediction_count: 150,
win_rate: 0.58,
directional_accuracy: 0.56,
information_coefficient: 0.045,
rank_information_coefficient: 0.038,
avg_return: 0.012,
avg_excess_return_vs_spy: 0.003,
avg_excess_return_vs_sector: 0.002,
calibration_error: 0.08,
brier_score: 0.21,
buy_win_rate: 0.61,
sell_win_rate: 0.54,
hold_win_rate: 0.50,
metadata: {},
},
gate_status: {
passed: true,
reason: 'all thresholds met',
threshold_results: [
{ name: 'min_prediction_count', threshold: 100, actual: 150, passed: true },
{ name: 'min_ic', threshold: 0.03, actual: 0.045, passed: true },
{ name: 'min_win_rate', threshold: 0.53, actual: 0.58, passed: true },
],
},
};
export const mockValidationCalibration = {
buckets: [
{ bucket_low: 0.50, bucket_high: 0.60, avg_confidence: 0.55, observed_win_rate: 0.52, prediction_count: 30, miscalibrated: false },
{ bucket_low: 0.60, bucket_high: 0.70, avg_confidence: 0.65, observed_win_rate: 0.58, prediction_count: 40, miscalibrated: false },
{ bucket_low: 0.70, bucket_high: 0.80, avg_confidence: 0.75, observed_win_rate: 0.55, prediction_count: 35, miscalibrated: true },
{ bucket_low: 0.80, bucket_high: 0.90, avg_confidence: 0.85, observed_win_rate: 0.70, prediction_count: 25, miscalibrated: false },
{ bucket_low: 0.90, bucket_high: 1.00, avg_confidence: 0.95, observed_win_rate: 0.72, prediction_count: 20, miscalibrated: true },
],
lookback: '30d',
horizon: '7d',
};
export const mockValidationGateStatus = {
gate_status: {
passed: false,
reason: 'failed: min_ic below threshold',
threshold_results: [
{ name: 'min_prediction_count', threshold: 100, actual: 150, passed: true },
{ name: 'min_ic', threshold: 0.03, actual: 0.02, passed: false },
{ name: 'min_win_rate', threshold: 0.53, actual: 0.58, passed: true },
],
},
};
export const mockValidationICByHorizon = {
horizons: [
{ horizon: '1h', information_coefficient: 0.02, rank_information_coefficient: 0.015, prediction_count: 120, generated_at: '2026-04-11T12:00:00Z' },
{ horizon: '7d', information_coefficient: 0.045, rank_information_coefficient: 0.038, prediction_count: 100, generated_at: '2026-04-11T12:00:00Z' },
],
lookback: '30d',
};
export const mockValidationAttributionSources = {
sources: [
{ source: 'Reuters', source_type: 'news_api', prediction_count: 50, avg_weight: 0.6, avg_contribution_score: 0.3, win_rate: 0.62, avg_future_return: 0.015, avg_excess_return_vs_spy: 0.005, information_coefficient: 0.05, duplicate_rate: 0.1 },
],
lookback: '30d',
horizon: '7d',
};
export const mockValidationAttributionCatalysts = {
catalysts: [
{ catalyst_type: 'earnings', prediction_count: 40, win_rate: 0.65, avg_future_return: 0.02, avg_excess_return_vs_spy: 0.008, information_coefficient: 0.06 },
],
lookback: '30d',
horizon: '7d',
};
export const mockValidationAttributionLayers = {
layers: [
{ layer: 'company', avg_contribution_pct: 0.55, dominant_win_rate: 0.60, dominant_ic: 0.04 },
{ layer: 'macro', avg_contribution_pct: 0.30, dominant_win_rate: 0.52, dominant_ic: 0.02 },
{ layer: 'competitive', avg_contribution_pct: 0.15, dominant_win_rate: 0.48, dominant_ic: null },
],
lookback: '30d',
horizon: '7d',
};
export const handlers = [
// Query API (proxied at /api/)
http.get('/api/companies', () => HttpResponse.json(mockCompanies)),
@@ -242,4 +333,13 @@ export const handlers = [
const body = await request.json() as Record<string, unknown>;
return HttpResponse.json({ enabled: body.enabled, previous_enabled: true, toggled_by: 'operator' });
}),
// Validation: Model Quality & Calibration endpoints
http.get('/api/validation/summary', () => HttpResponse.json(mockValidationSummary)),
http.get('/api/validation/calibration', () => HttpResponse.json(mockValidationCalibration)),
http.get('/api/validation/gate-status', () => HttpResponse.json(mockValidationGateStatus)),
http.get('/api/validation/ic-by-horizon', () => HttpResponse.json(mockValidationICByHorizon)),
http.get('/api/validation/attribution/sources', () => HttpResponse.json(mockValidationAttributionSources)),
http.get('/api/validation/attribution/catalysts', () => HttpResponse.json(mockValidationAttributionCatalysts)),
http.get('/api/validation/attribution/layers', () => HttpResponse.json(mockValidationAttributionLayers)),
];