feat: model validation, calibration, and signal quality layer
- Migration 035: prediction_snapshots, prediction_outcomes, signal_evidence_links, model_metric_snapshots tables + SQL views - Prediction snapshot writer with canonical evidence keys, duplicate detection, contribution scores - Outcome evaluator across 5 horizons (1h, 6h, 1d, 7d, 30d) - Metrics engine: ECE, Brier score, IC, Rank IC, benchmark comparison - Attribution engine: per-source, per-catalyst, per-layer performance - Calibration engine: Bayesian shrinkage source reliability - Quality gate for live trading eligibility with configurable thresholds - 7 new /api/validation/* endpoints - Upgraded OpsModel dashboard with validation tab - Enhanced recommendation display with calibration context - Backtest replay validation mode - 86 Python tests (unit + property-based), 179 frontend tests passing
This commit is contained in:
@@ -73,6 +73,97 @@ export const mockVariantPerfHistory = [
|
||||
{ hour: '2026-04-10T11:00:00Z', invocations: 12, successes: 11, avg_duration_ms: 1300, avg_confidence: 0.82 },
|
||||
];
|
||||
|
||||
// Validation: Model Quality & Calibration mock data
|
||||
export const mockValidationSummary = {
|
||||
snapshot: {
|
||||
id: 'ms-1',
|
||||
generated_at: '2026-04-11T12:00:00Z',
|
||||
lookback_window: '30d',
|
||||
horizon: '7d',
|
||||
prediction_count: 150,
|
||||
win_rate: 0.58,
|
||||
directional_accuracy: 0.56,
|
||||
information_coefficient: 0.045,
|
||||
rank_information_coefficient: 0.038,
|
||||
avg_return: 0.012,
|
||||
avg_excess_return_vs_spy: 0.003,
|
||||
avg_excess_return_vs_sector: 0.002,
|
||||
calibration_error: 0.08,
|
||||
brier_score: 0.21,
|
||||
buy_win_rate: 0.61,
|
||||
sell_win_rate: 0.54,
|
||||
hold_win_rate: 0.50,
|
||||
metadata: {},
|
||||
},
|
||||
gate_status: {
|
||||
passed: true,
|
||||
reason: 'all thresholds met',
|
||||
threshold_results: [
|
||||
{ name: 'min_prediction_count', threshold: 100, actual: 150, passed: true },
|
||||
{ name: 'min_ic', threshold: 0.03, actual: 0.045, passed: true },
|
||||
{ name: 'min_win_rate', threshold: 0.53, actual: 0.58, passed: true },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
export const mockValidationCalibration = {
|
||||
buckets: [
|
||||
{ bucket_low: 0.50, bucket_high: 0.60, avg_confidence: 0.55, observed_win_rate: 0.52, prediction_count: 30, miscalibrated: false },
|
||||
{ bucket_low: 0.60, bucket_high: 0.70, avg_confidence: 0.65, observed_win_rate: 0.58, prediction_count: 40, miscalibrated: false },
|
||||
{ bucket_low: 0.70, bucket_high: 0.80, avg_confidence: 0.75, observed_win_rate: 0.55, prediction_count: 35, miscalibrated: true },
|
||||
{ bucket_low: 0.80, bucket_high: 0.90, avg_confidence: 0.85, observed_win_rate: 0.70, prediction_count: 25, miscalibrated: false },
|
||||
{ bucket_low: 0.90, bucket_high: 1.00, avg_confidence: 0.95, observed_win_rate: 0.72, prediction_count: 20, miscalibrated: true },
|
||||
],
|
||||
lookback: '30d',
|
||||
horizon: '7d',
|
||||
};
|
||||
|
||||
export const mockValidationGateStatus = {
|
||||
gate_status: {
|
||||
passed: false,
|
||||
reason: 'failed: min_ic below threshold',
|
||||
threshold_results: [
|
||||
{ name: 'min_prediction_count', threshold: 100, actual: 150, passed: true },
|
||||
{ name: 'min_ic', threshold: 0.03, actual: 0.02, passed: false },
|
||||
{ name: 'min_win_rate', threshold: 0.53, actual: 0.58, passed: true },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
export const mockValidationICByHorizon = {
|
||||
horizons: [
|
||||
{ horizon: '1h', information_coefficient: 0.02, rank_information_coefficient: 0.015, prediction_count: 120, generated_at: '2026-04-11T12:00:00Z' },
|
||||
{ horizon: '7d', information_coefficient: 0.045, rank_information_coefficient: 0.038, prediction_count: 100, generated_at: '2026-04-11T12:00:00Z' },
|
||||
],
|
||||
lookback: '30d',
|
||||
};
|
||||
|
||||
export const mockValidationAttributionSources = {
|
||||
sources: [
|
||||
{ source: 'Reuters', source_type: 'news_api', prediction_count: 50, avg_weight: 0.6, avg_contribution_score: 0.3, win_rate: 0.62, avg_future_return: 0.015, avg_excess_return_vs_spy: 0.005, information_coefficient: 0.05, duplicate_rate: 0.1 },
|
||||
],
|
||||
lookback: '30d',
|
||||
horizon: '7d',
|
||||
};
|
||||
|
||||
export const mockValidationAttributionCatalysts = {
|
||||
catalysts: [
|
||||
{ catalyst_type: 'earnings', prediction_count: 40, win_rate: 0.65, avg_future_return: 0.02, avg_excess_return_vs_spy: 0.008, information_coefficient: 0.06 },
|
||||
],
|
||||
lookback: '30d',
|
||||
horizon: '7d',
|
||||
};
|
||||
|
||||
export const mockValidationAttributionLayers = {
|
||||
layers: [
|
||||
{ layer: 'company', avg_contribution_pct: 0.55, dominant_win_rate: 0.60, dominant_ic: 0.04 },
|
||||
{ layer: 'macro', avg_contribution_pct: 0.30, dominant_win_rate: 0.52, dominant_ic: 0.02 },
|
||||
{ layer: 'competitive', avg_contribution_pct: 0.15, dominant_win_rate: 0.48, dominant_ic: null },
|
||||
],
|
||||
lookback: '30d',
|
||||
horizon: '7d',
|
||||
};
|
||||
|
||||
export const handlers = [
|
||||
// Query API (proxied at /api/)
|
||||
http.get('/api/companies', () => HttpResponse.json(mockCompanies)),
|
||||
@@ -242,4 +333,13 @@ export const handlers = [
|
||||
const body = await request.json() as Record<string, unknown>;
|
||||
return HttpResponse.json({ enabled: body.enabled, previous_enabled: true, toggled_by: 'operator' });
|
||||
}),
|
||||
|
||||
// Validation: Model Quality & Calibration endpoints
|
||||
http.get('/api/validation/summary', () => HttpResponse.json(mockValidationSummary)),
|
||||
http.get('/api/validation/calibration', () => HttpResponse.json(mockValidationCalibration)),
|
||||
http.get('/api/validation/gate-status', () => HttpResponse.json(mockValidationGateStatus)),
|
||||
http.get('/api/validation/ic-by-horizon', () => HttpResponse.json(mockValidationICByHorizon)),
|
||||
http.get('/api/validation/attribution/sources', () => HttpResponse.json(mockValidationAttributionSources)),
|
||||
http.get('/api/validation/attribution/catalysts', () => HttpResponse.json(mockValidationAttributionCatalysts)),
|
||||
http.get('/api/validation/attribution/layers', () => HttpResponse.json(mockValidationAttributionLayers)),
|
||||
];
|
||||
|
||||
Reference in New Issue
Block a user