feat: model validation, calibration, and signal quality layer
ci/woodpecker/push/test Pipeline failed
ci/woodpecker/push/build-1 unknown status
ci/woodpecker/push/build-3 unknown status
ci/woodpecker/push/build-2 unknown status
ci/woodpecker/push/finalize unknown status
Build and Push / lint-and-test (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.adapters.broker_adapter name:broker-adapter]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.aggregation.worker name:aggregation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.extractor.worker name:extractor]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.ingestion.worker name:ingestion]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.lake_publisher.worker name:lake-publisher]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.parser.worker name:parser]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.recommendation.worker name:recommendation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.scheduler.app name:scheduler]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.api.app:app --host 0.0.0.0 --port 8000 name:query-api]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.risk.app:app --host 0.0.0.0 --port 8000 name:risk]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000 name:symbol-registry]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.trading.app:app --host 0.0.0.0 --port 8000 name:trading-engine]) (push) Has been cancelled
Build and Push / build-dashboard (push) Has been cancelled
Build and Push / build-superset (push) Has been cancelled
Build and Push / integration-test (push) Has been cancelled
Build and Push / beta-gate (push) Has been cancelled
ci/woodpecker/push/test Pipeline failed
ci/woodpecker/push/build-1 unknown status
ci/woodpecker/push/build-3 unknown status
ci/woodpecker/push/build-2 unknown status
ci/woodpecker/push/finalize unknown status
Build and Push / lint-and-test (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.adapters.broker_adapter name:broker-adapter]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.aggregation.worker name:aggregation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.extractor.worker name:extractor]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.ingestion.worker name:ingestion]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.lake_publisher.worker name:lake-publisher]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.parser.worker name:parser]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.recommendation.worker name:recommendation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.scheduler.app name:scheduler]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.api.app:app --host 0.0.0.0 --port 8000 name:query-api]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.risk.app:app --host 0.0.0.0 --port 8000 name:risk]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000 name:symbol-registry]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.trading.app:app --host 0.0.0.0 --port 8000 name:trading-engine]) (push) Has been cancelled
Build and Push / build-dashboard (push) Has been cancelled
Build and Push / build-superset (push) Has been cancelled
Build and Push / integration-test (push) Has been cancelled
Build and Push / beta-gate (push) Has been cancelled
- Migration 035: prediction_snapshots, prediction_outcomes, signal_evidence_links, model_metric_snapshots tables + SQL views - Prediction snapshot writer with canonical evidence keys, duplicate detection, contribution scores - Outcome evaluator across 5 horizons (1h, 6h, 1d, 7d, 30d) - Metrics engine: ECE, Brier score, IC, Rank IC, benchmark comparison - Attribution engine: per-source, per-catalyst, per-layer performance - Calibration engine: Bayesian shrinkage source reliability - Quality gate for live trading eligibility with configurable thresholds - 7 new /api/validation/* endpoints - Upgraded OpsModel dashboard with validation tab - Enhanced recommendation display with calibration context - Backtest replay validation mode - 86 Python tests (unit + property-based), 179 frontend tests passing
This commit is contained in:
@@ -885,3 +885,169 @@ export function useToggleMacro() {
|
||||
onSuccess: () => qc.invalidateQueries({ queryKey: ['macro-status'] }),
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Validation: Model Quality & Calibration (Requirements 12.1, 12.2, 12.3, 12.7)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface ModelMetricSnapshot {
|
||||
id: string;
|
||||
generated_at: string;
|
||||
lookback_window: string;
|
||||
horizon: string;
|
||||
prediction_count: number;
|
||||
win_rate: number | null;
|
||||
directional_accuracy: number | null;
|
||||
information_coefficient: number | null;
|
||||
rank_information_coefficient: number | null;
|
||||
avg_return: number | null;
|
||||
avg_excess_return_vs_spy: number | null;
|
||||
avg_excess_return_vs_sector: number | null;
|
||||
calibration_error: number | null;
|
||||
brier_score: number | null;
|
||||
buy_win_rate: number | null;
|
||||
sell_win_rate: number | null;
|
||||
hold_win_rate: number | null;
|
||||
metadata: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
export interface ValidationSummary {
|
||||
snapshot: ModelMetricSnapshot | null;
|
||||
gate_status: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
export interface CalibrationBucket {
|
||||
bucket_low: number;
|
||||
bucket_high: number;
|
||||
avg_confidence: number;
|
||||
observed_win_rate: number;
|
||||
prediction_count: number;
|
||||
miscalibrated: boolean;
|
||||
}
|
||||
|
||||
export interface ValidationCalibration {
|
||||
buckets: CalibrationBucket[];
|
||||
lookback: string;
|
||||
horizon: string;
|
||||
}
|
||||
|
||||
export interface ICByHorizonEntry {
|
||||
horizon: string;
|
||||
information_coefficient: number | null;
|
||||
rank_information_coefficient: number | null;
|
||||
prediction_count: number;
|
||||
generated_at: string | null;
|
||||
}
|
||||
|
||||
export interface ValidationICByHorizon {
|
||||
horizons: ICByHorizonEntry[];
|
||||
lookback: string;
|
||||
}
|
||||
|
||||
export interface ValidationGateStatus {
|
||||
gate_status: Record<string, unknown> | null;
|
||||
updated_at?: string | null;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export function useValidationSummary(lookback = '30d', horizon = '7d') {
|
||||
const qs = new URLSearchParams();
|
||||
if (lookback) qs.set('lookback', lookback);
|
||||
if (horizon) qs.set('horizon', horizon);
|
||||
const path = `/api/validation/summary${qs.toString() ? '?' + qs : ''}`;
|
||||
return useGet<ValidationSummary>(['validation-summary', lookback, horizon], 'query', path);
|
||||
}
|
||||
|
||||
export function useValidationCalibration(lookback = '30d', horizon = '7d') {
|
||||
const qs = new URLSearchParams();
|
||||
if (lookback) qs.set('lookback', lookback);
|
||||
if (horizon) qs.set('horizon', horizon);
|
||||
const path = `/api/validation/calibration${qs.toString() ? '?' + qs : ''}`;
|
||||
return useGet<ValidationCalibration>(['validation-calibration', lookback, horizon], 'query', path);
|
||||
}
|
||||
|
||||
export function useValidationICByHorizon(lookback = '30d') {
|
||||
const qs = new URLSearchParams();
|
||||
if (lookback) qs.set('lookback', lookback);
|
||||
const path = `/api/validation/ic-by-horizon${qs.toString() ? '?' + qs : ''}`;
|
||||
return useGet<ValidationICByHorizon>(['validation-ic-by-horizon', lookback], 'query', path);
|
||||
}
|
||||
|
||||
export function useValidationGateStatus() {
|
||||
return useGet<ValidationGateStatus>(['validation-gate-status'], 'query', '/api/validation/gate-status');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Validation: Attribution — Sources, Catalysts, Layers (Requirements 12.4, 12.5, 12.6)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface SourceAttribution {
|
||||
source: string;
|
||||
source_type: string;
|
||||
prediction_count: number;
|
||||
avg_weight: number;
|
||||
avg_contribution_score: number;
|
||||
win_rate: number;
|
||||
avg_future_return: number;
|
||||
avg_excess_return_vs_spy: number;
|
||||
information_coefficient: number | null;
|
||||
duplicate_rate: number;
|
||||
}
|
||||
|
||||
export interface SourceAttributionResponse {
|
||||
sources: SourceAttribution[];
|
||||
lookback: string;
|
||||
horizon: string;
|
||||
}
|
||||
|
||||
export interface CatalystAttribution {
|
||||
catalyst_type: string;
|
||||
prediction_count: number;
|
||||
win_rate: number;
|
||||
avg_future_return: number;
|
||||
avg_excess_return_vs_spy: number;
|
||||
information_coefficient: number | null;
|
||||
}
|
||||
|
||||
export interface CatalystAttributionResponse {
|
||||
catalysts: CatalystAttribution[];
|
||||
lookback: string;
|
||||
horizon: string;
|
||||
}
|
||||
|
||||
export interface LayerAttribution {
|
||||
layer: string;
|
||||
avg_contribution_pct: number;
|
||||
dominant_win_rate: number;
|
||||
dominant_ic: number | null;
|
||||
}
|
||||
|
||||
export interface LayerAttributionResponse {
|
||||
layers: LayerAttribution[];
|
||||
lookback: string;
|
||||
horizon: string;
|
||||
}
|
||||
|
||||
export function useValidationAttributionSources(lookback = '30d', horizon = '7d') {
|
||||
const qs = new URLSearchParams();
|
||||
if (lookback) qs.set('lookback', lookback);
|
||||
if (horizon) qs.set('horizon', horizon);
|
||||
const path = `/api/validation/attribution/sources${qs.toString() ? '?' + qs : ''}`;
|
||||
return useGet<SourceAttributionResponse>(['validation-attribution-sources', lookback, horizon], 'query', path);
|
||||
}
|
||||
|
||||
export function useValidationAttributionCatalysts(lookback = '30d', horizon = '7d') {
|
||||
const qs = new URLSearchParams();
|
||||
if (lookback) qs.set('lookback', lookback);
|
||||
if (horizon) qs.set('horizon', horizon);
|
||||
const path = `/api/validation/attribution/catalysts${qs.toString() ? '?' + qs : ''}`;
|
||||
return useGet<CatalystAttributionResponse>(['validation-attribution-catalysts', lookback, horizon], 'query', path);
|
||||
}
|
||||
|
||||
export function useValidationAttributionLayers(lookback = '30d', horizon = '7d') {
|
||||
const qs = new URLSearchParams();
|
||||
if (lookback) qs.set('lookback', lookback);
|
||||
if (horizon) qs.set('horizon', horizon);
|
||||
const path = `/api/validation/attribution/layers${qs.toString() ? '?' + qs : ''}`;
|
||||
return useGet<LayerAttributionResponse>(['validation-attribution-layers', lookback, horizon], 'query', path);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user