feat: model validation, calibration, and signal quality layer
- Migration 035: prediction_snapshots, prediction_outcomes, signal_evidence_links, model_metric_snapshots tables + SQL views - Prediction snapshot writer with canonical evidence keys, duplicate detection, contribution scores - Outcome evaluator across 5 horizons (1h, 6h, 1d, 7d, 30d) - Metrics engine: ECE, Brier score, IC, Rank IC, benchmark comparison - Attribution engine: per-source, per-catalyst, per-layer performance - Calibration engine: Bayesian shrinkage source reliability - Quality gate for live trading eligibility with configurable thresholds - 7 new /api/validation/* endpoints - Upgraded OpsModel dashboard with validation tab - Enhanced recommendation display with calibration context - Backtest replay validation mode - 86 Python tests (unit + property-based), 179 frontend tests passing
This commit is contained in:
Celes Renata
@@ -1,13 +1,92 @@
|
||||
/**
|
||||
* Recommendation detail page with validation context.
|
||||
*
|
||||
* Shows original confidence alongside calibrated confidence (historical win rate),
|
||||
* evidence quality indicators, source reliability, and live eligibility status.
|
||||
*
|
||||
* Requirements: 13.1, 13.2, 13.3, 13.4, 13.5, 13.6, 13.7
|
||||
*/
|
||||
import { useParams, Link } from '@tanstack/react-router';
|
||||
import { useRecommendation } from '../api/hooks';
|
||||
import { AlertTriangle, ShieldCheck, ShieldX, Info } from 'lucide-react';
|
||||
import {
|
||||
useRecommendation,
|
||||
useValidationCalibration,
|
||||
useValidationGateStatus,
|
||||
useValidationAttributionSources,
|
||||
} from '../api/hooks';
|
||||
import { StatusBadge, ConfidenceBar, LoadingSpinner, Card } from '../components/ui';
|
||||
|
||||
export function RecommendationDetailPage() {
|
||||
const { id } = useParams({ from: '/recommendations/$id' });
|
||||
const { data: rec, isLoading } = useRecommendation(id);
|
||||
const { data: calibration } = useValidationCalibration();
|
||||
const { data: gateData } = useValidationGateStatus();
|
||||
const { data: sourcesData } = useValidationAttributionSources();
|
||||
|
||||
if (isLoading || !rec) return <LoadingSpinner />;
|
||||
|
||||
// --- Calibration: find the bucket matching this recommendation's confidence ---
|
||||
const matchingBucket = calibration?.buckets?.find(
|
||||
(b) => rec.confidence >= b.bucket_low && rec.confidence < b.bucket_high,
|
||||
);
|
||||
// Handle edge case: confidence of exactly 1.0 falls in the last bucket [0.90, 1.00]
|
||||
const calibratedBucket =
|
||||
matchingBucket ??
|
||||
(rec.confidence >= 1.0
|
||||
? calibration?.buckets?.find((b) => b.bucket_high >= 1.0)
|
||||
: undefined);
|
||||
|
||||
const historicalWinRate = calibratedBucket?.observed_win_rate;
|
||||
|
||||
// --- Evidence counts ---
|
||||
const totalEvidenceCount = rec.evidence.length;
|
||||
// Compute duplicate evidence: group by normalized title, count extras
|
||||
const titleCounts = new Map<string, number>();
|
||||
for (const ev of rec.evidence) {
|
||||
const key = (ev.title ?? '').toLowerCase().trim();
|
||||
titleCounts.set(key, (titleCounts.get(key) ?? 0) + 1);
|
||||
}
|
||||
let duplicateEvidenceCount = 0;
|
||||
for (const count of titleCounts.values()) {
|
||||
if (count > 1) duplicateEvidenceCount += count - 1;
|
||||
}
|
||||
const uniqueEvidenceCount = totalEvidenceCount - duplicateEvidenceCount;
|
||||
const duplicateRatio = totalEvidenceCount > 0 ? duplicateEvidenceCount / totalEvidenceCount : 0;
|
||||
const hasDuplicateWarning = duplicateRatio > 0.2;
|
||||
|
||||
// --- Source reliability: find primary contributing sources ---
|
||||
const evidenceSources = new Map<string, number>();
|
||||
for (const ev of rec.evidence) {
|
||||
const src = ev.source_type ?? ev.publisher ?? 'unknown';
|
||||
evidenceSources.set(src, (evidenceSources.get(src) ?? 0) + ev.weight);
|
||||
}
|
||||
// Sort by total weight descending to find primary source
|
||||
const sortedSources = [...evidenceSources.entries()].sort((a, b) => b[1] - a[1]);
|
||||
const primarySourceType = sortedSources[0]?.[0];
|
||||
|
||||
// Look up source reliability from attribution data
|
||||
const primarySourceAttribution = sourcesData?.sources?.find(
|
||||
(s) => s.source_type === primarySourceType || s.source === primarySourceType,
|
||||
);
|
||||
// Source reliability is approximated from win_rate via Bayesian shrinkage
|
||||
// The attribution data has win_rate which is the observed metric
|
||||
const primarySourceWinRate = primarySourceAttribution?.win_rate;
|
||||
// Bayesian shrinkage: reliability = 0.5 + (n/(n+30)) * (win_rate - 0.5)
|
||||
const primarySourceCount = primarySourceAttribution?.prediction_count ?? 0;
|
||||
const primarySourceReliability =
|
||||
primarySourceWinRate != null
|
||||
? 0.5 + (primarySourceCount / (primarySourceCount + 30)) * (primarySourceWinRate - 0.5)
|
||||
: undefined;
|
||||
const hasLowReliabilityWarning =
|
||||
primarySourceReliability != null && primarySourceReliability < 0.4;
|
||||
|
||||
// --- Gate status ---
|
||||
const gateStatus = gateData?.gate_status as {
|
||||
passed?: boolean;
|
||||
reason?: string;
|
||||
threshold_results?: Array<{ name: string; threshold: number; actual: number; passed: boolean }>;
|
||||
} | null;
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div className="flex items-center gap-3">
|
||||
@@ -28,6 +107,137 @@ export function RecommendationDetailPage() {
|
||||
</dl>
|
||||
</Card>
|
||||
|
||||
{/* Validation Context Card — Requirements 13.1–13.7 */}
|
||||
<Card>
|
||||
<h2 className="mb-3 text-sm font-medium text-gray-400">Validation Context</h2>
|
||||
<dl className="grid grid-cols-2 gap-x-8 gap-y-3 text-sm sm:grid-cols-3">
|
||||
{/* 13.1: Original confidence alongside calibrated confidence */}
|
||||
<div>
|
||||
<dt className="text-gray-500">Original Confidence</dt>
|
||||
<dd className="text-gray-200">{(rec.confidence * 100).toFixed(1)}%</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt className="text-gray-500">Calibrated Confidence</dt>
|
||||
<dd className="text-gray-200">
|
||||
{historicalWinRate != null
|
||||
? `${(historicalWinRate * 100).toFixed(1)}%`
|
||||
: 'N/A'}
|
||||
</dd>
|
||||
</div>
|
||||
|
||||
{/* 13.2: Historical win rate for similar confidence levels */}
|
||||
<div>
|
||||
<dt className="text-gray-500">Historical Win Rate</dt>
|
||||
<dd className="text-gray-200">
|
||||
{historicalWinRate != null ? (
|
||||
<span>
|
||||
{(historicalWinRate * 100).toFixed(1)}%
|
||||
{calibratedBucket && (
|
||||
<span className="ml-1 text-xs text-gray-500">
|
||||
({calibratedBucket.prediction_count} predictions)
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
) : (
|
||||
'N/A'
|
||||
)}
|
||||
</dd>
|
||||
</div>
|
||||
|
||||
{/* 13.3: Evidence count, unique evidence count, duplicate evidence count */}
|
||||
<div>
|
||||
<dt className="text-gray-500">Evidence Count</dt>
|
||||
<dd className="text-gray-200">{totalEvidenceCount}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt className="text-gray-500">Unique Evidence</dt>
|
||||
<dd className="text-gray-200">{uniqueEvidenceCount}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt className="flex items-center gap-1 text-gray-500">
|
||||
Duplicate Evidence
|
||||
{/* 13.6: Warning badge when duplicate evidence count > 20% of total */}
|
||||
{hasDuplicateWarning && (
|
||||
<span
|
||||
className="inline-flex items-center gap-0.5 rounded-full border border-yellow-700/50 bg-yellow-900/40 px-1.5 py-0.5 text-[10px] font-medium text-yellow-400"
|
||||
title="Duplicate evidence exceeds 20% of total — potential evidence inflation"
|
||||
>
|
||||
<AlertTriangle size={10} />
|
||||
>20%
|
||||
</span>
|
||||
)}
|
||||
</dt>
|
||||
<dd className="text-gray-200">
|
||||
{duplicateEvidenceCount}
|
||||
{totalEvidenceCount > 0 && (
|
||||
<span className="ml-1 text-xs text-gray-500">
|
||||
({(duplicateRatio * 100).toFixed(0)}%)
|
||||
</span>
|
||||
)}
|
||||
</dd>
|
||||
</div>
|
||||
|
||||
{/* 13.4: Source reliability indicator */}
|
||||
<div>
|
||||
<dt className="flex items-center gap-1 text-gray-500">
|
||||
Primary Source Reliability
|
||||
{/* 13.7: Warning badge when primary source reliability < 0.4 */}
|
||||
{hasLowReliabilityWarning && (
|
||||
<span
|
||||
className="inline-flex items-center gap-0.5 rounded-full border border-red-700/50 bg-red-900/40 px-1.5 py-0.5 text-[10px] font-medium text-red-400"
|
||||
title="Primary source reliability is below 0.4 — low or unknown reliability"
|
||||
>
|
||||
<AlertTriangle size={10} />
|
||||
Low
|
||||
</span>
|
||||
)}
|
||||
</dt>
|
||||
<dd className="text-gray-200">
|
||||
{primarySourceReliability != null ? (
|
||||
<span>
|
||||
{primarySourceReliability.toFixed(3)}
|
||||
{primarySourceType && (
|
||||
<span className="ml-1 text-xs text-gray-500">({primarySourceType})</span>
|
||||
)}
|
||||
</span>
|
||||
) : (
|
||||
'N/A'
|
||||
)}
|
||||
</dd>
|
||||
</div>
|
||||
|
||||
{/* 13.5: Live eligibility status with reason */}
|
||||
<div className="col-span-2">
|
||||
<dt className="text-gray-500">Live Eligibility</dt>
|
||||
<dd>
|
||||
{gateStatus != null ? (
|
||||
<div className="flex items-center gap-2">
|
||||
{gateStatus.passed ? (
|
||||
<span className="inline-flex items-center gap-1 text-green-400">
|
||||
<ShieldCheck size={14} />
|
||||
Gate Passed
|
||||
</span>
|
||||
) : (
|
||||
<span className="inline-flex items-center gap-1 text-red-400">
|
||||
<ShieldX size={14} />
|
||||
Gate Failed
|
||||
</span>
|
||||
)}
|
||||
{gateStatus.reason && (
|
||||
<span className="text-xs text-gray-500">{gateStatus.reason}</span>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<span className="inline-flex items-center gap-1 text-gray-500">
|
||||
<Info size={14} />
|
||||
N/A — no gate evaluation available
|
||||
</span>
|
||||
)}
|
||||
</dd>
|
||||
</div>
|
||||
</dl>
|
||||
</Card>
|
||||
|
||||
{rec.thesis && (
|
||||
<Card>
|
||||
<h2 className="mb-2 text-sm font-medium text-gray-400">Thesis</h2>
|
||||
|
||||
Reference in New Issue
Block a user