Celes Renata
d018c75182
Document Stages now shows time-filtered counts from /health endpoint (consistent with Parsing/Extraction/Trends sections) with all-time totals shown as a subtle subtitle from the SSE stream. Added "(last Xh)" labels to all sections so the time window context is always visible.
236 lines
9.6 KiB
TypeScript
236 lines
9.6 KiB
TypeScript
import { useState, useEffect } from 'react';
|
|
import { usePipelineHealth, useRetryFailedExtractions, usePipelineToggle } from '../api/hooks';
|
|
import { LoadingSpinner, DateRangeSelector, Card } from '../components/ui';
|
|
|
|
const QUEUE_LABELS: Record<string, string> = {
|
|
ingestion: 'Ingestion',
|
|
parsing: 'Parsing',
|
|
extraction: 'Extraction',
|
|
macro_classification: 'Macro Classify',
|
|
aggregation: 'Aggregation',
|
|
recommendation: 'Recommendation',
|
|
lake_publish: 'Lake Publish',
|
|
trade: 'Trade',
|
|
trading_decisions: 'Trading Decisions',
|
|
broker_orders: 'Broker Orders',
|
|
};
|
|
|
|
interface StreamData {
|
|
queue_depths: Record<string, number>;
|
|
document_stages: Record<string, number>;
|
|
}
|
|
|
|
function usePipelineStream() {
|
|
const [data, setData] = useState<StreamData | null>(null);
|
|
|
|
useEffect(() => {
|
|
// EventSource is not available in test environments (jsdom)
|
|
if (typeof EventSource === 'undefined') return;
|
|
|
|
const base = import.meta.env.VITE_QUERY_API_BASE || '';
|
|
const url = `${base}/api/ops/pipeline/stream`;
|
|
const es = new EventSource(url);
|
|
|
|
es.onmessage = (event) => {
|
|
try {
|
|
setData(JSON.parse(event.data));
|
|
} catch {
|
|
// ignore parse errors
|
|
}
|
|
};
|
|
|
|
es.onerror = () => {
|
|
// EventSource auto-reconnects
|
|
};
|
|
|
|
return () => es.close();
|
|
}, []);
|
|
|
|
return data;
|
|
}
|
|
|
|
export function OpsPipelinePage() {
|
|
const [hours, setHours] = useState(24);
|
|
const { data, isLoading } = usePipelineHealth(hours);
|
|
const stream = usePipelineStream();
|
|
const retryMutation = useRetryFailedExtractions();
|
|
const toggleMutation = usePipelineToggle();
|
|
|
|
if (isLoading) return <LoadingSpinner />;
|
|
|
|
const parsing = (data?.parsing ?? {}) as Record<string, unknown>;
|
|
const extraction = (data?.extraction ?? {}) as Record<string, unknown>;
|
|
const aggregation = (data?.aggregation ?? {}) as Record<string, unknown>;
|
|
const pipelineEnabled = (data?.pipeline_enabled ?? true) as boolean;
|
|
|
|
// Queue depths come from live stream (always current)
|
|
// Document stages: show time-filtered from /health endpoint so numbers are consistent
|
|
// with Parsing/Extraction/Trends sections. Stream provides live queue data only.
|
|
const queueDepths = stream?.queue_depths
|
|
?? (data?.queue_depths as Record<string, number> | undefined)
|
|
?? {};
|
|
const docStages = Object.fromEntries(
|
|
((data?.document_stages as Array<{ status: string; doc_count: number }>) ?? [])
|
|
.map((s) => [s.status, s.doc_count]),
|
|
);
|
|
// All-time totals from stream for reference
|
|
const allTimeStages = stream?.document_stages ?? {};
|
|
|
|
const failedCount = docStages['extraction_failed'] ?? 0;
|
|
|
|
// Separate DLQ entries from regular queues
|
|
const dlqEntries = Object.entries(queueDepths).filter(([k]) => k.startsWith('dlq:'));
|
|
const regularQueues = Object.entries(QUEUE_LABELS);
|
|
|
|
return (
|
|
<div className="space-y-6">
|
|
<div className="flex items-center justify-between">
|
|
<h1 className="text-xl font-semibold text-gray-100">Pipeline Health</h1>
|
|
<div className="flex items-center gap-3">
|
|
<button
|
|
type="button"
|
|
onClick={() => toggleMutation.mutate(!pipelineEnabled)}
|
|
disabled={toggleMutation.isPending}
|
|
className={`rounded-md px-3 py-1.5 text-xs font-medium text-white ${pipelineEnabled ? 'bg-green-600 hover:bg-green-500' : 'bg-red-600 hover:bg-red-500'} disabled:opacity-50`}
|
|
>
|
|
{toggleMutation.isPending ? '…' : pipelineEnabled ? 'Pipeline ON' : 'Pipeline OFF'}
|
|
</button>
|
|
{failedCount > 0 && (
|
|
<button
|
|
type="button"
|
|
onClick={() => retryMutation.mutate()}
|
|
disabled={retryMutation.isPending}
|
|
className="rounded-md bg-amber-600 px-3 py-1.5 text-xs font-medium text-white hover:bg-amber-500 disabled:opacity-50"
|
|
>
|
|
{retryMutation.isPending ? 'Retrying…' : `Retry Failed (${failedCount})`}
|
|
</button>
|
|
)}
|
|
{retryMutation.isSuccess && (
|
|
<span className="text-xs text-green-400">{retryMutation.data.message}</span>
|
|
)}
|
|
{retryMutation.isError && (
|
|
<span className="text-xs text-red-400">Retry failed</span>
|
|
)}
|
|
{stream && (
|
|
<span className="flex items-center gap-1.5 text-xs text-green-400">
|
|
<span className="inline-block h-2 w-2 rounded-full bg-green-400 animate-pulse" aria-hidden="true" />
|
|
Live
|
|
</span>
|
|
)}
|
|
<DateRangeSelector value={hours} onChange={setHours} />
|
|
</div>
|
|
</div>
|
|
|
|
{/* Queue Depths */}
|
|
<Card>
|
|
<h2 className="mb-3 text-sm font-medium text-gray-400">Queue Depths</h2>
|
|
<div className="grid grid-cols-2 gap-3 sm:grid-cols-5 lg:grid-cols-5">
|
|
{regularQueues.map(([key, label]) => {
|
|
const depth = queueDepths[key] ?? 0;
|
|
const color = depth > 100 ? 'text-amber-400' : depth > 0 ? 'text-blue-400' : 'text-gray-500';
|
|
return (
|
|
<div key={key} className="rounded-lg border border-surface-700 bg-surface-950 p-3 text-center">
|
|
<div className={`text-2xl font-bold tabular-nums ${color}`}>{depth}</div>
|
|
<div className="text-xs text-gray-500">{label}</div>
|
|
</div>
|
|
);
|
|
})}
|
|
</div>
|
|
{dlqEntries.length > 0 && (
|
|
<div className="mt-3">
|
|
<h3 className="mb-2 text-xs font-medium text-red-400">Dead Letter Queues</h3>
|
|
<div className="flex flex-wrap gap-3">
|
|
{dlqEntries.map(([key, depth]) => (
|
|
<div key={key} className="rounded-lg border border-red-700/50 bg-red-900/20 px-3 py-2 text-center">
|
|
<div className="text-lg font-bold tabular-nums text-red-400">{depth}</div>
|
|
<div className="text-xs text-red-400/70">{key.replace('dlq:', '')}</div>
|
|
</div>
|
|
))}
|
|
</div>
|
|
</div>
|
|
)}
|
|
</Card>
|
|
|
|
{/* Document Stage Counts */}
|
|
<Card>
|
|
<h2 className="mb-3 text-sm font-medium text-gray-400">
|
|
Document Stages
|
|
<span className="ml-2 text-xs text-gray-600">(last {hours}h)</span>
|
|
</h2>
|
|
<div className="grid grid-cols-2 gap-3 sm:grid-cols-4">
|
|
{Object.entries(docStages).map(([status, count]) => {
|
|
const allTime = allTimeStages[status];
|
|
const color = status === 'extracted' ? 'text-green-400'
|
|
: status === 'parsed' ? 'text-yellow-400'
|
|
: status === 'extraction_failed' ? 'text-red-400'
|
|
: status === 'low_quality' ? 'text-orange-400'
|
|
: 'text-gray-100';
|
|
return (
|
|
<div key={status} className="rounded-lg border border-surface-700 bg-surface-950 p-3 text-center">
|
|
<div className={`text-2xl font-bold tabular-nums ${color}`}>{count}</div>
|
|
<div className="text-xs capitalize text-gray-500">{status.replace('_', ' ')}</div>
|
|
{allTime != null && allTime !== count && (
|
|
<div className="mt-0.5 text-[10px] text-gray-600">{allTime} total</div>
|
|
)}
|
|
</div>
|
|
);
|
|
})}
|
|
</div>
|
|
</Card>
|
|
|
|
{/* Parsing Quality */}
|
|
<Card>
|
|
<h2 className="mb-3 text-sm font-medium text-gray-400">
|
|
Parsing Quality
|
|
<span className="ml-2 text-xs text-gray-600">(last {hours}h)</span>
|
|
</h2>
|
|
<dl className="grid grid-cols-2 gap-3 text-sm sm:grid-cols-5">
|
|
<Stat label="Total Parsed" value={parsing.total_parsed} />
|
|
<Stat label="High Confidence" value={parsing.high_confidence} color="text-green-400" />
|
|
<Stat label="Medium" value={parsing.medium_confidence} color="text-yellow-400" />
|
|
<Stat label="Low" value={parsing.low_confidence} color="text-red-400" />
|
|
<Stat label="Avg Quality" value={parsing.avg_quality_score} />
|
|
</dl>
|
|
</Card>
|
|
|
|
{/* Extraction Stats */}
|
|
<Card>
|
|
<h2 className="mb-3 text-sm font-medium text-gray-400">
|
|
Extraction Validation
|
|
<span className="ml-2 text-xs text-gray-600">(last {hours}h)</span>
|
|
</h2>
|
|
<dl className="grid grid-cols-2 gap-3 text-sm sm:grid-cols-5">
|
|
<Stat label="Total" value={extraction.total_extractions} />
|
|
<Stat label="Valid" value={extraction.valid} color="text-green-400" />
|
|
<Stat label="Failed" value={extraction.failed} color="text-red-400" />
|
|
<Stat label="Avg Confidence" value={extraction.avg_confidence} />
|
|
<Stat label="Avg Retries" value={extraction.avg_retries} />
|
|
</dl>
|
|
</Card>
|
|
|
|
{/* Aggregation */}
|
|
<Card>
|
|
<h2 className="mb-3 text-sm font-medium text-gray-400">
|
|
Trend Generation
|
|
<span className="ml-2 text-xs text-gray-600">(last {hours}h)</span>
|
|
</h2>
|
|
<dl className="grid grid-cols-2 gap-3 text-sm sm:grid-cols-4">
|
|
<Stat label="Trends Generated" value={aggregation.trends_generated} />
|
|
<Stat label="Symbols Covered" value={aggregation.symbols_covered} />
|
|
<Stat label="Avg Confidence" value={aggregation.avg_trend_confidence} />
|
|
<Stat label="Avg Contradiction" value={aggregation.avg_contradiction} />
|
|
</dl>
|
|
</Card>
|
|
</div>
|
|
);
|
|
}
|
|
|
|
function Stat({ label, value, color = 'text-gray-100' }: { label: string; value: unknown; color?: string }) {
|
|
return (
|
|
<div className="rounded-lg border border-surface-700 bg-surface-950 p-3 text-center">
|
|
<div className={`text-xl font-bold ${color}`}>{value != null ? String(value) : '—'}</div>
|
|
<div className="text-xs text-gray-500">{label}</div>
|
|
</div>
|
|
);
|
|
}
|