import { useState, useEffect } from 'react'; import { usePipelineHealth } from '../api/hooks'; import { LoadingSpinner, DateRangeSelector, Card } from '../components/ui'; const QUEUE_LABELS: Record = { ingestion: 'Ingestion', parsing: 'Parsing', extraction: 'Extraction', macro_classification: 'Macro Classify', aggregation: 'Aggregation', recommendation: 'Recommendation', lake_publish: 'Lake Publish', trade: 'Trade', trading_decisions: 'Trading Decisions', broker_orders: 'Broker Orders', }; interface StreamData { queue_depths: Record; document_stages: Record; } function usePipelineStream() { const [data, setData] = useState(null); useEffect(() => { // EventSource is not available in test environments (jsdom) if (typeof EventSource === 'undefined') return; const base = import.meta.env.VITE_QUERY_API_BASE || ''; const url = `${base}/api/ops/pipeline/stream`; const es = new EventSource(url); es.onmessage = (event) => { try { setData(JSON.parse(event.data)); } catch { // ignore parse errors } }; es.onerror = () => { // EventSource auto-reconnects }; return () => es.close(); }, []); return data; } export function OpsPipelinePage() { const [hours, setHours] = useState(24); const { data, isLoading } = usePipelineHealth(hours); const stream = usePipelineStream(); if (isLoading) return ; const parsing = (data?.parsing ?? {}) as Record; const extraction = (data?.extraction ?? {}) as Record; const aggregation = (data?.aggregation ?? {}) as Record; // Prefer live stream data for queue depths and doc stages, fall back to initial fetch const queueDepths = stream?.queue_depths ?? (data?.queue_depths as Record | undefined) ?? {}; const docStages = stream?.document_stages ?? Object.fromEntries( ((data?.document_stages as Array<{ status: string; doc_count: number }>) ?? []) .map((s) => [s.status, s.doc_count]), ); // Separate DLQ entries from regular queues const dlqEntries = Object.entries(queueDepths).filter(([k]) => k.startsWith('dlq:')); const regularQueues = Object.entries(QUEUE_LABELS); return (

Pipeline Health

{stream && ( )}
{/* Queue Depths */}

Queue Depths

{regularQueues.map(([key, label]) => { const depth = queueDepths[key] ?? 0; const color = depth > 100 ? 'text-amber-400' : depth > 0 ? 'text-blue-400' : 'text-gray-500'; return (
{depth}
{label}
); })}
{dlqEntries.length > 0 && (

Dead Letter Queues

{dlqEntries.map(([key, depth]) => (
{depth}
{key.replace('dlq:', '')}
))}
)}
{/* Document Stage Counts */}

Document Stages

{Object.entries(docStages).map(([status, count]) => { const color = status === 'extracted' ? 'text-green-400' : status === 'parsed' ? 'text-yellow-400' : status === 'extraction_failed' ? 'text-red-400' : status === 'low_quality' ? 'text-orange-400' : 'text-gray-100'; return (
{count}
{status.replace('_', ' ')}
); })}
{/* Parsing Quality */}

Parsing Quality

{/* Extraction Stats */}

Extraction Validation

{/* Aggregation */}

Trend Generation

); } function Stat({ label, value, color = 'text-gray-100' }: { label: string; value: unknown; color?: string }) { return (
{value != null ? String(value) : '—'}
{label}
); }