feat: pipeline on/off toggle with per-stage Helm control
- Added pipelineEnabled flag to Helm values (default: true) - Worker services (scheduler, ingestion, parser, extractor, aggregation, recommendation, broker-adapter, lake-publisher) scale to 0 when disabled - API services always run regardless of toggle - Redis-based runtime toggle: POST /api/ops/pipeline/toggle - Scheduler checks the flag before each cycle - Frontend: green/red Pipeline ON/OFF button on the pipeline page - Beta defaults to pipelineEnabled: false - Base values.yaml: blanked external URLs (Ollama, Polygon, Alpaca) so stages only connect to what they explicitly configure
This commit is contained in:
@@ -533,6 +533,14 @@ export function useRetryFailedExtractions() {
|
||||
});
|
||||
}
|
||||
|
||||
export function usePipelineToggle() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: (enabled: boolean) => apiPost<{ pipeline_enabled: boolean }>('query', '/api/ops/pipeline/toggle', { enabled }),
|
||||
onSuccess: () => qc.invalidateQueries({ queryKey: ['pipeline-health'] }),
|
||||
});
|
||||
}
|
||||
|
||||
export function useIngestionSummary(hours = 24) {
|
||||
return useGet<Record<string, unknown>>(['ingestion-summary', hours], 'query', `/api/ops/ingestion/summary?hours=${hours}`);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { usePipelineHealth, useRetryFailedExtractions } from '../api/hooks';
|
||||
import { usePipelineHealth, useRetryFailedExtractions, usePipelineToggle } from '../api/hooks';
|
||||
import { LoadingSpinner, DateRangeSelector, Card } from '../components/ui';
|
||||
|
||||
const QUEUE_LABELS: Record<string, string> = {
|
||||
@@ -54,12 +54,14 @@ export function OpsPipelinePage() {
|
||||
const { data, isLoading } = usePipelineHealth(hours);
|
||||
const stream = usePipelineStream();
|
||||
const retryMutation = useRetryFailedExtractions();
|
||||
const toggleMutation = usePipelineToggle();
|
||||
|
||||
if (isLoading) return <LoadingSpinner />;
|
||||
|
||||
const parsing = (data?.parsing ?? {}) as Record<string, unknown>;
|
||||
const extraction = (data?.extraction ?? {}) as Record<string, unknown>;
|
||||
const aggregation = (data?.aggregation ?? {}) as Record<string, unknown>;
|
||||
const pipelineEnabled = (data?.pipeline_enabled ?? true) as boolean;
|
||||
|
||||
// Prefer live stream data for queue depths and doc stages, fall back to initial fetch
|
||||
const queueDepths = stream?.queue_depths
|
||||
@@ -82,6 +84,14 @@ export function OpsPipelinePage() {
|
||||
<div className="flex items-center justify-between">
|
||||
<h1 className="text-xl font-semibold text-gray-100">Pipeline Health</h1>
|
||||
<div className="flex items-center gap-3">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => toggleMutation.mutate(!pipelineEnabled)}
|
||||
disabled={toggleMutation.isPending}
|
||||
className={`rounded-md px-3 py-1.5 text-xs font-medium text-white ${pipelineEnabled ? 'bg-green-600 hover:bg-green-500' : 'bg-red-600 hover:bg-red-500'} disabled:opacity-50`}
|
||||
>
|
||||
{toggleMutation.isPending ? '…' : pipelineEnabled ? 'Pipeline ON' : 'Pipeline OFF'}
|
||||
</button>
|
||||
{failedCount > 0 && (
|
||||
<button
|
||||
type="button"
|
||||
|
||||
@@ -106,6 +106,8 @@ export const handlers = [
|
||||
http.delete('/api/admin/trading/lockouts/:id', () => HttpResponse.json({ status: 'deleted' })),
|
||||
http.get('/api/ops/pipeline/health', () => HttpResponse.json({ hours: 24, document_stages: [{ status: 'extracted', doc_count: 5 }], parsing: {}, extraction: {}, aggregation: {}, queue_depths: {} })),
|
||||
http.post('/api/ops/pipeline/retry-failed', () => HttpResponse.json({ retried: 10, message: 'Re-enqueued 10 documents for extraction' })),
|
||||
http.get('/api/ops/pipeline/toggle', () => HttpResponse.json({ pipeline_enabled: true })),
|
||||
http.post('/api/ops/pipeline/toggle', () => HttpResponse.json({ pipeline_enabled: true })),
|
||||
http.get('/api/ops/ingestion/summary', () => HttpResponse.json({ total_runs: 10, completed: 8, failed: 2, total_items_fetched: 50, total_items_new: 12, by_source_type: [] })),
|
||||
http.get('/api/ops/ingestion/throughput', () => HttpResponse.json([])),
|
||||
http.get('/api/ops/model/performance', () => HttpResponse.json({ total_extractions: 20, success_rate: 0.9, avg_duration_ms: 1500, retry_rate: 0.05, avg_confidence: 0.8 })),
|
||||
|
||||
Reference in New Issue
Block a user