feat: pipeline on/off toggle with per-stage Helm control

- Added pipelineEnabled flag to Helm values (default: true)
- Worker services (scheduler, ingestion, parser, extractor, aggregation,
  recommendation, broker-adapter, lake-publisher) scale to 0 when disabled
- API services always run regardless of toggle
- Redis-based runtime toggle: POST /api/ops/pipeline/toggle
- Scheduler checks the flag before each cycle
- Frontend: green/red Pipeline ON/OFF button on the pipeline page
- Beta defaults to pipelineEnabled: false
- Base values.yaml: blanked external URLs (Ollama, Polygon, Alpaca)
  so stages only connect to what they explicitly configure
This commit is contained in:
Celes Renata
2026-04-21 00:21:53 +00:00
parent a19ed086fe
commit be526ae614
14 changed files with 923 additions and 104 deletions
+11 -1
View File
@@ -1,5 +1,5 @@
import { useState, useEffect } from 'react';
import { usePipelineHealth, useRetryFailedExtractions } from '../api/hooks';
import { usePipelineHealth, useRetryFailedExtractions, usePipelineToggle } from '../api/hooks';
import { LoadingSpinner, DateRangeSelector, Card } from '../components/ui';
const QUEUE_LABELS: Record<string, string> = {
@@ -54,12 +54,14 @@ export function OpsPipelinePage() {
const { data, isLoading } = usePipelineHealth(hours);
const stream = usePipelineStream();
const retryMutation = useRetryFailedExtractions();
const toggleMutation = usePipelineToggle();
if (isLoading) return <LoadingSpinner />;
const parsing = (data?.parsing ?? {}) as Record<string, unknown>;
const extraction = (data?.extraction ?? {}) as Record<string, unknown>;
const aggregation = (data?.aggregation ?? {}) as Record<string, unknown>;
const pipelineEnabled = (data?.pipeline_enabled ?? true) as boolean;
// Prefer live stream data for queue depths and doc stages, fall back to initial fetch
const queueDepths = stream?.queue_depths
@@ -82,6 +84,14 @@ export function OpsPipelinePage() {
<div className="flex items-center justify-between">
<h1 className="text-xl font-semibold text-gray-100">Pipeline Health</h1>
<div className="flex items-center gap-3">
<button
type="button"
onClick={() => toggleMutation.mutate(!pipelineEnabled)}
disabled={toggleMutation.isPending}
className={`rounded-md px-3 py-1.5 text-xs font-medium text-white ${pipelineEnabled ? 'bg-green-600 hover:bg-green-500' : 'bg-red-600 hover:bg-red-500'} disabled:opacity-50`}
>
{toggleMutation.isPending ? '…' : pipelineEnabled ? 'Pipeline ON' : 'Pipeline OFF'}
</button>
{failedCount > 0 && (
<button
type="button"