feat: pipeline on/off toggle with per-stage Helm control
- Added pipelineEnabled flag to Helm values (default: true) - Worker services (scheduler, ingestion, parser, extractor, aggregation, recommendation, broker-adapter, lake-publisher) scale to 0 when disabled - API services always run regardless of toggle - Redis-based runtime toggle: POST /api/ops/pipeline/toggle - Scheduler checks the flag before each cycle - Frontend: green/red Pipeline ON/OFF button on the pipeline page - Beta defaults to pipelineEnabled: false - Base values.yaml: blanked external URLs (Ollama, Polygon, Alpaca) so stages only connect to what they explicitly configure
This commit is contained in:
@@ -19,6 +19,7 @@ from services.shared.config import load_config
|
||||
from services.shared.db import get_pg_pool, get_redis
|
||||
from services.shared.logging import setup_logging
|
||||
from services.shared.redis_keys import (
|
||||
PREFIX,
|
||||
QUEUE_EXTRACTION,
|
||||
QUEUE_INGESTION,
|
||||
QUEUE_MACRO_CLASSIFICATION,
|
||||
@@ -499,12 +500,19 @@ async def main() -> None:
|
||||
rds = get_redis(config)
|
||||
|
||||
logger.info("Scheduler started (tick=%ds)", SCHEDULER_TICK)
|
||||
pipeline_key = f"{PREFIX}:pipeline:enabled"
|
||||
recovery_counter = 0
|
||||
retry_counter = 0
|
||||
cleanup_counter = 0
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
# Check pipeline toggle — skip cycle if disabled
|
||||
flag = await rds.get(pipeline_key)
|
||||
if flag == "0":
|
||||
await asyncio.sleep(SCHEDULER_TICK)
|
||||
continue
|
||||
|
||||
if await acquire_lock(rds, "scheduler_cycle", ttl=30):
|
||||
try:
|
||||
await schedule_cycle(pool, rds)
|
||||
|
||||
Reference in New Issue
Block a user