fix: pipeline stop now halts all workers and flushes queues

Workers (ingestion, parser, extractor, aggregation, recommendation,
broker, lake-publisher) now check the pipeline:enabled Redis flag on
each loop iteration and sleep when disabled.

The toggle endpoint flushes all pipeline queues on disable so queued
jobs don't resume when workers eventually check. Broker/trading queues
are excluded from flush to avoid dropping in-flight orders.

services/lake_publisher/jobs.py

@@ -54,7 +54,7 @@ from services.lake_publisher.worker import (
 from services.shared.config import load_config
 from services.shared.db import get_minio, get_pg_pool, get_redis
 from services.shared.logging import setup_logging
-from services.shared.redis_keys import QUEUE_LAKE_PUBLISH, queue_key
+from services.shared.redis_keys import QUEUE_LAKE_PUBLISH, is_pipeline_enabled, queue_key
 
 logger = logging.getLogger(__name__)
 
@@ -865,6 +865,9 @@ async def run_worker(
     logger.info("Lake publisher worker started, listening on %s", queue)
 
     while True:
+        if not await is_pipeline_enabled(rds):
+            await asyncio.sleep(poll_interval)
+            continue
         raw = await rds.lpop(queue)  # type: ignore[misc]
         if raw is None:
             await asyncio.sleep(poll_interval)