fix: pipeline stop now halts all workers and flushes queues

Workers (ingestion, parser, extractor, aggregation, recommendation,
broker, lake-publisher) now check the pipeline:enabled Redis flag on
each loop iteration and sleep when disabled.

The toggle endpoint flushes all pipeline queues on disable so queued
jobs don't resume when workers eventually check. Broker/trading queues
are excluded from flush to avoid dropping in-flight orders.

services/api/app.py

@@ -41,7 +41,7 @@ from services.shared.audit import get_entity_audit_trail, get_order_audit_trail,
 from services.shared.config import load_config
 from services.shared.db import get_pg_pool, get_redis
 from services.shared.logging import new_trace_id, set_trace_context, setup_logging
-from services.shared.redis_keys import PREFIX, QUEUE_BROKER, QUEUE_PREFIX, queue_key
+from services.shared.redis_keys import PIPELINE_ENABLED_KEY, QUEUE_BROKER, QUEUE_PREFIX, queue_key
 from services.shared.schemas import MAJOR_DECISION_CATALYSTS
 
 logger = logging.getLogger("query_api")
@@ -1948,7 +1948,7 @@ async def retry_failed_extractions_endpoint():
 # Pipeline On/Off Toggle
 # ---------------------------------------------------------------------------
 
-_PIPELINE_ENABLED_KEY = f"{PREFIX}:pipeline:enabled"
+_PIPELINE_ENABLED_KEY = PIPELINE_ENABLED_KEY
 
 
 @app.get("/api/ops/pipeline/toggle")
@@ -1966,10 +1966,33 @@ async def set_pipeline_toggle(body: dict[str, Any]):
 
     Accepts: { "enabled": true/false }
     Workers check this flag before processing jobs.
+    When disabling, optionally flush all pipeline queues so in-flight
+    work stops immediately.
     """
     enabled = body.get("enabled", True)
+    flush = body.get("flush", not enabled)  # default: flush when disabling
     await rds.set(_PIPELINE_ENABLED_KEY, "1" if enabled else "0")
-    return {"pipeline_enabled": enabled, "message": f"Pipeline {'enabled' if enabled else 'disabled'}"}
+
+    flushed_counts: dict[str, int] = {}
+    if flush and not enabled:
+        from services.shared.redis_keys import QUEUE_PREFIX
+        # Flush all pipeline queues
+        queue_names = [
+            "ingestion", "parsing", "extraction", "macro_classification",
+            "aggregation", "recommendation", "lake_publish",
+        ]
+        for qname in queue_names:
+            qkey = f"{QUEUE_PREFIX}:{qname}"
+            count = await rds.llen(qkey)
+            if count > 0:
+                await rds.delete(qkey)
+                flushed_counts[qname] = count
+
+    msg = f"Pipeline {'enabled' if enabled else 'disabled'}"
+    if flushed_counts:
+        total = sum(flushed_counts.values())
+        msg += f" — flushed {total} queued jobs"
+    return {"pipeline_enabled": enabled, "flushed": flushed_counts, "message": msg}
 
 
 @app.get("/api/ops/sources/coverage-gaps")