feat: add live queue depths to pipeline health API and dashboard

services/api/app.py

@@ -22,6 +22,7 @@ from typing import Any, Optional
 
 import asyncpg
 import httpx
+import redis.asyncio as aioredis
 from fastapi import FastAPI, HTTPException, Query, Request
 from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
 from pydantic import BaseModel
@@ -35,7 +36,7 @@ from services.aggregation.pattern_matcher import (
 from services.extractor.metrics import get_model_performance_summary
 from services.shared.audit import get_entity_audit_trail, get_order_audit_trail, record_audit_event
 from services.shared.config import load_config
-from services.shared.db import get_pg_pool
+from services.shared.db import get_pg_pool, get_redis
 from services.shared.logging import new_trace_id, set_trace_context, setup_logging
 from services.shared.schemas import MAJOR_DECISION_CATALYSTS
 
@@ -43,15 +44,18 @@ logger = logging.getLogger("query_api")
 
 config = load_config()
 pool: Optional[asyncpg.Pool] = None
+rds: Optional[aioredis.Redis] = None
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    global pool
+    global pool, rds
     setup_logging("query_api", level=config.log_level, json_output=config.json_logs)
     pool = await get_pg_pool(config)
+    rds = get_redis(config)
     yield
     await pool.close()
+    await rds.close()
 
 
 app = FastAPI(title="Stonks Oracle - Query API", lifespan=lifespan)
@@ -1474,12 +1478,23 @@ async def get_pipeline_health(
         hours,
     )
 
+    # Queue depths from Redis
+    queue_depths: dict[str, int] = {}
+    if rds:
+        for qname in ("ingestion", "parsing", "extraction", "macro_classification", "aggregation", "recommendation", "broker_orders"):
+            try:
+                depth = await rds.llen(f"stonks:queue:{qname}")
+                queue_depths[qname] = depth
+            except Exception:
+                queue_depths[qname] = -1
+
     return {
         "hours": hours,
         "document_stages": [_row_to_dict(r) for r in doc_stages],
         "parsing": _row_to_dict(parse_quality) if parse_quality else {},
         "extraction": _row_to_dict(extraction_stats) if extraction_stats else {},
         "aggregation": _row_to_dict(trend_stats) if trend_stats else {},
+        "queue_depths": queue_depths,
     }