fix: ops pipeline — rate limit, validation cycle, snapshots, config query, rejection reason

- Polygon rate limit env-configurable, default 5 (free tier)
- v3_engine_enabled reads from JSONB config column correctly
- Validation cycle (outcome eval + metrics) wired into scheduler hourly
- Daily portfolio/risk snapshots after 16:30 ET with idempotency
- Prediction snapshot price fallback from positions table
- Order rejection_reason + rejected_at persisted on insert
- Lake-publisher scaled to 0 replicas (redundant)
- Test fix: isolate per-type rate limit tests from Polygon global

services/scheduler/app.py

@@ -75,7 +75,7 @@ DEFAULT_RATE_LIMITS: dict[str, int] = {
 # market_api + news_api share a single Polygon API key, so we cap the combined
 # throughput to stay safely under the plan limit.
 POLYGON_SOURCE_TYPES: set[str] = {"market_api", "news_api"}
-POLYGON_GLOBAL_RATE_LIMIT: int = 45
+POLYGON_GLOBAL_RATE_LIMIT: int = int(os.getenv("POLYGON_GLOBAL_RATE_LIMIT", "5"))
 
 # How long to wait before retrying a failed source (seconds)
 DEFAULT_BACKOFF_BASE: int = 60
@@ -89,6 +89,13 @@ SCHEDULER_TICK: int = 15
 # 15s tick × 60 cycles = 15 minutes
 AGGREGATION_CYCLE_INTERVAL: int = 60
 
+# Periodic snapshot capture: every N cycles (15s tick × 240 = ~60 minutes)
+SNAPSHOT_CYCLE_INTERVAL = int(os.getenv("SNAPSHOT_CYCLE_INTERVAL", "240"))
+
+# Periodic validation cycle: evaluate matured predictions + compute metric snapshots
+# 15s tick × 240 cycles = ~60 minutes
+VALIDATION_CYCLE_INTERVAL = int(os.getenv("VALIDATION_CYCLE_INTERVAL", "240"))
+
 
 def get_cadence_for_source(source_type: str, config: Optional[dict[str, Any]]) -> int:
     """Return the polling interval for a source.
@@ -682,6 +689,96 @@ async def enqueue_periodic_aggregation(pool: asyncpg.Pool, rds: aioredis.Redis)
     return count
 
 
+async def run_validation_cycle(pool: asyncpg.Pool) -> None:
+    """Run outcome evaluation and metric computation (hourly).
+
+    Requirements: 2.3, 2.4, 2.5
+    """
+    from services.validation.metrics import compute_and_store_metric_snapshots
+    from services.validation.outcome_evaluator import evaluate_matured_predictions
+
+    try:
+        outcomes = await evaluate_matured_predictions(pool)
+        logger.info("Validation: evaluated %d prediction outcomes", outcomes)
+    except Exception:
+        logger.exception("Validation: outcome evaluation failed")
+        return  # Skip metrics if outcomes failed
+
+    try:
+        snapshots = await compute_and_store_metric_snapshots(pool)
+        logger.info("Validation: computed %d metric snapshots", len(snapshots))
+    except Exception:
+        logger.exception("Validation: metric computation failed")
+
+
+async def maybe_capture_daily_snapshots(pool: asyncpg.Pool) -> None:
+    """Capture portfolio and risk snapshots once daily after market close.
+
+    Requirements: 2.6, 2.7
+    """
+    et_now = datetime.now(ZoneInfo("America/New_York"))
+
+    # Only after 4:30 PM ET
+    if et_now.hour < 16 or (et_now.hour == 16 and et_now.minute < 30):
+        return
+
+    today = et_now.date()
+
+    # Idempotency: already captured today?
+    existing = await pool.fetchval(
+        "SELECT 1 FROM portfolio_snapshots WHERE snapshot_date = $1 LIMIT 1",
+        today,
+    )
+    if existing:
+        return
+
+    # Portfolio snapshot from positions
+    positions: list = []
+    portfolio_value = 0.0
+    unrealized_pnl = 0.0
+    try:
+        positions = await pool.fetch("SELECT * FROM positions WHERE quantity > 0")
+        portfolio_value = sum(
+            float(r["current_price"] or 0) * float(r["quantity"])
+            for r in positions
+        )
+        unrealized_pnl = sum(float(r["unrealized_pnl"] or 0) for r in positions)
+
+        await pool.execute(
+            """INSERT INTO portfolio_snapshots
+               (snapshot_date, portfolio_value, unrealized_pnl, positions)
+               VALUES ($1, $2, $3, $4::jsonb)""",
+            today,
+            portfolio_value,
+            unrealized_pnl,
+            json.dumps([dict(r) for r in positions], default=str),
+        )
+        logger.info("Captured portfolio snapshot: value=%.2f", portfolio_value)
+    except Exception:
+        logger.exception("Failed to capture portfolio snapshot")
+
+    # Risk snapshot from daily activity
+    try:
+        daily_orders = await pool.fetchval(
+            "SELECT count(*) FROM orders WHERE created_at::date = $1", today
+        )
+        daily_pnl = unrealized_pnl
+
+        await pool.execute(
+            """INSERT INTO daily_risk_snapshots
+               (account_id, snapshot_date, portfolio_value, daily_pnl, daily_trade_count)
+               VALUES ((SELECT id FROM broker_accounts LIMIT 1), $1, $2, $3, $4)
+               ON CONFLICT DO NOTHING""",
+            today,
+            portfolio_value,
+            daily_pnl,
+            daily_orders or 0,
+        )
+        logger.info("Captured risk snapshot: pnl=%.2f trades=%d", daily_pnl, daily_orders or 0)
+    except Exception:
+        logger.exception("Failed to capture risk snapshot")
+
+
 async def main() -> None:
     config = load_config()
     setup_logging("scheduler", level=config.log_level, json_output=config.json_logs)
@@ -705,6 +802,8 @@ async def main() -> None:
     aggregation_counter = 0
     report_consumer_counter = 0
     report_schedule_counter = 0
+    snapshot_counter = 0
+    validation_counter = 0
     try:
         while True:
             try:
@@ -747,6 +846,16 @@ async def main() -> None:
                         if report_schedule_counter >= REPORT_SCHEDULE_CYCLE_INTERVAL:
                             report_schedule_counter = 0
                             await check_report_schedule(rds)
+                        # Validation cycle: outcome evaluation + metric snapshots (~60 minutes)
+                        validation_counter += 1
+                        if validation_counter >= VALIDATION_CYCLE_INTERVAL:
+                            validation_counter = 0
+                            await run_validation_cycle(pool)
+                        # Snapshot cycle: portfolio + risk snapshots daily after market close
+                        snapshot_counter += 1
+                        if snapshot_counter >= SNAPSHOT_CYCLE_INTERVAL:
+                            snapshot_counter = 0
+                            await maybe_capture_daily_snapshots(pool)
                     finally:
                         await release_lock(rds, "scheduler_cycle")
             except Exception: