From ea6c2b3f5417ba749110643f479ec25b6f1e7fe2 Mon Sep 17 00:00:00 2001
From: Celes Renata <celes@frameshift.net>
Date: Wed, 15 Apr 2026 22:19:44 +0000
Subject: [PATCH] fix: market data rate limiting and backtest price lookup

- Increase market_api polling cadence from 60s to 900s (15 min).
  The prev-day bar endpoint returns the same data all day, so polling
  every minute wastes API quota. 50 tickers at 15-min cadence = ~3.3
  req/min, well within the 5/min rate limit.
- Reduce market_api rate limit from 30/min to 5/min to match.
- Fix backtest replay to query market_snapshots with data->>'c' for
  close prices instead of nonexistent market_data.close_price column.
- Enrich backtest recommendations with prices from market_snapshots
  and sectors from companies table.
---
 services/scheduler/app.py           |  4 ++--
 services/trading/backtest_replay.py | 34 +++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/services/scheduler/app.py b/services/scheduler/app.py
index 6b1bcae..9451377 100644
--- a/services/scheduler/app.py
+++ b/services/scheduler/app.py
@@ -45,7 +45,7 @@ def _ensure_dict(val: Any) -> Optional[dict]:
 # Default polling cadences by source class (seconds).
 # Individual sources can override via config.polling_interval_seconds.
 DEFAULT_CADENCES: dict[str, int] = {
-    "market_api": 60,
+    "market_api": 900,
     "news_api": 300,
     "filings_api": 3600,
     "web_scrape": 1800,
@@ -55,7 +55,7 @@ DEFAULT_CADENCES: dict[str, int] = {
 
 # Default rate limits per source type (requests per minute)
 DEFAULT_RATE_LIMITS: dict[str, int] = {
-    "market_api": 30,
+    "market_api": 5,
     "news_api": 20,
     "filings_api": 10,
     "web_scrape": 10,
diff --git a/services/trading/backtest_replay.py b/services/trading/backtest_replay.py
index 4f85c7b..6e86839 100644
--- a/services/trading/backtest_replay.py
+++ b/services/trading/backtest_replay.py
@@ -86,6 +86,32 @@ class BacktestReplay:
             prev_value = config.initial_capital
             trade_log: list[dict] = []
 
+            # Pre-load company sectors and latest prices for enrichment
+            company_sectors: dict[str, str] = {}
+            company_prices: dict[str, float] = {}
+            if self.pool is not None:
+                try:
+                    sector_rows = await self.pool.fetch(
+                        "SELECT ticker, sector FROM companies WHERE active = TRUE"
+                    )
+                    for sr in sector_rows:
+                        company_sectors[sr["ticker"]] = sr["sector"] or "Unknown"
+                except Exception:
+                    logger.debug("Could not load company sectors")
+
+                # Load latest market prices (use most recent close from market_snapshots JSONB)
+                try:
+                    price_rows = await self.pool.fetch(
+                        "SELECT DISTINCT ON (ticker) ticker, (data->>'c')::float as close_price "
+                        "FROM market_snapshots WHERE snapshot_type = 'bar' "
+                        "ORDER BY ticker, captured_at DESC"
+                    )
+                    for pr in price_rows:
+                        if pr["close_price"]:
+                            company_prices[pr["ticker"]] = float(pr["close_price"])
+                except Exception:
+                    logger.debug("Could not load market prices — using portfolio_pct fallback")
+
             # Group recommendations by date
             recs_by_date: dict[date, list[dict]] = {}
             for rec in recs:
@@ -94,6 +120,14 @@ class BacktestReplay:
                     d = rec_date.date()
                 else:
                     d = rec_date
+
+                # Enrich rec with price and sector if missing
+                ticker = rec.get("ticker", "")
+                if "current_price" not in rec or not rec.get("current_price"):
+                    rec["current_price"] = company_prices.get(ticker, 50.0)
+                if "sector" not in rec or not rec.get("sector"):
+                    rec["sector"] = company_sectors.get(ticker, "Unknown")
+
                 recs_by_date.setdefault(d, []).append(rec)
 
             # Iterate through each trading day