From 0ee7f2663331135fefb4408ec825e9729792bbb5 Mon Sep 17 00:00:00 2001 From: Celes Renata Date: Thu, 16 Apr 2026 07:26:10 +0000 Subject: [PATCH] feat: raise market_api rate to 20/min, add global Polygon cap at 45/min, add rate-limit API + watchlist warning --- frontend/src/api/hooks.ts | 18 +++++++++++ frontend/src/pages/Watchlists.tsx | 49 +++++++++++++++++++++++++--- services/api/app.py | 54 +++++++++++++++++++++++++++++++ services/scheduler/app.py | 32 ++++++++++++++++-- tests/test_scheduler.py | 4 +-- 5 files changed, 148 insertions(+), 9 deletions(-) diff --git a/frontend/src/api/hooks.ts b/frontend/src/api/hooks.ts index bf0164d..7ff5a7a 100644 --- a/frontend/src/api/hooks.ts +++ b/frontend/src/api/hooks.ts @@ -717,6 +717,24 @@ export function useTrendProjection(trendId: string | undefined) { return useGet(['trend-projection', trendId], 'query', `/api/trends/${trendId}/projection`, !!trendId); } +// --------------------------------------------------------------------------- +// System: Rate Limits +// --------------------------------------------------------------------------- + +export interface RateLimitInfo { + polygon_global_limit: number; + market_api: { + rate_per_minute: number; + cadence_seconds: number; + max_tickers_per_cycle: number; + active_sources: number; + }; +} + +export function useRateLimits() { + return useGet(['rate-limits'], 'query', '/api/system/rate-limits'); +} + // --------------------------------------------------------------------------- // Macro: Admin Toggle (Task 17.6) // --------------------------------------------------------------------------- diff --git a/frontend/src/pages/Watchlists.tsx b/frontend/src/pages/Watchlists.tsx index 9f01e91..4995698 100644 --- a/frontend/src/pages/Watchlists.tsx +++ b/frontend/src/pages/Watchlists.tsx @@ -1,9 +1,12 @@ import { useState } from 'react'; -import { useWatchlists, useWatchlistMembers, useCreateWatchlist } from '../api/hooks'; +import { useWatchlists, useWatchlistMembers, useCreateWatchlist, useRateLimits } from '../api/hooks'; +import type { Company } from '../api/hooks'; import { LoadingSpinner, Card } from '../components/ui'; +import { AlertTriangle } from 'lucide-react'; export function WatchlistsPage() { const { data: watchlists, isLoading } = useWatchlists(); + const { data: rateLimits } = useRateLimits(); const [selected, setSelected] = useState(null); const [showCreate, setShowCreate] = useState(false); @@ -34,21 +37,44 @@ export function WatchlistsPage() { ))} - {selected && } + {selected && } ); } -function WatchlistMembers({ watchlistId }: { watchlistId: string }) { +interface RateLimitInfo { + polygon_global_limit: number; + market_api: { + rate_per_minute: number; + cadence_seconds: number; + max_tickers_per_cycle: number; + active_sources: number; + }; +} + +function WatchlistMembers({ watchlistId, rateLimits }: { watchlistId: string; rateLimits?: RateLimitInfo }) { const { data: members, isLoading } = useWatchlistMembers(watchlistId); if (isLoading) return ; if (!members?.length) return

No members in this watchlist

; + const maxTickers = rateLimits?.market_api.max_tickers_per_cycle; + const activeSources = rateLimits?.market_api.active_sources ?? 0; + const cadenceSec = rateLimits?.market_api.cadence_seconds ?? 300; + const cadenceMin = Math.round(cadenceSec / 60); + const overCapacity = maxTickers != null && activeSources > maxTickers; + return (

Members

+ {overCapacity && ( + + )}
- {members.map((m) => ( + {members.map((m: Company) => ( {m.ticker} — {m.legal_name} @@ -58,6 +84,21 @@ function WatchlistMembers({ watchlistId }: { watchlistId: string }) { ); } +function RateLimitWarning({ activeSources, maxTickers, cadenceMin }: { activeSources: number; maxTickers: number; cadenceMin: number }) { + const cycleMinutes = Math.ceil(activeSources / (maxTickers / cadenceMin)); + return ( +
+
+ ); +} + function CreateWatchlistForm({ onClose }: { onClose: () => void }) { const [name, setName] = useState(''); const [desc, setDesc] = useState(''); diff --git a/services/api/app.py b/services/api/app.py index c51cd1c..65ca1ad 100644 --- a/services/api/app.py +++ b/services/api/app.py @@ -1534,6 +1534,60 @@ async def get_source_coverage_gaps(): } +# --------------------------------------------------------------------------- +# System: Rate Limit Info +# --------------------------------------------------------------------------- + + +@app.get("/api/system/rate-limits") +async def get_rate_limits(): + """Return current rate limit configuration and usage. + + Exposes the scheduler's rate limits so the frontend can calculate + how many tickers can be refreshed within the configured cadence. + """ + from services.scheduler.app import ( + DEFAULT_CADENCES, + DEFAULT_RATE_LIMITS, + POLYGON_GLOBAL_RATE_LIMIT, + POLYGON_SOURCE_TYPES, + ) + + # Count active market_api sources to report current load + market_count = await pool.fetchval( + "SELECT count(*) FROM sources WHERE active = TRUE AND source_type = 'market_api'" + ) + news_count = await pool.fetchval( + "SELECT count(*) FROM sources WHERE active = TRUE AND source_type = 'news_api'" + ) + + market_cadence = DEFAULT_CADENCES.get("market_api", 300) + market_rate = DEFAULT_RATE_LIMITS.get("market_api", 20) + + # How many tickers can we refresh within one cadence window? + # cadence_minutes * rate_per_minute = max tickers per cycle + cadence_minutes = market_cadence / 60 + max_tickers_per_cycle = int(cadence_minutes * market_rate) + + return { + "polygon_global_limit": POLYGON_GLOBAL_RATE_LIMIT, + "polygon_source_types": sorted(POLYGON_SOURCE_TYPES), + "per_type_limits": DEFAULT_RATE_LIMITS, + "cadences_seconds": DEFAULT_CADENCES, + "market_api": { + "rate_per_minute": market_rate, + "cadence_seconds": market_cadence, + "max_tickers_per_cycle": max_tickers_per_cycle, + "active_sources": market_count, + }, + "news_api": { + "rate_per_minute": DEFAULT_RATE_LIMITS.get("news_api", 20), + "cadence_seconds": DEFAULT_CADENCES.get("news_api", 300), + "active_sources": news_count, + }, + } + + # --------------------------------------------------------------------------- # Analytics: Trino SQL Proxy (Requirement 10.1, 10.3, 13.7) # --------------------------------------------------------------------------- diff --git a/services/scheduler/app.py b/services/scheduler/app.py index 6c9cade..34dfbe7 100644 --- a/services/scheduler/app.py +++ b/services/scheduler/app.py @@ -45,7 +45,7 @@ def _ensure_dict(val: Any) -> Optional[dict]: # Default polling cadences by source class (seconds). # Individual sources can override via config.polling_interval_seconds. DEFAULT_CADENCES: dict[str, int] = { - "market_api": 900, + "market_api": 300, "news_api": 300, "filings_api": 3600, "web_scrape": 1800, @@ -55,7 +55,7 @@ DEFAULT_CADENCES: dict[str, int] = { # Default rate limits per source type (requests per minute) DEFAULT_RATE_LIMITS: dict[str, int] = { - "market_api": 5, + "market_api": 20, "news_api": 20, "filings_api": 10, "web_scrape": 10, @@ -63,6 +63,12 @@ DEFAULT_RATE_LIMITS: dict[str, int] = { "macro_news": 10, } +# Global rate limit across all Polygon-backed source types (requests per minute). +# market_api + news_api share a single Polygon API key, so we cap the combined +# throughput to stay safely under the plan limit. +POLYGON_SOURCE_TYPES: set[str] = {"market_api", "news_api"} +POLYGON_GLOBAL_RATE_LIMIT: int = 45 + # How long to wait before retrying a failed source (seconds) DEFAULT_BACKOFF_BASE: int = 60 MAX_BACKOFF: int = 3600 @@ -173,15 +179,35 @@ async def check_rate_limit( ) -> bool: """Check whether the source type is within its rate limit window. + Enforces two limits: + 1. Per-source-type limit (e.g. market_api: 20/min) + 2. Global Polygon limit across all Polygon-backed types (45/min combined) + Returns True if the request is allowed, False if rate-limited. """ limit = max_per_minute or DEFAULT_RATE_LIMITS.get(source_type, 30) window = now.strftime("%Y%m%d%H%M") + + # Per-source-type check key = rate_limit_key(source_type, window) count = await rds.incr(key) if count == 1: await rds.expire(key, 120) - return count <= limit + if count > limit: + return False + + # Global Polygon check for source types that share the Polygon API key + if source_type in POLYGON_SOURCE_TYPES: + global_key = rate_limit_key("_polygon_global", window) + global_count = await rds.incr(global_key) + if global_count == 1: + await rds.expire(global_key, 120) + if global_count > POLYGON_GLOBAL_RATE_LIMIT: + # Roll back the per-type counter since we won't actually make the call + await rds.decr(key) + return False + + return True async def fetch_active_sources(pool: asyncpg.Pool) -> list[asyncpg.Record]: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 105a578..7e2133b 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -13,7 +13,7 @@ from services.scheduler.app import ( class TestGetCadenceForSource: def test_default_cadence_market_api(self): - assert get_cadence_for_source("market_api", None) == 60 + assert get_cadence_for_source("market_api", None) == 300 def test_default_cadence_news_api(self): assert get_cadence_for_source("news_api", {}) == 300 @@ -57,7 +57,7 @@ class TestIsSourceDue: assert not is_source_due("market_api", None, last, "completed", 0, None, self._now()) def test_completed_past_cadence_is_due(self): - last = self._now() - timedelta(seconds=120) + last = self._now() - timedelta(seconds=400) assert is_source_due("market_api", None, last, "completed", 0, None, self._now()) def test_running_not_due(self):