fix: track last_published_at per source to avoid re-fetching same articles — applies to both news_api and macro_news
This commit is contained in:
@@ -46,6 +46,9 @@ class MacroNewsAdapter(BaseAdapter):
|
||||
The ticker parameter is ignored for macro sources — these are
|
||||
global/geopolitical news, not company-specific.
|
||||
|
||||
Uses published_utc.gt to only fetch articles newer than the last
|
||||
successful fetch, avoiding re-fetching the same articles.
|
||||
|
||||
Args:
|
||||
ticker: Ignored for macro sources (may be empty string).
|
||||
config: Source-specific configuration with url, params, etc.
|
||||
@@ -67,6 +70,11 @@ class MacroNewsAdapter(BaseAdapter):
|
||||
limit = config.get("limit", 20)
|
||||
params["limit"] = str(min(int(limit), 1000))
|
||||
|
||||
# Use last_published_at from config to only fetch newer articles
|
||||
last_published = config.get("last_published_at")
|
||||
if last_published:
|
||||
params["published_utc.gt"] = last_published
|
||||
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user