fix: track last_published_at per source to avoid re-fetching same articles — applies to both news_api and macro_news

2026-04-16 18:12:12 +00:00
parent 513310abba
commit 1043710b6d
3 changed files with 29 additions and 0 deletions
@@ -128,6 +128,10 @@ class PolygonNewsAdapter(NewsDataAdapter):
        if config.get("published_utc_lte"):
            params["published_utc.lte"] = config["published_utc_lte"]

+        # Auto-filter to only fetch articles newer than last successful fetch
+        if config.get("last_published_at") and "published_utc.gt" not in params:
+            params["published_utc.gt"] = config["last_published_at"]
+
        url = f"{self.base_url}{self.NEWS_ENDPOINT}"
        return url, params