phase 17: fix scheduler config parsing, worker entry points, and seed data for Polygon sources
This commit is contained in:
@@ -12,6 +12,21 @@ import logging
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
def _ensure_dict(val: Any) -> Optional[dict]:
|
||||
"""Coerce a JSONB value (dict or JSON string) to a Python dict."""
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, dict):
|
||||
return val
|
||||
if isinstance(val, str):
|
||||
try:
|
||||
parsed = json.loads(val)
|
||||
return parsed if isinstance(parsed, dict) else None
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return None
|
||||
return None
|
||||
|
||||
import asyncpg
|
||||
import redis.asyncio as aioredis
|
||||
|
||||
@@ -132,7 +147,7 @@ def build_job_payload(
|
||||
"aliases": aliases,
|
||||
"source_type": source["source_type"],
|
||||
"source_name": source["source_name"],
|
||||
"config": dict(source["config"]) if source["config"] else {},
|
||||
"config": _ensure_dict(source["config"]) or {},
|
||||
"credibility_score": float(source["credibility_score"]) if source["credibility_score"] else 0.5,
|
||||
"scheduled_at": now.isoformat(),
|
||||
}
|
||||
@@ -223,7 +238,7 @@ async def schedule_cycle(pool: asyncpg.Pool, rds: aioredis.Redis) -> int:
|
||||
for src in sources:
|
||||
source_id = src["source_id"]
|
||||
source_type = src["source_type"]
|
||||
source_config = dict(src["config"]) if src["config"] else None
|
||||
source_config = _ensure_dict(src["config"])
|
||||
|
||||
# Check last run status and timing
|
||||
last_run = await fetch_last_run(pool, source_id)
|
||||
|
||||
Reference in New Issue
Block a user