diff --git a/services/extractor/schemas.py b/services/extractor/schemas.py index 64d3678..495bc07 100644 --- a/services/extractor/schemas.py +++ b/services/extractor/schemas.py @@ -362,15 +362,19 @@ _HORIZON_MAP: dict[str, str] = { def _normalize_extraction_data(data: dict[str, Any]) -> dict[str, Any]: """Fix common model output issues before Pydantic validation.""" - # Fill missing top-level required fields with defaults - data.setdefault("summary", "") - data.setdefault("companies", []) - data.setdefault("macro_themes", []) - if "novelty_score" not in data: + # Fill missing or null top-level required fields with defaults + if not data.get("summary"): + data["summary"] = "" + if not isinstance(data.get("companies"), list): + data["companies"] = [] + if not isinstance(data.get("macro_themes"), list): + data["macro_themes"] = [] + if data.get("novelty_score") is None: data["novelty_score"] = 0.5 - if "confidence" not in data: + if data.get("confidence") is None: data["confidence"] = 0.3 - data.setdefault("extraction_warnings", ["incomplete_model_output"]) + if not isinstance(data.get("extraction_warnings"), list): + data["extraction_warnings"] = ["incomplete_model_output"] # Clamp novelty_score and confidence to [0, 1] for field in ("novelty_score", "confidence"): @@ -384,17 +388,22 @@ def _normalize_extraction_data(data: dict[str, Any]) -> dict[str, Any]: for comp in companies: if not isinstance(comp, dict): continue - # Fill missing required company fields with defaults - comp.setdefault("ticker", "") - comp.setdefault("company_name", "") - comp.setdefault("relevance", 0.5) - comp.setdefault("sentiment", "neutral") - comp.setdefault("impact_score", 0.5) - comp.setdefault("impact_horizon", "1d_30d") - comp.setdefault("catalyst_type", "other") - comp.setdefault("key_facts", []) - comp.setdefault("risks", []) - comp.setdefault("evidence_spans", []) + # Replace None values with defaults (model returns null for fields) + _company_defaults = { + "ticker": "", + "company_name": "", + "relevance": 0.5, + "sentiment": "neutral", + "impact_score": 0.5, + "impact_horizon": "1d_30d", + "catalyst_type": "other", + "key_facts": [], + "risks": [], + "evidence_spans": [], + } + for key, default in _company_defaults.items(): + if comp.get(key) is None: + comp[key] = default # Clamp numeric fields for f in ("relevance", "impact_score"): v = comp.get(f) @@ -412,6 +421,13 @@ def _normalize_extraction_data(data: dict[str, Any]) -> dict[str, Any]: mapped_cat = _CATALYST_ALIASES.get(cat.lower().strip(), "other") comp["catalyst_type"] = mapped_cat + # Filter out company entries with no ticker (model returned null/empty) + # — keep the rest of the extraction (summary, macro_themes, etc.) + data["companies"] = [ + c for c in companies + if isinstance(c, dict) and c.get("ticker") + ] + return data