fix: handle null fields from NuExtract3 in extraction normalization

- Replace setdefault() with explicit None checks in _normalize_extraction_data()
- Coerce null top-level fields (summary, novelty_score, confidence) to defaults
- Coerce null company fields (ticker, impact_score, impact_horizon, etc.) to defaults
- Filter out company entries with empty ticker after normalization
- Prevents schema validation failures when model returns null for required fields

services/extractor/schemas.py

@@ -362,15 +362,19 @@ _HORIZON_MAP: dict[str, str] = {
 
 def _normalize_extraction_data(data: dict[str, Any]) -> dict[str, Any]:
     """Fix common model output issues before Pydantic validation."""
-    # Fill missing top-level required fields with defaults
-    data.setdefault("summary", "")
-    data.setdefault("companies", [])
-    data.setdefault("macro_themes", [])
-    if "novelty_score" not in data:
+    # Fill missing or null top-level required fields with defaults
+    if not data.get("summary"):
+        data["summary"] = ""
+    if not isinstance(data.get("companies"), list):
+        data["companies"] = []
+    if not isinstance(data.get("macro_themes"), list):
+        data["macro_themes"] = []
+    if data.get("novelty_score") is None:
         data["novelty_score"] = 0.5
-    if "confidence" not in data:
+    if data.get("confidence") is None:
         data["confidence"] = 0.3
-    data.setdefault("extraction_warnings", ["incomplete_model_output"])
+    if not isinstance(data.get("extraction_warnings"), list):
+        data["extraction_warnings"] = ["incomplete_model_output"]
 
     # Clamp novelty_score and confidence to [0, 1]
     for field in ("novelty_score", "confidence"):
@@ -384,17 +388,22 @@ def _normalize_extraction_data(data: dict[str, Any]) -> dict[str, Any]:
         for comp in companies:
             if not isinstance(comp, dict):
                 continue
-            # Fill missing required company fields with defaults
-            comp.setdefault("ticker", "")
-            comp.setdefault("company_name", "")
-            comp.setdefault("relevance", 0.5)
-            comp.setdefault("sentiment", "neutral")
-            comp.setdefault("impact_score", 0.5)
-            comp.setdefault("impact_horizon", "1d_30d")
-            comp.setdefault("catalyst_type", "other")
-            comp.setdefault("key_facts", [])
-            comp.setdefault("risks", [])
-            comp.setdefault("evidence_spans", [])
+            # Replace None values with defaults (model returns null for fields)
+            _company_defaults = {
+                "ticker": "",
+                "company_name": "",
+                "relevance": 0.5,
+                "sentiment": "neutral",
+                "impact_score": 0.5,
+                "impact_horizon": "1d_30d",
+                "catalyst_type": "other",
+                "key_facts": [],
+                "risks": [],
+                "evidence_spans": [],
+            }
+            for key, default in _company_defaults.items():
+                if comp.get(key) is None:
+                    comp[key] = default
             # Clamp numeric fields
             for f in ("relevance", "impact_score"):
                 v = comp.get(f)
@@ -412,6 +421,13 @@ def _normalize_extraction_data(data: dict[str, Any]) -> dict[str, Any]:
                 mapped_cat = _CATALYST_ALIASES.get(cat.lower().strip(), "other")
                 comp["catalyst_type"] = mapped_cat
 
+        # Filter out company entries with no ticker (model returned null/empty)
+        # — keep the rest of the extraction (summary, macro_themes, etc.)
+        data["companies"] = [
+            c for c in companies
+            if isinstance(c, dict) and c.get("ticker")
+        ]
+
     return data