From c5b7bddadb3d40a3bca9580a586c68d8fe22dd85 Mon Sep 17 00:00:00 2001
From: Celes Renata <celes@frameshift.net>
Date: Fri, 17 Apr 2026 07:37:14 +0000
Subject: [PATCH] fix: backfill recommendation evidence for existing
 recommendations

Migration 028: For each recommendation with no evidence rows, finds
the closest matching trend_window (by ticker + time_horizon + timestamp)
and re-inserts evidence from top_supporting/opposing_evidence arrays.
Filters out non-UUID pattern IDs and verifies documents exist.

This fixes 'No evidence linked' on recommendations created before the
UUID filtering fix in persist_recommendation.
---
 .kiro/steering/project-context.md             |  2 +-
 .../028_backfill_recommendation_evidence.sql  | 73 +++++++++++++++++++
 2 files changed, 74 insertions(+), 1 deletion(-)
 create mode 100644 infra/migrations/028_backfill_recommendation_evidence.sql

diff --git a/.kiro/steering/project-context.md b/.kiro/steering/project-context.md
index ba4714f..e0a2635 100644
--- a/.kiro/steering/project-context.md
+++ b/.kiro/steering/project-context.md
@@ -76,7 +76,7 @@ When a full reset is needed:
 ## Database Migrations
 - Located in `infra/migrations/001_*.sql` through `027_*.sql`
 - Applied automatically by `runmefirst.sh` in sorted order
-- Next migration number: **028**
+- Next migration number: **029**
 - Key migrations:
   - 016: Global news interpolation (global_events, macro_impact_records, exposure_profiles, trend_projections)
   - 017: Competitive intelligence (competitor_relationships, competitive_signal_records)
diff --git a/infra/migrations/028_backfill_recommendation_evidence.sql b/infra/migrations/028_backfill_recommendation_evidence.sql
new file mode 100644
index 0000000..97ba342
--- /dev/null
+++ b/infra/migrations/028_backfill_recommendation_evidence.sql
@@ -0,0 +1,73 @@
+-- Migration 028: Backfill recommendation_evidence for existing recommendations
+--
+-- The recommendation worker had a bug where non-UUID document IDs (synthetic
+-- "pattern:..." IDs from competitive signal propagation) caused the entire
+-- executemany INSERT to fail, silently dropping ALL evidence rows.
+--
+-- This migration re-links recommendations to their evidence using the
+-- trend_windows data. For each recommendation with no evidence, we find
+-- the closest matching trend_window and insert its document references.
+
+-- Step 1: Create a temp table with the backfill data
+CREATE TEMP TABLE _backfill_evidence AS
+WITH recs_without_evidence AS (
+    SELECT r.id AS rec_id, r.ticker, r.time_horizon, r.generated_at
+    FROM recommendations r
+    WHERE NOT EXISTS (
+        SELECT 1 FROM recommendation_evidence re WHERE re.recommendation_id = r.id
+    )
+),
+matched_trends AS (
+    SELECT DISTINCT ON (rwe.rec_id)
+        rwe.rec_id,
+        tw.top_supporting_evidence,
+        tw.top_opposing_evidence
+    FROM recs_without_evidence rwe
+    JOIN trend_windows tw
+      ON tw.entity_id = rwe.ticker
+     AND tw.window = rwe.time_horizon
+    ORDER BY rwe.rec_id, ABS(EXTRACT(EPOCH FROM (tw.generated_at - rwe.generated_at)))
+),
+supporting AS (
+    SELECT
+        mt.rec_id,
+        elem.doc_id,
+        'supporting'::text AS evidence_type,
+        elem.idx
+    FROM matched_trends mt,
+    LATERAL jsonb_array_elements_text(COALESCE(mt.top_supporting_evidence, '[]'::jsonb))
+        WITH ORDINALITY AS elem(doc_id, idx)
+    WHERE elem.doc_id ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+),
+opposing AS (
+    SELECT
+        mt.rec_id,
+        elem.doc_id,
+        'opposing'::text AS evidence_type,
+        elem.idx
+    FROM matched_trends mt,
+    LATERAL jsonb_array_elements_text(COALESCE(mt.top_opposing_evidence, '[]'::jsonb))
+        WITH ORDINALITY AS elem(doc_id, idx)
+    WHERE elem.doc_id ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+),
+all_evidence AS (
+    SELECT * FROM supporting
+    UNION ALL
+    SELECT * FROM opposing
+)
+SELECT
+    ae.rec_id AS recommendation_id,
+    ae.doc_id::uuid AS document_id,
+    ae.evidence_type,
+    ROUND((1.0 / (1.0 + (ae.idx - 1) * 0.1))::numeric, 4) AS weight
+FROM all_evidence ae
+WHERE EXISTS (SELECT 1 FROM documents d WHERE d.id = ae.doc_id::uuid);
+
+-- Step 2: Insert the backfill data
+INSERT INTO recommendation_evidence (recommendation_id, document_id, evidence_type, weight)
+SELECT recommendation_id, document_id, evidence_type, weight
+FROM _backfill_evidence
+ON CONFLICT DO NOTHING;
+
+-- Step 3: Clean up
+DROP TABLE _backfill_evidence;