From 949324dc89517a0bb0a8770e3c56994907a8b7ef Mon Sep 17 00:00:00 2001
From: Celes Renata <celes@frameshift.net>
Date: Thu, 16 Apr 2026 01:06:49 +0000
Subject: [PATCH] feat: SQL Explorer with PostgreSQL schema browser and
 pre-built queries
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The SQL Explorer was querying Trino which has zero tables. Rewrote to
use PostgreSQL directly:

Backend:
- GET /api/analytics/pg-schema: returns all public tables with column
  names, types, and nullability from information_schema
- POST /api/analytics/pg-query: read-only SQL execution against
  PostgreSQL with SELECT-only enforcement, auto LIMIT, and descriptive
  error messages for syntax/table/query errors

Frontend:
- Schema browser shows all PostgreSQL tables with columns and types
- Click a table name → generates SELECT * FROM table LIMIT 100
- Pre-built Queries section with 12 seeded queries covering companies,
  recommendations, trends, market prices, documents, global events,
  trading decisions, ingestion health, reserve pool, sector exposure
- User-saved queries shown separately with delete buttons
- Chart builder, Monaco editor, and save functionality preserved

Migration 021: seeds 12 pre-built saved queries
---
 frontend/src/pages/SqlExplorer.tsx          | 58 +++++++++++++++---
 frontend/src/test/mocks/handlers.ts         | 20 +++++++
 infra/migrations/021_seed_saved_queries.sql | 38 ++++++++++++
 services/api/app.py                         | 65 +++++++++++++++++++++
 4 files changed, 173 insertions(+), 8 deletions(-)
 create mode 100644 infra/migrations/021_seed_saved_queries.sql

diff --git a/frontend/src/pages/SqlExplorer.tsx b/frontend/src/pages/SqlExplorer.tsx
index 98bcd35..a9a6780 100644
--- a/frontend/src/pages/SqlExplorer.tsx
+++ b/frontend/src/pages/SqlExplorer.tsx
@@ -26,7 +26,7 @@ interface SavedQuery {
 interface SchemaInfo {
   catalog: string;
   schema: string;
-  tables: Array<{ name: string; columns: Array<{ name: string; type: string }> }>;
+  tables: Array<{ name: string; columns: Array<{ name: string; type: string; nullable?: boolean }> }>;
 }
 
 type ChartType = 'none' | 'bar' | 'line' | 'scatter';
@@ -41,8 +41,8 @@ export function SqlExplorerPage() {
   const [yCol, setYCol] = useState(1);
 
   const { data: schema } = useQuery<SchemaInfo>({
-    queryKey: ['trino-schema'],
-    queryFn: () => apiGet<SchemaInfo>('query', '/api/analytics/schema'),
+    queryKey: ['pg-schema'],
+    queryFn: () => apiGet<SchemaInfo>('query', '/api/analytics/pg-schema'),
   });
 
   const { data: savedQueries } = useQuery<SavedQuery[]>({
@@ -51,7 +51,7 @@ export function SqlExplorerPage() {
   });
 
   const executeMutation = useMutation({
-    mutationFn: (sqlText: string) => apiPost<QueryResult>('query', '/api/analytics/query', { sql: sqlText, limit: 1000 }),
+    mutationFn: (sqlText: string) => apiPost<QueryResult>('query', '/api/analytics/pg-query', { sql: sqlText, limit: 1000 }),
     onSuccess: (data) => { setResult(data); setError(null); },
     onError: (err: Error) => { setError(err.message); setResult(null); },
   });
@@ -75,6 +75,11 @@ export function SqlExplorerPage() {
     if (name) saveMutation.mutate({ name, sql_text: sql });
   }, [sql, saveMutation]);
 
+  const handleQuickInsert = useCallback((tableName: string) => {
+    const query = `SELECT * FROM ${tableName} LIMIT 100`;
+    setSql(query);
+  }, []);
+
   // Build chart data from result
   const chartData = result && result.columns.length >= 2
     ? result.rows.map((row) => ({
@@ -84,16 +89,27 @@ export function SqlExplorerPage() {
       }))
     : [];
 
+  // Split saved queries into pre-built (no id-based delete) and user-saved
+  const preBuiltNames = new Set([
+    'Company Overview', 'Recent Recommendations', 'High Confidence Buys',
+    'Trend Windows Summary', 'Market Prices', 'Document Counts by Type',
+    'Global Events', 'Trading Decisions', 'Ingestion Health',
+    'Reserve Pool Ledger', 'Sector Exposure', 'Source Coverage Matrix',
+  ]);
+  const preBuiltQueries = (savedQueries ?? []).filter((sq) => preBuiltNames.has(sq.name));
+  const userQueries = (savedQueries ?? []).filter((sq) => !preBuiltNames.has(sq.name));
+
   return (
     <div className="flex h-[calc(100vh-3rem)] gap-4">
       {/* Schema browser sidebar */}
-      <div className="w-56 shrink-0 overflow-y-auto rounded-lg border border-surface-700 bg-surface-900 p-3">
+      <div className="w-64 shrink-0 overflow-y-auto rounded-lg border border-surface-700 bg-surface-900 p-3">
         <h2 className="mb-2 text-xs font-semibold uppercase tracking-wider text-gray-500">Schema</h2>
         {schema?.tables.map((t) => (
           <div key={t.name} className="mb-2">
             <button
               className="text-xs font-medium text-brand-300 hover:underline"
-              onClick={() => setSql((prev) => prev + ` ${t.name}`)}
+              onClick={() => handleQuickInsert(t.name)}
+              title={`SELECT * FROM ${t.name} LIMIT 100`}
             >
               {t.name}
             </button>
@@ -108,9 +124,35 @@ export function SqlExplorerPage() {
           </div>
         ))}
 
-        {/* Saved queries */}
+        {/* Pre-built queries */}
+        {preBuiltQueries.length > 0 && (
+          <>
+            <h2 className="mb-2 mt-4 text-xs font-semibold uppercase tracking-wider text-gray-500">Pre-built Queries</h2>
+            {preBuiltQueries.map((sq) => (
+              <div key={sq.id} className="mb-1.5">
+                <button
+                  className="block w-full truncate text-left text-xs text-brand-300 hover:text-brand-200"
+                  onClick={() => setSql(sq.sql_text)}
+                  title={sq.sql_text}
+                >
+                  {sq.name}
+                </button>
+                {sq.description && (
+                  <p className="ml-0 truncate text-[10px] text-gray-600" title={sq.description}>
+                    {sq.description}
+                  </p>
+                )}
+              </div>
+            ))}
+          </>
+        )}
+
+        {/* User saved queries */}
         <h2 className="mb-2 mt-4 text-xs font-semibold uppercase tracking-wider text-gray-500">Saved Queries</h2>
-        {(savedQueries ?? []).map((sq) => (
+        {userQueries.length === 0 && (
+          <p className="text-[10px] text-gray-600">No saved queries yet</p>
+        )}
+        {userQueries.map((sq) => (
           <div key={sq.id} className="mb-1 flex items-center justify-between">
             <button
               className="truncate text-xs text-gray-300 hover:text-brand-300"
diff --git a/frontend/src/test/mocks/handlers.ts b/frontend/src/test/mocks/handlers.ts
index 4b3c80a..7668a52 100644
--- a/frontend/src/test/mocks/handlers.ts
+++ b/frontend/src/test/mocks/handlers.ts
@@ -95,6 +95,26 @@ export const handlers = [
     return HttpResponse.json({ id: 'w1', name: body.name, description: body.description ?? null, active: true }, { status: 201 });
   }),
 
+  // Analytics: PostgreSQL schema + query
+  http.get('/api/analytics/pg-schema', () => HttpResponse.json({
+    catalog: 'postgresql', schema: 'public',
+    tables: [
+      { name: 'companies', columns: [{ name: 'id', type: 'uuid', nullable: false }, { name: 'ticker', type: 'character varying', nullable: false }, { name: 'legal_name', type: 'text', nullable: false }] },
+      { name: 'recommendations', columns: [{ name: 'id', type: 'uuid', nullable: false }, { name: 'ticker', type: 'character varying', nullable: false }, { name: 'action', type: 'character varying', nullable: false }] },
+    ],
+  })),
+  http.post('/api/analytics/pg-query', () => HttpResponse.json({
+    columns: [{ name: 'test', type: 'text' }], rows: [['1']], row_count: 1, elapsed_ms: 5,
+  })),
+  http.get('/api/analytics/saved-queries', () => HttpResponse.json([
+    { id: 'sq1', name: 'Company Overview', description: 'All tracked companies with sector and status', sql_text: 'SELECT ticker, legal_name, sector, active, created_at FROM companies ORDER BY ticker', created_at: '2026-04-10T12:00:00Z' },
+  ])),
+  http.post('/api/analytics/saved-queries', async ({ request }) => {
+    const body = await request.json() as Record<string, string>;
+    return HttpResponse.json({ id: 'sq-new', name: body.name, description: '', sql_text: body.sql_text, created_at: '2026-04-10T12:00:00Z' }, { status: 201 });
+  }),
+  http.delete('/api/analytics/saved-queries/:id', () => HttpResponse.json({ status: 'deleted' })),
+
   // Health
   http.get('/api/health', () => HttpResponse.json({ status: 'ok' })),
 
diff --git a/infra/migrations/021_seed_saved_queries.sql b/infra/migrations/021_seed_saved_queries.sql
new file mode 100644
index 0000000..c06bc63
--- /dev/null
+++ b/infra/migrations/021_seed_saved_queries.sql
@@ -0,0 +1,38 @@
+-- Seed pre-built saved queries for the SQL Explorer
+INSERT INTO saved_queries (name, description, sql_text) VALUES
+('Company Overview', 'All tracked companies with sector and status',
+ 'SELECT ticker, legal_name, sector, active, created_at FROM companies ORDER BY ticker'),
+
+('Recent Recommendations', 'Latest buy/sell recommendations with confidence',
+ 'SELECT ticker, action, confidence, mode, risk_classification, generated_at FROM recommendations WHERE action IN (''buy'', ''sell'') ORDER BY generated_at DESC LIMIT 50'),
+
+('High Confidence Buys', 'Buy recommendations with confidence >= 0.75',
+ 'SELECT ticker, confidence, mode, thesis, generated_at FROM recommendations WHERE action = ''buy'' AND confidence >= 0.75 ORDER BY confidence DESC LIMIT 50'),
+
+('Trend Windows Summary', 'Current trend direction and strength by ticker',
+ 'SELECT entity_id AS ticker, window, trend_direction, trend_strength, confidence FROM trend_windows ORDER BY trend_strength DESC LIMIT 100'),
+
+('Market Prices', 'Latest close prices for tracked companies',
+ 'SELECT DISTINCT ON (ticker) ticker, (data->>''c'')::float AS close, (data->>''o'')::float AS open, (data->>''h'')::float AS high, (data->>''l'')::float AS low, (data->>''v'')::float AS volume, captured_at FROM market_snapshots WHERE snapshot_type = ''bar'' ORDER BY ticker, captured_at DESC'),
+
+('Document Counts by Type', 'Count of documents by type and status',
+ 'SELECT document_type, status, count(*) FROM documents GROUP BY document_type, status ORDER BY document_type, count(*) DESC'),
+
+('Global Events', 'Recent macro events with severity',
+ 'SELECT severity, LEFT(summary, 100) AS summary, event_types, affected_sectors, created_at FROM global_events ORDER BY created_at DESC LIMIT 30'),
+
+('Trading Decisions', 'Recent trading engine decisions',
+ 'SELECT ticker, decision, skip_reason, computed_position_size, computed_share_quantity, risk_tier_at_decision, created_at FROM trading_decisions ORDER BY created_at DESC LIMIT 50'),
+
+('Ingestion Health', 'Source ingestion run stats (last 24h)',
+ 'SELECT s.source_type, s.source_name, count(*) AS runs, count(*) FILTER (WHERE ir.status = ''completed'') AS completed, count(*) FILTER (WHERE ir.status = ''failed'') AS failed FROM ingestion_runs ir JOIN sources s ON ir.source_id = s.id WHERE ir.started_at > NOW() - INTERVAL ''24 hours'' GROUP BY s.source_type, s.source_name ORDER BY failed DESC, runs DESC'),
+
+('Reserve Pool Ledger', 'Reserve pool transaction history',
+ 'SELECT amount, balance_after, trigger_type, notes, created_at FROM reserve_pool_ledger ORDER BY created_at DESC LIMIT 30'),
+
+('Sector Exposure', 'Market value by sector from positions',
+ 'SELECT c.sector, count(*) AS positions, sum((ms.data->>''c'')::float) AS total_close_price FROM companies c JOIN market_snapshots ms ON c.ticker = ms.ticker WHERE ms.snapshot_type = ''bar'' GROUP BY c.sector ORDER BY total_close_price DESC'),
+
+('Source Coverage Matrix', 'Active sources per company',
+ 'SELECT c.ticker, c.legal_name, count(*) FILTER (WHERE s.source_type = ''market_api'') AS market, count(*) FILTER (WHERE s.source_type = ''news_api'') AS news, count(*) FILTER (WHERE s.source_type = ''filings_api'') AS filings, count(*) AS total FROM companies c LEFT JOIN sources s ON c.id = s.company_id AND s.active = TRUE WHERE c.active = TRUE GROUP BY c.ticker, c.legal_name ORDER BY c.ticker')
+ON CONFLICT DO NOTHING;
diff --git a/services/api/app.py b/services/api/app.py
index 22062be..2854a48 100644
--- a/services/api/app.py
+++ b/services/api/app.py
@@ -1673,6 +1673,71 @@ async def analytics_schema():
         return {"catalog": trino_catalog, "schema": trino_schema, "tables": []}
 
 
+# ---------------------------------------------------------------------------
+# Analytics: PostgreSQL Direct Query  (Schema browser + read-only SQL)
+# ---------------------------------------------------------------------------
+
+
+@app.get("/api/analytics/pg-schema")
+async def pg_schema():
+    """Return PostgreSQL table/column metadata for the schema browser."""
+    rows = await pool.fetch("""
+        SELECT t.table_name, c.column_name, c.data_type, c.is_nullable
+        FROM information_schema.tables t
+        JOIN information_schema.columns c
+            ON t.table_name = c.table_name AND t.table_schema = c.table_schema
+        WHERE t.table_schema = 'public' AND t.table_type = 'BASE TABLE'
+        ORDER BY t.table_name, c.ordinal_position
+    """)
+    tables: dict[str, dict[str, Any]] = {}
+    for row in rows:
+        tname = row["table_name"]
+        if tname not in tables:
+            tables[tname] = {"name": tname, "columns": []}
+        tables[tname]["columns"].append({
+            "name": row["column_name"],
+            "type": row["data_type"],
+            "nullable": row["is_nullable"] == "YES",
+        })
+    return {"catalog": "postgresql", "schema": "public", "tables": list(tables.values())}
+
+
+@app.post("/api/analytics/pg-query")
+async def pg_query(body: dict[str, Any]):
+    """Run read-only SQL against PostgreSQL directly."""
+    sql = body.get("sql", "").strip()
+    if not sql:
+        raise HTTPException(400, "sql is required")
+
+    limit = min(int(body.get("limit", 1000)), 10000)
+
+    # Safety: only allow SELECT statements
+    if not sql.upper().startswith("SELECT"):
+        raise HTTPException(400, "Only SELECT queries are allowed")
+
+    # Add LIMIT if not present
+    if "LIMIT" not in sql.upper():
+        sql = f"{sql} LIMIT {limit}"
+
+    start = _time.monotonic()
+    try:
+        rows = await pool.fetch(sql)
+        elapsed_ms = round((_time.monotonic() - start) * 1000)
+        columns = [{"name": k, "type": "text"} for k in rows[0].keys()] if rows else []
+        return {
+            "columns": columns,
+            "rows": [[str(v) for v in row.values()] for row in rows],
+            "row_count": len(rows),
+            "elapsed_ms": elapsed_ms,
+        }
+    except asyncpg.PostgresSyntaxError as exc:
+        raise HTTPException(400, f"SQL syntax error: {exc}")
+    except asyncpg.UndefinedTableError as exc:
+        raise HTTPException(400, f"Table not found: {exc}")
+    except asyncpg.PostgresError as exc:
+        raise HTTPException(400, f"Query error: {exc}")
+
+
 # ---------------------------------------------------------------------------
 # Analytics: Saved Queries  (Requirement 13.7)
 # ---------------------------------------------------------------------------