phase 17: remove embedded JSON schema from user prompt (4.7KB saved), Ollama format param handles it

scripts/check_input.py

@@ -0,0 +1,45 @@
+from minio import Minio
+import os, json
+
+mc = Minio(os.environ["MINIO_ENDPOINT"], access_key=os.environ["MINIO_ACCESS_KEY"], secret_key=os.environ["MINIO_SECRET_KEY"], secure=False)
+
+# Check the most recent extraction - what text did the model get?
+# Look at the normalized text for a known doc
+import asyncio, asyncpg
+
+async def main():
+    pool = await asyncpg.create_pool(
+        host=os.environ["POSTGRES_HOST"],
+        port=int(os.environ["POSTGRES_PORT"]),
+        database=os.environ["POSTGRES_DB"],
+        user=os.environ["POSTGRES_USER"],
+        password=os.environ["POSTGRES_PASSWORD"],
+    )
+    
+    # Get a recently extracted doc
+    row = await pool.fetchrow(
+        "SELECT id, title, normalized_storage_ref, parse_quality_score "
+        "FROM documents WHERE source_type = 'news_api' AND parse_quality_score > 0.8 "
+        "ORDER BY updated_at DESC LIMIT 1"
+    )
+    
+    if row:
+        print(f"Doc: {row['id']}")
+        print(f"Title: {row['title']}")
+        print(f"Quality: {row['parse_quality_score']}")
+        print(f"Ref: {row['normalized_storage_ref']}")
+        
+        ref = row["normalized_storage_ref"]
+        parts = ref.replace("s3://", "").split("/", 1)
+        if len(parts) == 2:
+            obj = mc.get_object(parts[0], parts[1])
+            text = obj.read().decode("utf-8")
+            obj.close()
+            obj.release_conn()
+            print(f"Text length: {len(text)} chars")
+            print(f"First 500 chars:")
+            print(text[:500])
+    
+    await pool.close()
+
+asyncio.run(main())

scripts/reenqueue_test.py

@@ -0,0 +1,29 @@
+import asyncio, asyncpg, json, os, redis
+
+async def main():
+    pool = await asyncpg.create_pool(
+        host=os.environ["POSTGRES_HOST"],
+        port=int(os.environ["POSTGRES_PORT"]),
+        database=os.environ["POSTGRES_DB"],
+        user=os.environ["POSTGRES_USER"],
+        password=os.environ["POSTGRES_PASSWORD"],
+    )
+    r = redis.from_url(f"redis://:{os.environ.get('REDIS_PASSWORD', '')}@{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}/0")
+    
+    rows = await pool.fetch(
+        "SELECT d.id, dcm.ticker FROM documents d "
+        "LEFT JOIN document_company_mentions dcm ON d.id = dcm.document_id "
+        "WHERE d.source_type = 'news_api' AND d.parse_quality_score > 0.7 "
+        "ORDER BY d.parse_quality_score DESC LIMIT 5"
+    )
+    
+    for row in rows:
+        r.rpush("stonks:queue:extraction", json.dumps({
+            "document_id": str(row["id"]),
+            "ticker": row["ticker"] or "",
+        }))
+    
+    print(f"Enqueued {len(rows)} high-quality docs for re-extraction")
+    await pool.close()
+
+asyncio.run(main())