ci: fix lint errors across project, update ruff.toml per-file ignores
ci/woodpecker/push/woodpecker Pipeline failed
Build and Push / lint-and-test (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.adapters.broker_adapter name:broker-adapter]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.aggregation.worker name:aggregation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.extractor.worker name:extractor]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.ingestion.worker name:ingestion]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.lake_publisher.worker name:lake-publisher]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.parser.worker name:parser]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.recommendation.worker name:recommendation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.scheduler.app name:scheduler]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.api.app:app --host 0.0.0.0 --port 8000 name:query-api]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.risk.app:app --host 0.0.0.0 --port 8000 name:risk]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000 name:symbol-registry]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.trading.app:app --host 0.0.0.0 --port 8000 name:trading-engine]) (push) Has been cancelled
Build and Push / build-dashboard (push) Has been cancelled
Build and Push / build-superset (push) Has been cancelled
Build and Push / integration-test (push) Has been cancelled
ci/woodpecker/push/woodpecker Pipeline failed
Build and Push / lint-and-test (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.adapters.broker_adapter name:broker-adapter]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.aggregation.worker name:aggregation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.extractor.worker name:extractor]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.ingestion.worker name:ingestion]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.lake_publisher.worker name:lake-publisher]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.parser.worker name:parser]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.recommendation.worker name:recommendation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.scheduler.app name:scheduler]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.api.app:app --host 0.0.0.0 --port 8000 name:query-api]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.risk.app:app --host 0.0.0.0 --port 8000 name:risk]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000 name:symbol-registry]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.trading.app:app --host 0.0.0.0 --port 8000 name:trading-engine]) (push) Has been cancelled
Build and Push / build-dashboard (push) Has been cancelled
Build and Push / build-superset (push) Has been cancelled
Build and Push / integration-test (push) Has been cancelled
This commit is contained in:
+11
-7
@@ -1,11 +1,15 @@
|
||||
import os
|
||||
|
||||
from minio import Minio
|
||||
import os, json
|
||||
|
||||
mc = Minio(os.environ["MINIO_ENDPOINT"], access_key=os.environ["MINIO_ACCESS_KEY"], secret_key=os.environ["MINIO_SECRET_KEY"], secure=False)
|
||||
|
||||
# Check the most recent extraction - what text did the model get?
|
||||
# Look at the normalized text for a known doc
|
||||
import asyncio, asyncpg
|
||||
import asyncio
|
||||
|
||||
import asyncpg
|
||||
|
||||
|
||||
async def main():
|
||||
pool = await asyncpg.create_pool(
|
||||
@@ -15,20 +19,20 @@ async def main():
|
||||
user=os.environ["POSTGRES_USER"],
|
||||
password=os.environ["POSTGRES_PASSWORD"],
|
||||
)
|
||||
|
||||
|
||||
# Get a recently extracted doc
|
||||
row = await pool.fetchrow(
|
||||
"SELECT id, title, normalized_storage_ref, parse_quality_score "
|
||||
"FROM documents WHERE source_type = 'news_api' AND parse_quality_score > 0.8 "
|
||||
"ORDER BY updated_at DESC LIMIT 1"
|
||||
)
|
||||
|
||||
|
||||
if row:
|
||||
print(f"Doc: {row['id']}")
|
||||
print(f"Title: {row['title']}")
|
||||
print(f"Quality: {row['parse_quality_score']}")
|
||||
print(f"Ref: {row['normalized_storage_ref']}")
|
||||
|
||||
|
||||
ref = row["normalized_storage_ref"]
|
||||
parts = ref.replace("s3://", "").split("/", 1)
|
||||
if len(parts) == 2:
|
||||
@@ -37,9 +41,9 @@ async def main():
|
||||
obj.close()
|
||||
obj.release_conn()
|
||||
print(f"Text length: {len(text)} chars")
|
||||
print(f"First 500 chars:")
|
||||
print("First 500 chars:")
|
||||
print(text[:500])
|
||||
|
||||
|
||||
await pool.close()
|
||||
|
||||
asyncio.run(main())
|
||||
|
||||
Reference in New Issue
Block a user