diff --git a/.kiro/specs/stonks-oracle/tasks.md b/.kiro/specs/stonks-oracle/tasks.md index 9e92d3a..2878478 100644 --- a/.kiro/specs/stonks-oracle/tasks.md +++ b/.kiro/specs/stonks-oracle/tasks.md @@ -137,7 +137,7 @@ ## Phase 15 - CI Validation, Helm Deployment, and Cluster Rollout - [-] 15. Commit, push, validate CI, create Helm chart, and deploy to cluster -- [-] 15.1 Commit and push code to GitHub +- [x] 15.1 Commit and push code to GitHub - Configure git with SSH key for the private repo - Commit all current changes with message `phase 14-15: docker build validation and helm deployment` - Push to main branch diff --git a/services/adapters/__init__.py b/services/adapters/__init__.py index b4022bf..4f884f2 100644 --- a/services/adapters/__init__.py +++ b/services/adapters/__init__.py @@ -1,6 +1,5 @@ # Ingestion Adapters from .base import AdapterResult, BaseAdapter -from .resilient import ResilientAdapter, RetryConfig, RetryStats, compute_delay from .broker_adapter import ( AccountInfo, AlpacaBrokerAdapter, @@ -17,6 +16,7 @@ from .broker_adapter import ( from .filings_adapter import FilingsDataAdapter, SECEdgarAdapter from .market_adapter import MarketDataAdapter, PolygonMarketAdapter from .news_adapter import NewsDataAdapter, PolygonNewsAdapter +from .resilient import ResilientAdapter, RetryConfig, RetryStats, compute_delay __all__ = [ "AccountInfo", diff --git a/services/adapters/broker_service.py b/services/adapters/broker_service.py index 6695377..0b0cf4a 100644 --- a/services/adapters/broker_service.py +++ b/services/adapters/broker_service.py @@ -36,19 +36,24 @@ from services.adapters.broker_adapter import ( OrderType, TradingMode, ) +from services.lake_publisher.worker import ( + LAKEHOUSE_BUCKET, + publish_positions_daily_batch, + publish_trade_fill, + publish_trade_order, +) +from services.risk.approval import ( + ApprovalRequest, + compute_expiry, + create_approval_request, + requires_approval, +) from services.risk.engine import ( AccountRiskState, PortfolioRiskConfig, ProposedOrder, evaluate_order, ) -from services.risk.approval import ( - ApprovalRequest, - ApprovalStatus, - compute_expiry, - create_approval_request, - requires_approval, -) from services.shared.audit import ( audit_approval_requested, audit_duplicate_prevented, @@ -57,15 +62,9 @@ from services.shared.audit import ( audit_order_submitted, audit_risk_evaluated, ) -from services.lake_publisher.worker import ( - publish_trade_order, - publish_trade_fill, - publish_positions_daily_batch, - LAKEHOUSE_BUCKET, -) from services.shared.config import load_config from services.shared.db import get_pg_pool, get_redis -from services.shared.logging import Span, new_trace_id, set_trace_context, setup_logging +from services.shared.logging import setup_logging from services.shared.metrics import ( ORDERS_DUPLICATES_PREVENTED, ORDERS_FILLED, diff --git a/services/adapters/paper_trading.py b/services/adapters/paper_trading.py index 701ae06..739d1b1 100644 --- a/services/adapters/paper_trading.py +++ b/services/adapters/paper_trading.py @@ -18,6 +18,7 @@ from typing import Any import asyncpg +from services.adapters.base import AdapterResult from services.adapters.broker_adapter import ( AccountInfo, BrokerDataAdapter, @@ -30,7 +31,6 @@ from services.adapters.broker_adapter import ( PositionInfo, TradingMode, ) -from services.adapters.base import AdapterResult logger = logging.getLogger("paper_trading") diff --git a/services/adapters/web_scrape_adapter.py b/services/adapters/web_scrape_adapter.py index 1aae573..0203f3f 100644 --- a/services/adapters/web_scrape_adapter.py +++ b/services/adapters/web_scrape_adapter.py @@ -13,8 +13,8 @@ import json import logging import time from datetime import datetime, timezone -from urllib.parse import urlparse from typing import Any +from urllib.parse import urlparse import httpx from bs4 import BeautifulSoup diff --git a/services/aggregation/scoring.py b/services/aggregation/scoring.py index b66818b..7e3e62b 100644 --- a/services/aggregation/scoring.py +++ b/services/aggregation/scoring.py @@ -15,7 +15,6 @@ from datetime import datetime, timezone from services.shared.schemas import MarketContext - @dataclass(frozen=True) class ScoringConfig: """Tunable parameters for signal scoring.""" diff --git a/services/aggregation/worker.py b/services/aggregation/worker.py index e56823c..700cf9c 100644 --- a/services/aggregation/worker.py +++ b/services/aggregation/worker.py @@ -21,9 +21,11 @@ from services.aggregation.contradiction import CatalystEntry, detect_contradicti from services.aggregation.evidence import ( EvidenceRankConfig, RankedEvidence, - rank_evidence as _rank_evidence_composite, rank_evidence_detailed, ) +from services.aggregation.evidence import ( + rank_evidence as _rank_evidence_composite, +) from services.aggregation.market_context import fetch_market_context from services.aggregation.scoring import ( ScoringConfig, @@ -32,13 +34,13 @@ from services.aggregation.scoring import ( sentiment_to_numeric, weighted_sentiment_average, ) -from services.shared.schemas import TrendDirection, TrendSummary, TrendWindow from services.shared.metrics import ( AGGREGATION_CONTRADICTION_SCORE, AGGREGATION_DURATION, AGGREGATION_SIGNALS_PROCESSED, AGGREGATION_WINDOWS_COMPUTED, ) +from services.shared.schemas import TrendDirection, TrendSummary, TrendWindow logger = logging.getLogger(__name__) diff --git a/services/api/app.py b/services/api/app.py index 2d3f2ae..b3f066b 100644 --- a/services/api/app.py +++ b/services/api/app.py @@ -20,15 +20,15 @@ from typing import Any, Optional import asyncpg from fastapi import FastAPI, HTTPException, Query, Request +from prometheus_client import CONTENT_TYPE_LATEST, generate_latest from starlette.middleware.base import BaseHTTPMiddleware +from starlette.responses import Response +from services.extractor.metrics import get_model_performance_summary from services.shared.audit import get_entity_audit_trail, get_order_audit_trail from services.shared.config import load_config from services.shared.db import get_pg_pool from services.shared.logging import new_trace_id, set_trace_context, setup_logging -from services.extractor.metrics import get_model_performance_summary -from prometheus_client import generate_latest, CONTENT_TYPE_LATEST -from starlette.responses import Response logger = logging.getLogger("query_api") @@ -339,7 +339,7 @@ async def list_trends( offset: int = 0, ): """List trend summaries with optional filters.""" - conditions = [f"entity_type = $1"] + conditions = ["entity_type = $1"] params: list[Any] = [entity_type] idx = 2 diff --git a/services/extractor/main.py b/services/extractor/main.py index c4b0ae6..20634d5 100644 --- a/services/extractor/main.py +++ b/services/extractor/main.py @@ -30,6 +30,7 @@ async def main() -> None: ollama = OllamaClient(config.ollama) import json + import redis.asyncio as aioredis redis_client = aioredis.from_url(config.redis.url) diff --git a/services/extractor/prompts.py b/services/extractor/prompts.py index b62c2df..8b20a7a 100644 --- a/services/extractor/prompts.py +++ b/services/extractor/prompts.py @@ -11,7 +11,7 @@ from __future__ import annotations import json from typing import Any -from services.extractor.schemas import generate_json_schema, SCHEMA_VERSION +from services.extractor.schemas import SCHEMA_VERSION, generate_json_schema from services.shared.schemas import ( DocumentType, ) diff --git a/services/extractor/worker.py b/services/extractor/worker.py index d6afe69..1e4a76f 100644 --- a/services/extractor/worker.py +++ b/services/extractor/worker.py @@ -25,13 +25,6 @@ from services.shared.metadata import ( persist_document_intelligence, update_document_status, ) -from services.shared.storage import ( - upload_extraction_intelligence, - upload_extraction_prompt, - upload_extraction_raw_output, - upload_extraction_validation, -) -from services.shared.logging import Span from services.shared.metrics import ( EXTRACTION_ATTEMPTS, EXTRACTION_CONFIDENCE, @@ -41,6 +34,12 @@ from services.shared.metrics import ( EXTRACTION_TOKEN_ESTIMATE, EXTRACTION_VALIDATION_ERRORS, ) +from services.shared.storage import ( + upload_extraction_intelligence, + upload_extraction_prompt, + upload_extraction_raw_output, + upload_extraction_validation, +) logger = logging.getLogger("extractor_worker") diff --git a/services/ingestion/worker.py b/services/ingestion/worker.py index b1b8728..4403324 100644 --- a/services/ingestion/worker.py +++ b/services/ingestion/worker.py @@ -16,20 +16,19 @@ from services.adapters.web_scrape_adapter import WebScrapeAdapter from services.shared.config import load_config from services.shared.db import get_minio, get_pg_pool, get_redis from services.shared.dedupe import dedupe_items, mark_as_seen +from services.shared.logging import ( + Span, + inject_trace_context, + new_trace_id, + set_trace_context, + setup_logging, +) from services.shared.metadata import ( persist_ingestion_items, record_retrieval_failure, reset_source_retry_state, ) -from services.shared.redis_keys import ( - QUEUE_INGESTION, - QUEUE_PARSING, - dedupe_key, - queue_key, -) -from services.shared.logging import Span, extract_trace_context, inject_trace_context, new_trace_id, set_trace_context, setup_logging from services.shared.metrics import ( - ACTIVE_JOBS, INGESTION_ADAPTER_DURATION, INGESTION_ERRORS, INGESTION_ITEMS_DEDUPED, @@ -37,8 +36,13 @@ from services.shared.metrics import ( INGESTION_ITEMS_NEW, INGESTION_JOBS_TOTAL, ) +from services.shared.redis_keys import ( + QUEUE_INGESTION, + QUEUE_PARSING, + dedupe_key, + queue_key, +) from services.shared.storage import ( - bucket_for_source, ensure_buckets, upload_raw_artifact, ) @@ -88,7 +92,6 @@ async def process_job( return # Store raw payload in MinIO - bucket = bucket_for_source(source_type) artifact_type = "raw_html" if source_type == "web_scrape" else "raw_json" storage_uri = upload_raw_artifact( minio_client, diff --git a/services/lake_publisher/iceberg.py b/services/lake_publisher/iceberg.py index 6b4a3d8..b7ecf06 100644 --- a/services/lake_publisher/iceberg.py +++ b/services/lake_publisher/iceberg.py @@ -30,8 +30,8 @@ from services.lake_publisher.partitions import ( ) from services.lake_publisher.worker import ( COMPANY_EVENTS_SCHEMA, - DOCUMENTS_SCHEMA, DOCUMENT_EXTRACTIONS_SCHEMA, + DOCUMENTS_SCHEMA, MARKET_BARS_SCHEMA, MARKET_QUOTES_SCHEMA, MODEL_PERFORMANCE_SCHEMA, diff --git a/services/lake_publisher/jobs.py b/services/lake_publisher/jobs.py index 5c1c6c0..9c3468e 100644 --- a/services/lake_publisher/jobs.py +++ b/services/lake_publisher/jobs.py @@ -33,19 +33,19 @@ import asyncpg import redis.asyncio as aioredis from minio import Minio +from services.lake_publisher.partitions import partition_values from services.lake_publisher.worker import ( publish_document_extraction, + publish_document_extractions_batch, publish_document_fact, + publish_documents_batch, publish_market_bar, publish_market_quote, - publish_trade_order, - publish_trade_fill, publish_pnl_daily, - publish_documents_batch, - publish_document_extractions_batch, publish_positions_daily_batch, + publish_trade_fill, + publish_trade_order, ) -from services.lake_publisher.partitions import partition_values from services.shared.config import load_config from services.shared.db import get_minio, get_pg_pool, get_redis from services.shared.logging import setup_logging diff --git a/services/lake_publisher/partitions.py b/services/lake_publisher/partitions.py index 0b47cd7..49b4eab 100644 --- a/services/lake_publisher/partitions.py +++ b/services/lake_publisher/partitions.py @@ -21,8 +21,7 @@ from __future__ import annotations import uuid from dataclasses import dataclass, field -from datetime import date, datetime, timezone - +from datetime import date, datetime LAKEHOUSE_BUCKET = "stonks-lakehouse" WAREHOUSE_PREFIX = "warehouse" diff --git a/services/lake_publisher/worker.py b/services/lake_publisher/worker.py index c7dfe19..c55fba6 100644 --- a/services/lake_publisher/worker.py +++ b/services/lake_publisher/worker.py @@ -28,7 +28,6 @@ from services.shared.metrics import ( LAKE_FACTS_PUBLISHED, LAKE_PUBLISH_BYTES, LAKE_PUBLISH_DURATION, - LAKE_PUBLISH_ERRORS, ) from services.shared.schemas import Recommendation diff --git a/services/parser/worker.py b/services/parser/worker.py index b30efab..5547947 100644 --- a/services/parser/worker.py +++ b/services/parser/worker.py @@ -22,15 +22,19 @@ from minio import Minio from services.parser.html_parser import ParsedDocument, detect_company_mentions, parse_html from services.shared.config import load_config from services.shared.db import get_minio, get_pg_pool, get_redis -from services.shared.logging import Span, extract_trace_context, inject_trace_context, new_trace_id, set_trace_context, setup_logging +from services.shared.logging import ( + inject_trace_context, + new_trace_id, + set_trace_context, + setup_logging, +) +from services.shared.metadata import update_document_parse_results from services.shared.metrics import ( - ACTIVE_JOBS, PARSE_DURATION, PARSE_JOBS_TOTAL, PARSE_LOW_QUALITY_TOTAL, PARSE_QUALITY_SCORE, ) -from services.shared.metadata import update_document_parse_results from services.shared.redis_keys import QUEUE_EXTRACTION, QUEUE_PARSING, queue_key from services.shared.storage import upload_normalized_text, upload_parser_output diff --git a/services/recommendation/worker.py b/services/recommendation/worker.py index b332e8e..5ec11f9 100644 --- a/services/recommendation/worker.py +++ b/services/recommendation/worker.py @@ -13,7 +13,9 @@ import logging from datetime import datetime, timezone import asyncpg +from minio import Minio +from services.lake_publisher.worker import publish_recommendation_facts from services.recommendation.eligibility import ( EligibilityConfig, EligibilityResult, @@ -29,10 +31,12 @@ from services.recommendation.thesis_llm import ( THESIS_PROMPT_VERSION, rewrite_thesis_with_llm, ) -from minio import Minio - -from services.lake_publisher.worker import publish_recommendation_facts from services.shared.config import OllamaConfig +from services.shared.metrics import ( + RECOMMENDATION_CONFIDENCE, + RECOMMENDATION_GENERATED, + RECOMMENDATION_SUPPRESSED, +) from services.shared.schemas import ( ModelMetadata, PositionSizing, @@ -42,11 +46,6 @@ from services.shared.schemas import ( TrendSummary, TrendWindow, ) -from services.shared.metrics import ( - RECOMMENDATION_CONFIDENCE, - RECOMMENDATION_GENERATED, - RECOMMENDATION_SUPPRESSED, -) logger = logging.getLogger(__name__) diff --git a/services/risk/approval.py b/services/risk/approval.py index ff2c672..3381a55 100644 --- a/services/risk/approval.py +++ b/services/risk/approval.py @@ -19,7 +19,6 @@ from typing import Any import asyncpg from services.risk.engine import ( - OperatorApproval, PortfolioRiskConfig, TradingMode, ) diff --git a/services/risk/engine.py b/services/risk/engine.py index a7d7837..f0a9dc9 100644 --- a/services/risk/engine.py +++ b/services/risk/engine.py @@ -19,7 +19,6 @@ from typing import Any from pydantic import BaseModel, Field - # --------------------------------------------------------------------------- # Enums # ---------------------------------------------------------------------------