phase 0+1: project scaffold, k8s manifests, CI pipeline, steering, hooks, tests
- Repository structure for all services, infra, lakehouse, dashboards - K8s manifests targeting stonks-oracle namespace with GHCR images - Ingress via Traefik with ca-issuer TLS for internal services - ConfigMap wired to existing cluster services (pg, redis, minio, ollama) - GitHub Actions workflow for lint, test, multi-service container builds - Dockerfile with build-arg CMD per service - Makefile for local build/push/deploy - Steering rules for TDD workflow, K8s conventions, project context - Agent hooks for lint-on-save, test-on-save, k8s-validate, phase-commit - Ruff linter config, all lint issues fixed - 14 passing tests for schemas, config, redis keys - PostgreSQL migrations, Trino catalogs, Superset config, MinIO lifecycle
This commit is contained in:
Celes Renata
@@ -0,0 +1 @@
|
||||
# Stonks Oracle - Shared modules
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Shared configuration loader for all services."""
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class PostgresConfig:
|
||||
host: str = "localhost"
|
||||
port: int = 5432
|
||||
database: str = "stonks"
|
||||
user: str = "stonks"
|
||||
password: str = "stonks_dev"
|
||||
|
||||
@property
|
||||
def dsn(self) -> str:
|
||||
return f"postgresql://{self.user}:{self.password}@{self.host}:{self.port}/{self.database}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class RedisConfig:
|
||||
host: str = "localhost"
|
||||
port: int = 6379
|
||||
db: int = 0
|
||||
password: Optional[str] = None
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
auth = f":{self.password}@" if self.password else ""
|
||||
return f"redis://{auth}{self.host}:{self.port}/{self.db}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MinioConfig:
|
||||
endpoint: str = "localhost:9000"
|
||||
access_key: str = "minioadmin"
|
||||
secret_key: str = "minioadmin"
|
||||
secure: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class OllamaConfig:
|
||||
base_url: str = "http://localhost:11434"
|
||||
model: str = "llama3.1:8b"
|
||||
timeout: int = 120
|
||||
|
||||
|
||||
@dataclass
|
||||
class TrinoConfig:
|
||||
host: str = "localhost"
|
||||
port: int = 8080
|
||||
catalog: str = "lakehouse"
|
||||
schema: str = "stonks"
|
||||
|
||||
|
||||
@dataclass
|
||||
class BrokerConfig:
|
||||
mode: str = "paper" # paper | live
|
||||
api_key: Optional[str] = None
|
||||
api_secret: Optional[str] = None
|
||||
base_url: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AppConfig:
|
||||
postgres: PostgresConfig = field(default_factory=PostgresConfig)
|
||||
redis: RedisConfig = field(default_factory=RedisConfig)
|
||||
minio: MinioConfig = field(default_factory=MinioConfig)
|
||||
ollama: OllamaConfig = field(default_factory=OllamaConfig)
|
||||
trino: TrinoConfig = field(default_factory=TrinoConfig)
|
||||
broker: BrokerConfig = field(default_factory=BrokerConfig)
|
||||
log_level: str = "INFO"
|
||||
|
||||
|
||||
def load_config() -> AppConfig:
|
||||
"""Load configuration from environment variables with sensible defaults."""
|
||||
return AppConfig(
|
||||
postgres=PostgresConfig(
|
||||
host=os.getenv("POSTGRES_HOST", "localhost"),
|
||||
port=int(os.getenv("POSTGRES_PORT", "5432")),
|
||||
database=os.getenv("POSTGRES_DB", "stonks"),
|
||||
user=os.getenv("POSTGRES_USER", "stonks"),
|
||||
password=os.getenv("POSTGRES_PASSWORD", "stonks_dev"),
|
||||
),
|
||||
redis=RedisConfig(
|
||||
host=os.getenv("REDIS_HOST", "localhost"),
|
||||
port=int(os.getenv("REDIS_PORT", "6379")),
|
||||
db=int(os.getenv("REDIS_DB", "0")),
|
||||
password=os.getenv("REDIS_PASSWORD", None),
|
||||
),
|
||||
minio=MinioConfig(
|
||||
endpoint=os.getenv("MINIO_ENDPOINT", "localhost:9000"),
|
||||
access_key=os.getenv("MINIO_ACCESS_KEY", "minioadmin"),
|
||||
secret_key=os.getenv("MINIO_SECRET_KEY", "minioadmin"),
|
||||
secure=os.getenv("MINIO_SECURE", "false").lower() == "true",
|
||||
),
|
||||
ollama=OllamaConfig(
|
||||
base_url=os.getenv("OLLAMA_BASE_URL", "http://localhost:11434"),
|
||||
model=os.getenv("OLLAMA_MODEL", "llama3.1:8b"),
|
||||
timeout=int(os.getenv("OLLAMA_TIMEOUT", "120")),
|
||||
),
|
||||
trino=TrinoConfig(
|
||||
host=os.getenv("TRINO_HOST", "localhost"),
|
||||
port=int(os.getenv("TRINO_PORT", "8080")),
|
||||
catalog=os.getenv("TRINO_CATALOG", "lakehouse"),
|
||||
schema=os.getenv("TRINO_SCHEMA", "stonks"),
|
||||
),
|
||||
broker=BrokerConfig(
|
||||
mode=os.getenv("BROKER_MODE", "paper"),
|
||||
api_key=os.getenv("BROKER_API_KEY", None),
|
||||
api_secret=os.getenv("BROKER_API_SECRET", None),
|
||||
base_url=os.getenv("BROKER_BASE_URL", None),
|
||||
),
|
||||
log_level=os.getenv("LOG_LEVEL", "INFO"),
|
||||
)
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Database connection helpers."""
|
||||
import asyncpg
|
||||
import redis.asyncio as aioredis
|
||||
from minio import Minio
|
||||
|
||||
from .config import AppConfig
|
||||
|
||||
|
||||
async def get_pg_pool(config: AppConfig) -> asyncpg.Pool:
|
||||
"""Create a PostgreSQL connection pool."""
|
||||
return await asyncpg.create_pool(
|
||||
dsn=config.postgres.dsn,
|
||||
min_size=2,
|
||||
max_size=10,
|
||||
)
|
||||
|
||||
|
||||
def get_redis(config: AppConfig) -> aioredis.Redis:
|
||||
"""Create a Redis async client."""
|
||||
return aioredis.from_url(
|
||||
config.redis.url,
|
||||
decode_responses=True,
|
||||
)
|
||||
|
||||
|
||||
def get_minio(config: AppConfig) -> Minio:
|
||||
"""Create a MinIO client."""
|
||||
return Minio(
|
||||
config.minio.endpoint,
|
||||
access_key=config.minio.access_key,
|
||||
secret_key=config.minio.secret_key,
|
||||
secure=config.minio.secure,
|
||||
)
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Redis key conventions and queue abstractions."""
|
||||
|
||||
# --- Key prefixes ---
|
||||
PREFIX = "stonks"
|
||||
|
||||
# Distributed locks
|
||||
LOCK_PREFIX = f"{PREFIX}:lock"
|
||||
|
||||
# Rate limit counters
|
||||
RATE_LIMIT_PREFIX = f"{PREFIX}:ratelimit"
|
||||
|
||||
# Job queues
|
||||
QUEUE_PREFIX = f"{PREFIX}:queue"
|
||||
|
||||
# Dedupe markers
|
||||
DEDUPE_PREFIX = f"{PREFIX}:dedupe"
|
||||
|
||||
# Cache
|
||||
CACHE_PREFIX = f"{PREFIX}:cache"
|
||||
|
||||
# Retry backoff state
|
||||
RETRY_PREFIX = f"{PREFIX}:retry"
|
||||
|
||||
|
||||
def lock_key(resource: str) -> str:
|
||||
return f"{LOCK_PREFIX}:{resource}"
|
||||
|
||||
|
||||
def rate_limit_key(source: str, window: str) -> str:
|
||||
return f"{RATE_LIMIT_PREFIX}:{source}:{window}"
|
||||
|
||||
|
||||
def queue_key(queue_name: str) -> str:
|
||||
return f"{QUEUE_PREFIX}:{queue_name}"
|
||||
|
||||
|
||||
def dedupe_key(content_hash: str) -> str:
|
||||
return f"{DEDUPE_PREFIX}:{content_hash}"
|
||||
|
||||
|
||||
def cache_key(namespace: str, key: str) -> str:
|
||||
return f"{CACHE_PREFIX}:{namespace}:{key}"
|
||||
|
||||
|
||||
def retry_key(job_id: str) -> str:
|
||||
return f"{RETRY_PREFIX}:{job_id}"
|
||||
|
||||
|
||||
# --- Queue names ---
|
||||
QUEUE_INGESTION = "ingestion"
|
||||
QUEUE_PARSING = "parsing"
|
||||
QUEUE_EXTRACTION = "extraction"
|
||||
QUEUE_AGGREGATION = "aggregation"
|
||||
QUEUE_RECOMMENDATION = "recommendation"
|
||||
QUEUE_LAKE_PUBLISH = "lake_publish"
|
||||
QUEUE_TRADE = "trade"
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Typed JSON schemas for document intelligence, trend summaries, and recommendations."""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# --- Enums ---
|
||||
|
||||
class DocumentType(str, Enum):
|
||||
ARTICLE = "article"
|
||||
FILING = "filing"
|
||||
TRANSCRIPT = "transcript"
|
||||
PRESS_RELEASE = "press_release"
|
||||
|
||||
|
||||
class SourceType(str, Enum):
|
||||
MARKET_API = "market_api"
|
||||
NEWS_API = "news_api"
|
||||
FILINGS_API = "filings_api"
|
||||
WEB_SCRAPE = "web_scrape"
|
||||
BROKER = "broker"
|
||||
|
||||
|
||||
class Sentiment(str, Enum):
|
||||
POSITIVE = "positive"
|
||||
NEGATIVE = "negative"
|
||||
NEUTRAL = "neutral"
|
||||
MIXED = "mixed"
|
||||
|
||||
|
||||
class CatalystType(str, Enum):
|
||||
EARNINGS = "earnings"
|
||||
PRODUCT = "product"
|
||||
LEGAL = "legal"
|
||||
MACRO = "macro"
|
||||
SUPPLY_CHAIN = "supply_chain"
|
||||
M_AND_A = "m_and_a"
|
||||
RATING_CHANGE = "rating_change"
|
||||
OTHER = "other"
|
||||
|
||||
|
||||
class TrendDirection(str, Enum):
|
||||
BULLISH = "bullish"
|
||||
BEARISH = "bearish"
|
||||
MIXED = "mixed"
|
||||
NEUTRAL = "neutral"
|
||||
|
||||
|
||||
class ActionType(str, Enum):
|
||||
BUY = "buy"
|
||||
SELL = "sell"
|
||||
HOLD = "hold"
|
||||
WATCH = "watch"
|
||||
|
||||
|
||||
class RecommendationMode(str, Enum):
|
||||
INFORMATIONAL = "informational"
|
||||
PAPER_ELIGIBLE = "paper_eligible"
|
||||
LIVE_ELIGIBLE = "live_eligible"
|
||||
|
||||
|
||||
class TrendWindow(str, Enum):
|
||||
INTRADAY = "intraday"
|
||||
ONE_DAY = "1d"
|
||||
SEVEN_DAY = "7d"
|
||||
THIRTY_DAY = "30d"
|
||||
NINETY_DAY = "90d"
|
||||
|
||||
|
||||
# --- Document Intelligence ---
|
||||
|
||||
class CompanyImpact(BaseModel):
|
||||
ticker: str
|
||||
company_name: str
|
||||
relevance: float = Field(ge=0, le=1)
|
||||
sentiment: Sentiment
|
||||
impact_score: float = Field(ge=0, le=1)
|
||||
impact_horizon: str
|
||||
catalyst_type: CatalystType
|
||||
key_facts: List[str] = Field(default_factory=list)
|
||||
risks: List[str] = Field(default_factory=list)
|
||||
evidence_spans: List[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ModelMetadata(BaseModel):
|
||||
provider: str = "ollama"
|
||||
model_name: str = ""
|
||||
prompt_version: str = ""
|
||||
schema_version: str = "2.0.0"
|
||||
|
||||
|
||||
class DocumentIntelligence(BaseModel):
|
||||
document_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
||||
document_type: DocumentType = DocumentType.ARTICLE
|
||||
summary: str = ""
|
||||
companies: List[CompanyImpact] = Field(default_factory=list)
|
||||
macro_themes: List[str] = Field(default_factory=list)
|
||||
novelty_score: float = Field(ge=0, le=1, default=0.5)
|
||||
source_credibility: float = Field(ge=0, le=1, default=0.5)
|
||||
extraction_warnings: List[str] = Field(default_factory=list)
|
||||
confidence: float = Field(ge=0, le=1, default=0.5)
|
||||
model: ModelMetadata = Field(default_factory=ModelMetadata)
|
||||
|
||||
|
||||
# --- Trend Summary ---
|
||||
|
||||
class TrendSummary(BaseModel):
|
||||
entity_type: str = "company"
|
||||
entity_id: str = ""
|
||||
window: TrendWindow = TrendWindow.SEVEN_DAY
|
||||
trend_direction: TrendDirection = TrendDirection.NEUTRAL
|
||||
trend_strength: float = Field(ge=0, le=1, default=0.5)
|
||||
confidence: float = Field(ge=0, le=1, default=0.5)
|
||||
top_supporting_evidence: List[str] = Field(default_factory=list)
|
||||
top_opposing_evidence: List[str] = Field(default_factory=list)
|
||||
dominant_catalysts: List[str] = Field(default_factory=list)
|
||||
material_risks: List[str] = Field(default_factory=list)
|
||||
contradiction_score: float = Field(ge=0, le=1, default=0.0)
|
||||
generated_at: datetime = Field(default_factory=datetime.utcnow)
|
||||
|
||||
|
||||
# --- Recommendation ---
|
||||
|
||||
class PositionSizing(BaseModel):
|
||||
portfolio_pct: float = Field(ge=0, le=1, default=0.02)
|
||||
max_loss_pct: float = Field(ge=0, le=1, default=0.005)
|
||||
|
||||
|
||||
class Recommendation(BaseModel):
|
||||
recommendation_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
||||
ticker: str = ""
|
||||
action: ActionType = ActionType.WATCH
|
||||
mode: RecommendationMode = RecommendationMode.INFORMATIONAL
|
||||
confidence: float = Field(ge=0, le=1, default=0.5)
|
||||
time_horizon: str = ""
|
||||
thesis: str = ""
|
||||
invalidation_conditions: List[str] = Field(default_factory=list)
|
||||
position_sizing: PositionSizing = Field(default_factory=PositionSizing)
|
||||
evidence_refs: List[str] = Field(default_factory=list)
|
||||
model_metadata: ModelMetadata = Field(default_factory=ModelMetadata)
|
||||
generated_at: datetime = Field(default_factory=datetime.utcnow)
|
||||
|
||||
|
||||
# --- Document Metadata ---
|
||||
|
||||
class StorageRefs(BaseModel):
|
||||
raw_html: Optional[str] = None
|
||||
raw_payload: Optional[str] = None
|
||||
normalized_text: Optional[str] = None
|
||||
|
||||
|
||||
class DocumentMetadata(BaseModel):
|
||||
document_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
||||
document_type: DocumentType = DocumentType.ARTICLE
|
||||
symbol_candidates: List[str] = Field(default_factory=list)
|
||||
source_type: SourceType = SourceType.NEWS_API
|
||||
publisher: str = ""
|
||||
url: Optional[str] = None
|
||||
canonical_url: Optional[str] = None
|
||||
title: str = ""
|
||||
published_at: Optional[datetime] = None
|
||||
retrieved_at: datetime = Field(default_factory=datetime.utcnow)
|
||||
language: str = "en"
|
||||
content_hash: str = ""
|
||||
storage_refs: StorageRefs = Field(default_factory=StorageRefs)
|
||||
Reference in New Issue
Block a user