88ad1e8d99
- Add scheduler and ingestion unit tests (test_scheduler_unit.py, test_ingestion_unit.py) - Add all 13 app services + dashboard to docker-compose.yml - Add full documentation suite: API reference, Helm reference, Docker deployment guide, 3 architecture diagrams (K8s, Docker Compose, data pipeline), AI agent guide, backup/restore guide, observability/metrics reference, per-service docs - Add intelligence pipeline deep-dive docs with Mermaid diagrams - Update README with documentation index and links - Add specs for comprehensive-quality-docs, intelligence-pipeline-deep-dive, sanitized-pipeline-docs
418 lines
10 KiB
YAML
418 lines
10 KiB
YAML
version: "3.9"
|
|
|
|
x-app-env: &app-env
|
|
POSTGRES_HOST: postgres
|
|
POSTGRES_PORT: "5432"
|
|
POSTGRES_DB: stonks
|
|
POSTGRES_USER: stonks
|
|
POSTGRES_PASSWORD: stonks_dev
|
|
REDIS_HOST: redis
|
|
REDIS_PORT: "6379"
|
|
MINIO_ENDPOINT: minio:9000
|
|
MINIO_ACCESS_KEY: minioadmin
|
|
MINIO_SECRET_KEY: minioadmin
|
|
OLLAMA_BASE_URL: http://ollama:11434
|
|
|
|
services:
|
|
# ── Infrastructure ──────────────────────────────────────────────
|
|
|
|
postgres:
|
|
image: postgres:16-alpine
|
|
environment:
|
|
POSTGRES_DB: stonks
|
|
POSTGRES_USER: stonks
|
|
POSTGRES_PASSWORD: stonks_dev
|
|
ports:
|
|
- "5432:5432"
|
|
volumes:
|
|
- pgdata:/var/lib/postgresql/data
|
|
- ./infra/migrations:/docker-entrypoint-initdb.d
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U stonks"]
|
|
interval: 5s
|
|
retries: 5
|
|
|
|
redis:
|
|
image: redis:7-alpine
|
|
ports:
|
|
- "6379:6379"
|
|
healthcheck:
|
|
test: ["CMD", "redis-cli", "ping"]
|
|
interval: 5s
|
|
retries: 5
|
|
|
|
minio:
|
|
image: minio/minio:latest
|
|
command: server /data --console-address ":9001"
|
|
environment:
|
|
MINIO_ROOT_USER: minioadmin
|
|
MINIO_ROOT_PASSWORD: minioadmin
|
|
ports:
|
|
- "9000:9000"
|
|
- "9001:9001"
|
|
volumes:
|
|
- miniodata:/data
|
|
healthcheck:
|
|
test: ["CMD", "mc", "ready", "local"]
|
|
interval: 5s
|
|
retries: 5
|
|
|
|
minio-init:
|
|
image: minio/mc:latest
|
|
depends_on:
|
|
minio:
|
|
condition: service_healthy
|
|
entrypoint: >
|
|
/bin/sh -c "
|
|
mc alias set local http://minio:9000 minioadmin minioadmin;
|
|
mc mb --ignore-existing local/stonks-raw-market;
|
|
mc mb --ignore-existing local/stonks-raw-news;
|
|
mc mb --ignore-existing local/stonks-raw-filings;
|
|
mc mb --ignore-existing local/stonks-normalized;
|
|
mc mb --ignore-existing local/stonks-llm-prompts;
|
|
mc mb --ignore-existing local/stonks-llm-results;
|
|
mc mb --ignore-existing local/stonks-lakehouse;
|
|
mc mb --ignore-existing local/stonks-audit;
|
|
exit 0;
|
|
"
|
|
|
|
ollama:
|
|
image: ollama/ollama:latest
|
|
ports:
|
|
- "11434:11434"
|
|
volumes:
|
|
- ollama_models:/root/.ollama
|
|
|
|
trino:
|
|
image: trinodb/trino:latest
|
|
ports:
|
|
- "8080:8080"
|
|
environment:
|
|
MINIO_ACCESS_KEY: minioadmin
|
|
MINIO_SECRET_KEY: minioadmin
|
|
volumes:
|
|
- ./infra/trino/catalog:/etc/trino/catalog
|
|
depends_on:
|
|
- minio
|
|
- hive-metastore
|
|
|
|
hive-metastore:
|
|
image: apache/hive:4.0.0
|
|
environment:
|
|
SERVICE_NAME: metastore
|
|
DB_DRIVER: derby
|
|
ports:
|
|
- "9083:9083"
|
|
volumes:
|
|
- hive_data:/opt/hive/data
|
|
- ./infra/hive/core-site.xml:/opt/hive/conf/core-site.xml:ro
|
|
- ./infra/hive/metastore-site.xml:/opt/hive/conf/metastore-site.xml:ro
|
|
depends_on:
|
|
- minio
|
|
|
|
superset:
|
|
image: apache/superset:latest
|
|
ports:
|
|
- "8088:8088"
|
|
environment:
|
|
SUPERSET_SECRET_KEY: stonks-dev-secret-key-change-me
|
|
ADMIN_USERNAME: admin
|
|
ADMIN_PASSWORD: admin
|
|
ADMIN_EMAIL: admin@stonks.local
|
|
volumes:
|
|
- superset_data:/app/superset_home
|
|
depends_on:
|
|
- trino
|
|
|
|
# ── Application Services ────────────────────────────────────────
|
|
|
|
scheduler:
|
|
build:
|
|
context: .
|
|
dockerfile: docker/Dockerfile.scheduler
|
|
environment:
|
|
<<: *app-env
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
redis:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pgrep -f 'python -m services.scheduler.app' || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 15s
|
|
restart: unless-stopped
|
|
|
|
symbol-registry:
|
|
build:
|
|
context: .
|
|
dockerfile: docker/Dockerfile
|
|
args:
|
|
SERVICE_CMD: "uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000"
|
|
environment:
|
|
<<: *app-env
|
|
ports:
|
|
- "8001:8000"
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 15s
|
|
restart: unless-stopped
|
|
|
|
ingestion:
|
|
build:
|
|
context: .
|
|
dockerfile: docker/Dockerfile
|
|
args:
|
|
SERVICE_CMD: "python -m services.ingestion.worker"
|
|
environment:
|
|
<<: *app-env
|
|
env_file:
|
|
- .env
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
redis:
|
|
condition: service_healthy
|
|
minio:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pgrep -f 'python -m services.ingestion.worker' || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 15s
|
|
restart: unless-stopped
|
|
|
|
parser:
|
|
build:
|
|
context: .
|
|
dockerfile: docker/Dockerfile
|
|
args:
|
|
SERVICE_CMD: "python -m services.parser.worker"
|
|
environment:
|
|
<<: *app-env
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
redis:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pgrep -f 'python -m services.parser.worker' || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 15s
|
|
restart: unless-stopped
|
|
|
|
extractor:
|
|
build:
|
|
context: .
|
|
dockerfile: docker/Dockerfile
|
|
args:
|
|
SERVICE_CMD: "python -m services.extractor.main"
|
|
environment:
|
|
<<: *app-env
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
redis:
|
|
condition: service_healthy
|
|
ollama:
|
|
condition: service_started
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pgrep -f 'python -m services.extractor.main' || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 15s
|
|
restart: unless-stopped
|
|
|
|
aggregation:
|
|
build:
|
|
context: .
|
|
dockerfile: docker/Dockerfile
|
|
args:
|
|
SERVICE_CMD: "python -m services.aggregation.main"
|
|
environment:
|
|
<<: *app-env
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
redis:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pgrep -f 'python -m services.aggregation.main' || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 15s
|
|
restart: unless-stopped
|
|
|
|
recommendation:
|
|
build:
|
|
context: .
|
|
dockerfile: docker/Dockerfile
|
|
args:
|
|
SERVICE_CMD: "python -m services.recommendation.main"
|
|
environment:
|
|
<<: *app-env
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
redis:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pgrep -f 'python -m services.recommendation.main' || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 15s
|
|
restart: unless-stopped
|
|
|
|
trading-engine:
|
|
build:
|
|
context: .
|
|
dockerfile: docker/Dockerfile
|
|
args:
|
|
SERVICE_CMD: "uvicorn services.trading.app:app --host 0.0.0.0 --port 8000"
|
|
environment:
|
|
<<: *app-env
|
|
env_file:
|
|
- .env
|
|
ports:
|
|
- "8002:8000"
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
redis:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 15s
|
|
restart: unless-stopped
|
|
|
|
risk-engine:
|
|
build:
|
|
context: .
|
|
dockerfile: docker/Dockerfile
|
|
args:
|
|
SERVICE_CMD: "uvicorn services.risk.app:app --host 0.0.0.0 --port 8000"
|
|
environment:
|
|
<<: *app-env
|
|
ports:
|
|
- "8003:8000"
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 15s
|
|
restart: unless-stopped
|
|
|
|
broker-adapter:
|
|
build:
|
|
context: .
|
|
dockerfile: docker/Dockerfile
|
|
args:
|
|
SERVICE_CMD: "python -m services.adapters.broker_service"
|
|
environment:
|
|
<<: *app-env
|
|
env_file:
|
|
- .env
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
redis:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pgrep -f 'python -m services.adapters.broker_service' || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 15s
|
|
restart: unless-stopped
|
|
|
|
lake-publisher:
|
|
build:
|
|
context: .
|
|
dockerfile: docker/Dockerfile
|
|
args:
|
|
SERVICE_CMD: "python -m services.lake_publisher.jobs"
|
|
environment:
|
|
<<: *app-env
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
minio:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pgrep -f 'python -m services.lake_publisher.jobs' || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 15s
|
|
restart: unless-stopped
|
|
|
|
query-api:
|
|
build:
|
|
context: .
|
|
dockerfile: docker/Dockerfile
|
|
args:
|
|
SERVICE_CMD: "uvicorn services.api.app:app --host 0.0.0.0 --port 8000"
|
|
environment:
|
|
<<: *app-env
|
|
ports:
|
|
- "8004:8000"
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
redis:
|
|
condition: service_healthy
|
|
minio:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 15s
|
|
restart: unless-stopped
|
|
|
|
dashboard:
|
|
build:
|
|
context: ./frontend
|
|
dockerfile: Dockerfile
|
|
ports:
|
|
- "3000:8080"
|
|
depends_on:
|
|
query-api:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl -f http://localhost:8080/ || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 10s
|
|
restart: unless-stopped
|
|
|
|
volumes:
|
|
pgdata:
|
|
miniodata:
|
|
ollama_models:
|
|
hive_data:
|
|
superset_data:
|