diff --git a/.kiro/steering/development-process.md b/.kiro/steering/development-process.md index fd4c0e7..2146d95 100644 --- a/.kiro/steering/development-process.md +++ b/.kiro/steering/development-process.md @@ -1,28 +1,41 @@ # Development Process — Test-Develop-Debug +## Local Environment +- Python 3.12 via NixOS, virtualenv at `.venv/` +- Always use `.venv/bin/python` or activate with `source .venv/bin/activate` before running Python commands +- When running `pytest`, `ruff`, or any Python tool, use the `.venv` — e.g. `python -m pytest` (not bare `pytest` which may resolve to system Python) +- Node.js 24 available for frontend work; `frontend/` has its own `node_modules/` + ## Workflow 1. Write or update tests for the target behavior 2. Implement the minimal code to pass 3. Debug failures, fix, re-run 4. Commit and push after each phase completes -5. GitHub Actions builds container images and pushes to GHCR -6. Deploy to cluster via `kubectl apply` +5. GitHub Actions CI automatically builds container images and pushes to GHCR +6. Deploy to cluster via Helm or `kubectl apply` ## Testing - Use `pytest` with `pytest-asyncio` for async code -- Tests live alongside service code or in a top-level `tests/` directory -- Run tests with `pytest --tb=short -q` or `pytest -x` for fail-fast +- Tests live in the top-level `tests/` directory +- Run tests with `python -m pytest tests/ -x --tb=short -q` - Focus on core logic, not mocking infrastructure -## Build and Deploy -- Always build and test Docker images locally before pushing to GitHub -- Only push to GitHub after local build succeeds — don't waste CI credits on broken builds -- Dockerfile at `docker/Dockerfile` -- GitHub workflow at `.github/workflows/build.yml` -- Images tagged as `ghcr.io/celesrenata/stonks-oracle/:` and `:latest` -- K8s manifests reference GHCR images -- Deploy: `kubectl apply -f infra/k8s/` -- Local build: `make build` → verify → `git push` → CI builds and pushes to GHCR +## CI/CD — GitHub Actions +- Workflow file: `.github/workflows/build.yml` +- Triggers on push to `main` and PRs +- Jobs: + - `lint-and-test`: runs ruff lint + pytest on ubuntu with Python 3.12 + - `build-services`: matrix build of all Python services via `docker/Dockerfile`, pushes to GHCR with `:` and `:latest` tags + - `build-dashboard`: builds `frontend/Dockerfile` separately, pushes `dashboard` image to GHCR +- CI handles image building and pushing — do NOT manually `docker push` unless CI is broken or you need to bypass it +- After pushing to `main`, wait for CI to complete before deploying (check GitHub Actions status) +- If you need to build locally for testing: `make build` or `docker build` directly, but let CI do the GHCR push + +## Deploy +- Helm chart at `infra/helm/stonks-oracle/` +- Deploy: `helm upgrade --install stonks-oracle infra/helm/stonks-oracle -n stonks-oracle` +- Alternative raw manifests: `kubectl apply -f infra/k8s/` +- To restart a deployment after CI pushes new images: `kubectl rollout restart deployment/ -n stonks-oracle` ## Git Conventions - Commit after each completed phase task @@ -35,10 +48,10 @@ - FastAPI for HTTP services - asyncio + asyncpg/aioredis for async I/O - Minimal dependencies, prefer stdlib where possible +- Frontend: React 19, TypeScript strict mode, Tailwind CSS, TanStack Router/Query ## Documentation - Do NOT create large summary/success markdown files after each step - Keep notes short, concise, and organized under `docs/notes/` - Name note files to match the task they relate to (e.g. `docs/notes/phase0-k8s-manifests.md`) -- This makes them recallable by task without guessing - If a note isn't useful for future reference, don't write it diff --git a/.kiro/steering/project-context.md b/.kiro/steering/project-context.md index f1a3bdf..031bdf0 100644 --- a/.kiro/steering/project-context.md +++ b/.kiro/steering/project-context.md @@ -2,7 +2,13 @@ ## Overview Stonks Oracle is a Kubernetes-native AI market intelligence and paper-trading platform. -Python monorepo with services under `services/`, infrastructure under `infra/`, lakehouse schemas under `lakehouse/`, and dashboards under `dashboards/`. +Python monorepo with services under `services/`, infrastructure under `infra/`, lakehouse schemas under `lakehouse/`, frontend React dashboard under `frontend/`, and dashboards under `dashboards/`. + +## Local Dev Environment +- NixOS dev environment, Python 3.12 +- Virtual environment at `.venv/` — always use it for Python commands +- Node.js 24 for frontend (`frontend/` directory) +- Docker available locally for image builds ## Infrastructure - Kubernetes cluster: 4x NixOS nodes (gremlin-1 through gremlin-4), reachable via `kubectl`, `virtctl`, `ssh root@gremlin-{1,2,3,4}` @@ -10,7 +16,14 @@ Python monorepo with services under `services/`, infrastructure under `infra/`, - Ingress: Traefik, domain `*.celestium.life` - Cert-Manager: `ca-issuer` (local CA) for internal services, `celestium-le-production` (Let's Encrypt) for public-facing - Container registry: `ghcr.io/celesrenata/stonks-oracle` -- CI: GitHub Actions builds containers, cluster pulls from GHCR + +## CI/CD +- GitHub Actions workflow at `.github/workflows/build.yml` +- Push to `main` triggers: lint → test → build all service images + dashboard image → push to GHCR +- Images tagged as `ghcr.io/celesrenata/stonks-oracle/:` and `:latest` +- Dashboard image built from `frontend/Dockerfile` (multi-stage: node → nginx) +- Python service images built from `docker/Dockerfile` with `SERVICE_CMD` build arg +- Let CI handle image builds and pushes — only build locally for testing or when CI is unavailable ## Existing Cluster Services (do NOT redeploy these) - PostgreSQL: `postgresql-rw.postgresql-service.svc.cluster.local:5432` @@ -18,16 +31,10 @@ Python monorepo with services under `services/`, infrastructure under `infra/`, - MinIO: `minio.minio-service.svc.cluster.local:80` (API), console at `minio-crawler-console.minio-service.svc.cluster.local:9090` - Ollama: `ollama.ollama-service.svc.cluster.local:11434` (cluster-internal), also at `http://10.1.1.12:2701` (external), GPU: 4070 Ti Super 16GB -## Development Process -- Test-Develop-Debug (TDD) cycle -- After each phase: commit, push, build via GitHub Actions, deploy to cluster -- Local builds for dev iteration, GitHub workflows for CI/CD -- Python 3.12, NixOS dev environment - ## Key Conventions - All services use `services/shared/config.py` for configuration via env vars - Redis queues defined in `services/shared/redis_keys.py` - Pydantic schemas in `services/shared/schemas.py` -- K8s manifests in `infra/k8s/`, all in `stonks-oracle` namespace +- K8s manifests in `infra/k8s/`, Helm chart in `infra/helm/stonks-oracle/`, all in `stonks-oracle` namespace - Lakehouse DDL in `lakehouse/schemas/` - Crawler patterns inspired by Noctipede (`~/sources/splinterstice/noctipede`): BeautifulSoup + requests with retry adapters, content hashing, boilerplate stripping, quality scoring diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 1a741cb..e026b66 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,5 +1,5 @@ # Stage 1: Build -FROM node:20-alpine AS build +FROM node:24-alpine AS build WORKDIR /app COPY package.json package-lock.json ./ RUN npm ci diff --git a/frontend/src/pages/OpsCoverage.tsx b/frontend/src/pages/OpsCoverage.tsx index 17beda5..c14c2fe 100644 --- a/frontend/src/pages/OpsCoverage.tsx +++ b/frontend/src/pages/OpsCoverage.tsx @@ -35,8 +35,8 @@ export function OpsCoveragePage() { {matrix.map((row, i) => ( - {row.ticker as string} - {row.legal_name as string} + {String(row.ticker)} + {String(row.legal_name)} @@ -61,7 +61,7 @@ export function OpsCoveragePage() { const missingTypes = expected.filter((t) => !activeTypes.includes(t)); return (
- {m.ticker as string} + {String(m.ticker)} missing: {missingTypes.map((t) => ( @@ -81,12 +81,12 @@ export function OpsCoveragePage() { {stale.map((s, i) => (
- {s.ticker as string} - - {s.source_name as string} + {String(s.ticker)} + + {String(s.source_name)}
- Last success: {s.last_success ? new Date(s.last_success as string).toLocaleString() : 'never'} + Last success: {s.last_success ? new Date(String(s.last_success)).toLocaleString() : 'never'} {s.recent_failures ? ` | ${s.recent_failures} failures (24h)` : ''}
diff --git a/frontend/src/pages/OpsIngestion.tsx b/frontend/src/pages/OpsIngestion.tsx index 2014d1a..b32e6fc 100644 --- a/frontend/src/pages/OpsIngestion.tsx +++ b/frontend/src/pages/OpsIngestion.tsx @@ -45,11 +45,11 @@ export function OpsIngestionPage() { {/* Summary stats */}
- - - - - + + + + +
{/* Throughput chart */} @@ -68,7 +68,7 @@ export function OpsIngestionPage() { {/* By source type */} - {s.by_source_type && ( + {s.by_source_type ? (

By Source Type

@@ -85,7 +85,7 @@ export function OpsIngestionPage() { {(s.by_source_type as Array>).map((row, i) => ( - {row.source_type as string} + {String(row.source_type)} {String(row.runs)} {String(row.completed)} {String(row.failed)} @@ -96,15 +96,15 @@ export function OpsIngestionPage() {
- )} + ) : null}
); } -function StatCard({ label, value, color = 'text-gray-100' }: { label: string; value: unknown; color?: string }) { +function StatCard({ label, value, color = 'text-gray-100' }: { label: string; value: string; color?: string }) { return ( -
{value != null ? String(value) : '—'}
+
{value}
{label}
 ); diff --git a/frontend/src/pages/OpsModel.tsx b/frontend/src/pages/OpsModel.tsx index e00973f..b16ed56 100644 --- a/frontend/src/pages/OpsModel.tsx +++ b/frontend/src/pages/OpsModel.tsx @@ -20,7 +20,7 @@ export function OpsModelPage() { {/* Key metrics */}
- + @@ -40,20 +40,20 @@ export function OpsModelPage() {
- {f.ticker as string} + {String(f.ticker)} - {f.model_name as string} + {String(f.model_name)}
- {f.recorded_at ? new Date(f.recorded_at as string).toLocaleString() : ''} + {f.recorded_at ? new Date(String(f.recorded_at)).toLocaleString() : ''}
- {f.document_title as string} ({f.document_type as string}) + {f.document_title ? String(f.document_title) : ''} ({String(f.document_type)})
- {f.validation_errors && ( + {f.validation_errors ? (
- {JSON.stringify(f.validation_errors)} + {JSON.stringify(f.validation_errors) ?? ''}
- )} + ) : null}
))}
@@ -63,10 +63,10 @@ export function OpsModelPage() { ); } -function StatCard({ label, value, color = 'text-gray-100' }: { label: string; value: unknown; color?: string }) { +function StatCard({ label, value, color = 'text-gray-100' }: { label: string; value: string; color?: string }) { return ( -
{value != null ? String(value) : '—'}
+
{value}
{label}
 ); diff --git a/frontend/src/pages/OrderDetail.tsx b/frontend/src/pages/OrderDetail.tsx index 3e22824..4dbb7b5 100644 --- a/frontend/src/pages/OrderDetail.tsx +++ b/frontend/src/pages/OrderDetail.tsx @@ -51,9 +51,9 @@ export function OrderDetailPage() {
{new Date(ev.created_at).toLocaleString()}
- {ev.data && ( - 
{JSON.stringify(ev.data, null, 2)}
- )} + {ev.data ? ( + 
{JSON.stringify(ev.data, null, 2) ?? ''}
+ ) : null}
))} @@ -68,9 +68,9 @@ export function OrderDetailPage() {
{(order.audit_trail as Array>).map((entry, i) => (
- {entry.created_at ? new Date(entry.created_at as string).toLocaleString() : ''} - {entry.event_type as string} - {entry.description as string} + {entry.created_at ? new Date(String(entry.created_at)).toLocaleString() : ''} + {String(entry.event_type)} + {String(entry.description)}
))}
diff --git a/frontend/src/pages/TrendDetail.tsx b/frontend/src/pages/TrendDetail.tsx index 71fff76..a286982 100644 --- a/frontend/src/pages/TrendDetail.tsx +++ b/frontend/src/pages/TrendDetail.tsx @@ -79,23 +79,23 @@ export function TrendDetailPage() {
- - {(ev.title as string) ?? 'Untitled'} + + {String(ev.title ?? 'Untitled')}
rank: {((ev.rank_score as number) ?? 0).toFixed(3)}
- {ev.document_type as string} - {ev.source_type as string} - {ev.publisher && {ev.publisher as string}} - {ev.published_at && {new Date(ev.published_at as string).toLocaleDateString()}} + {String(ev.document_type)} + {String(ev.source_type)} + {ev.publisher ? {String(ev.publisher)} : null} + {ev.published_at ? {new Date(String(ev.published_at)).toLocaleDateString()} : null}
- {ev.intelligence && ( + {ev.intelligence ? (
Summary: - {((ev.intelligence as Record).summary as string) ?? '—'} + {String((ev.intelligence as Record).summary ?? '—')}
- )} + ) : null}
))}