From 5be3ce2db95ebfa8b62030213d65cea817e0a64a Mon Sep 17 00:00:00 2001 From: Celes Renata Date: Sun, 19 Apr 2026 07:34:28 +0000 Subject: [PATCH] feat: migrate CI/CD from GHCR to local Harbor registry - Makefile: GHCR -> registry.celestium.life/stonks-oracle - GitHub Actions: login to Harbor, use HARBOR_PASSWORD secret - infra/k8s/*.yaml: all image refs -> registry.celestium.life - inttest pipeline: remove GHCR pull secret (local registry, no auth) - Steering docs: update registry/git endpoints --- .github/workflows/build.yml | 24 +++++++++++------------- .kiro/steering/kubernetes-conventions.md | 5 ++--- .kiro/steering/project-context.md | 8 +++++--- Makefile | 22 +++++++++++----------- infra/inttest/run_pipeline.sh | 11 ++++------- infra/k8s/aggregation-worker.yaml | 2 +- infra/k8s/broker-adapter.yaml | 2 +- infra/k8s/extractor-worker.yaml | 2 +- infra/k8s/ingestion-worker.yaml | 2 +- infra/k8s/lake-publisher.yaml | 2 +- infra/k8s/parser-worker.yaml | 2 +- infra/k8s/query-api.yaml | 2 +- infra/k8s/recommendation-worker.yaml | 2 +- infra/k8s/risk-engine.yaml | 2 +- infra/k8s/scheduler.yaml | 2 +- infra/k8s/symbol-registry.yaml | 2 +- 16 files changed, 44 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f11cfb7..b6590a3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,8 +7,8 @@ on: branches: [main] env: - REGISTRY: ghcr.io - IMAGE_BASE: ghcr.io/${{ github.repository_owner }}/stonks-oracle + REGISTRY: registry.celestium.life + IMAGE_BASE: registry.celestium.life/stonks-oracle jobs: lint-and-test: @@ -83,12 +83,12 @@ jobs: steps: - uses: actions/checkout@v5 - - name: Log in to GHCR + - name: Log in to Harbor uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: admin + password: ${{ secrets.HARBOR_PASSWORD }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -117,12 +117,12 @@ jobs: steps: - uses: actions/checkout@v5 - - name: Log in to GHCR + - name: Log in to Harbor uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: admin + password: ${{ secrets.HARBOR_PASSWORD }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -149,12 +149,12 @@ jobs: steps: - uses: actions/checkout@v5 - - name: Log in to GHCR + - name: Log in to Harbor uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: admin + password: ${{ secrets.HARBOR_PASSWORD }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -217,8 +217,6 @@ jobs: kubectl cluster-info || echo "WARNING: kubectl cannot reach cluster API" - name: Run integration tests - env: - GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | bash infra/inttest/run_pipeline.sh \ --image-tag ${{ github.sha }} \ diff --git a/.kiro/steering/kubernetes-conventions.md b/.kiro/steering/kubernetes-conventions.md index e00cecc..0741a99 100644 --- a/.kiro/steering/kubernetes-conventions.md +++ b/.kiro/steering/kubernetes-conventions.md @@ -13,7 +13,7 @@ The namespace is NOT managed by Helm — it's created by `runmefirst.sh` with He - Services defined in `values.yaml` under `services:` — the deployments template iterates over them - Adding a new service: add entry to `values.yaml`, add network policy if it needs ingress, add ingress if it needs external access - Dashboard uses nginx-unprivileged on port 8080 (not 80) -- Superset uses custom image `ghcr.io/celesrenata/stonks-oracle/superset:latest` with trino + psycopg2 drivers +- Superset uses custom image `registry.celestium.life/stonks-oracle/superset:latest` with trino + psycopg2 drivers ## TLS - Internal services: use `ca-issuer` ClusterIssuer (local CA) @@ -44,9 +44,8 @@ The namespace is NOT managed by Helm — it's created by `runmefirst.sh` with He - Ollama: `ollama.ollama-service.svc.cluster.local:11434` ## Images -- All images from `ghcr.io/celesrenata/stonks-oracle/:latest` +- All images from `registry.celestium.life/stonks-oracle/:latest` - Use `imagePullPolicy: Always` -- Use `imagePullSecrets` referencing `ghcr-credentials` ## Labels - `app.kubernetes.io/part-of: stonks-oracle` diff --git a/.kiro/steering/project-context.md b/.kiro/steering/project-context.md index e0a2635..ffd814f 100644 --- a/.kiro/steering/project-context.md +++ b/.kiro/steering/project-context.md @@ -29,18 +29,20 @@ Three-layer signal aggregation engine: - Trading Engine: `https://stonks-trading.celestium.life` - Superset: `https://stonks-dash.celestium.life` - Trino: `https://stonks-trino.celestium.life` +- Gitea: `https://git.celestium.life` +- Harbor Registry: `https://registry.celestium.life` ## Infrastructure - Kubernetes cluster: 4x NixOS nodes (gremlin-1 through gremlin-4), reachable via `kubectl`, `virtctl`, `ssh root@gremlin-{1,2,3,4}` - NixOS configs stored at `/etc/nixos` on gremlin-1, git-pushed to other hosts - Ingress: Traefik, domain `*.celestium.life` - Cert-Manager: `ca-issuer` (local CA) for internal services -- Container registry: `ghcr.io/celesrenata/stonks-oracle` +- Container registry: `registry.celestium.life/stonks-oracle` ## CI/CD - GitHub Actions workflow at `.github/workflows/build.yml` -- Push to `main` triggers: lint → pytest → frontend vitest → build all service images + dashboard + superset → push to GHCR -- Images tagged as `ghcr.io/celesrenata/stonks-oracle/:` and `:latest` +- Push to `main` triggers: lint → pytest → frontend vitest → build all service images + dashboard + superset → push to Harbor +- Images tagged as `registry.celestium.life/stonks-oracle/:` and `:latest` - Dashboard image: `frontend/Dockerfile` (multi-stage: node:24 → nginx-unprivileged on port 8080) - Superset image: `docker/Dockerfile.superset` (apache/superset + trino + psycopg2) - Python service images: `docker/Dockerfile` with `SERVICE_CMD` build arg diff --git a/Makefile b/Makefile index de95de6..1cf0f41 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ REPO_OWNER := celesrenata REPO_NAME := stonks-oracle -GHCR := ghcr.io/$(REPO_OWNER)/$(REPO_NAME) +REGISTRY := registry.celestium.life/stonks-oracle SHA := $(shell git rev-parse --short HEAD 2>/dev/null || echo "dev") SERVICES := scheduler symbol-registry ingestion parser extractor aggregation recommendation risk broker-adapter lake-publisher query-api @@ -12,7 +12,7 @@ help: @echo " lint - Run ruff linter" @echo " test - Run pytest" @echo " build - Build all service images locally" - @echo " push - Push all images to GHCR" + @echo " push - Push all images to registry" @echo " deploy - Apply K8s manifests" @echo " clean - Remove local images" @@ -40,26 +40,26 @@ build: echo "Building $$svc ($$cmd)..."; \ docker build \ --build-arg "SERVICE_CMD=$$cmd" \ - -t $(GHCR)/$$svc:$(SHA) \ - -t $(GHCR)/$$svc:latest \ + -t $(REGISTRY)/$$svc:$(SHA) \ + -t $(REGISTRY)/$$svc:latest \ -f docker/Dockerfile . || exit 1; \ done @echo "Building dashboard..." docker build \ - -t $(GHCR)/dashboard:$(SHA) \ - -t $(GHCR)/dashboard:latest \ + -t $(REGISTRY)/dashboard:$(SHA) \ + -t $(REGISTRY)/dashboard:latest \ -f frontend/Dockerfile frontend/ || exit 1 @echo "Building superset..." docker build \ - -t $(GHCR)/superset:$(SHA) \ - -t $(GHCR)/superset:latest \ + -t $(REGISTRY)/superset:$(SHA) \ + -t $(REGISTRY)/superset:latest \ -f docker/Dockerfile.superset docker/ || exit 1 push: @for svc in $(SERVICES); do \ echo "Pushing $$svc..."; \ - docker push $(GHCR)/$$svc:$(SHA); \ - docker push $(GHCR)/$$svc:latest; \ + docker push $(REGISTRY)/$$svc:$(SHA); \ + docker push $(REGISTRY)/$$svc:latest; \ done deploy: @@ -70,5 +70,5 @@ deploy: clean: @for svc in $(SERVICES); do \ - docker rmi $(GHCR)/$$svc:$(SHA) $(GHCR)/$$svc:latest 2>/dev/null || true; \ + docker rmi $(REGISTRY)/$$svc:$(SHA) $(REGISTRY)/$$svc:latest 2>/dev/null || true; \ done diff --git a/infra/inttest/run_pipeline.sh b/infra/inttest/run_pipeline.sh index a595db1..518bd49 100755 --- a/infra/inttest/run_pipeline.sh +++ b/infra/inttest/run_pipeline.sh @@ -236,15 +236,12 @@ if ! kubectl create namespace "$NAMESPACE"; then fi # ── Create GHCR image pull secret (if token available) ─────────────────────── +# NOTE: Images now served from Harbor at registry.celestium.life (no auth needed for pulls) +# This block is kept for backward compatibility but is no longer required if [ -n "${GHCR_TOKEN:-}" ]; then - log "Creating ghcr-credentials secret ..." - kubectl create secret docker-registry ghcr-credentials \ - --docker-server=ghcr.io \ - --docker-username=celesrenata \ - --docker-password="$GHCR_TOKEN" \ - -n "$NAMESPACE" || true + log "GHCR_TOKEN set but images are on local Harbor — skipping GHCR secret" else - log "GHCR_TOKEN not set — skipping image pull secret (images must be pullable without auth)" + log "Images served from registry.celestium.life (no pull secret needed)" fi # ── Create Docker Hub pull secret (avoid rate limits) ──────────────────────── diff --git a/infra/k8s/aggregation-worker.yaml b/infra/k8s/aggregation-worker.yaml index 06a4d58..39068e7 100644 --- a/infra/k8s/aggregation-worker.yaml +++ b/infra/k8s/aggregation-worker.yaml @@ -28,7 +28,7 @@ spec: type: RuntimeDefault containers: - name: aggregation-worker - image: ghcr.io/celesrenata/stonks-oracle/aggregation:latest + image: registry.celestium.life/stonks-oracle/aggregation:latest imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false diff --git a/infra/k8s/broker-adapter.yaml b/infra/k8s/broker-adapter.yaml index 0c48eb1..9f02bb4 100644 --- a/infra/k8s/broker-adapter.yaml +++ b/infra/k8s/broker-adapter.yaml @@ -28,7 +28,7 @@ spec: type: RuntimeDefault containers: - name: broker-adapter - image: ghcr.io/celesrenata/stonks-oracle/broker-adapter:latest + image: registry.celestium.life/stonks-oracle/broker-adapter:latest imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false diff --git a/infra/k8s/extractor-worker.yaml b/infra/k8s/extractor-worker.yaml index 09d76f6..a40646e 100644 --- a/infra/k8s/extractor-worker.yaml +++ b/infra/k8s/extractor-worker.yaml @@ -28,7 +28,7 @@ spec: type: RuntimeDefault containers: - name: extractor-worker - image: ghcr.io/celesrenata/stonks-oracle/extractor:latest + image: registry.celestium.life/stonks-oracle/extractor:latest imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false diff --git a/infra/k8s/ingestion-worker.yaml b/infra/k8s/ingestion-worker.yaml index 555b273..8573c0c 100644 --- a/infra/k8s/ingestion-worker.yaml +++ b/infra/k8s/ingestion-worker.yaml @@ -28,7 +28,7 @@ spec: type: RuntimeDefault containers: - name: ingestion-worker - image: ghcr.io/celesrenata/stonks-oracle/ingestion:latest + image: registry.celestium.life/stonks-oracle/ingestion:latest imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false diff --git a/infra/k8s/lake-publisher.yaml b/infra/k8s/lake-publisher.yaml index 78189b1..64002b2 100644 --- a/infra/k8s/lake-publisher.yaml +++ b/infra/k8s/lake-publisher.yaml @@ -28,7 +28,7 @@ spec: type: RuntimeDefault containers: - name: lake-publisher - image: ghcr.io/celesrenata/stonks-oracle/lake-publisher:latest + image: registry.celestium.life/stonks-oracle/lake-publisher:latest imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false diff --git a/infra/k8s/parser-worker.yaml b/infra/k8s/parser-worker.yaml index 3818429..aeefb91 100644 --- a/infra/k8s/parser-worker.yaml +++ b/infra/k8s/parser-worker.yaml @@ -28,7 +28,7 @@ spec: type: RuntimeDefault containers: - name: parser-worker - image: ghcr.io/celesrenata/stonks-oracle/parser:latest + image: registry.celestium.life/stonks-oracle/parser:latest imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false diff --git a/infra/k8s/query-api.yaml b/infra/k8s/query-api.yaml index aea7adf..9d3fa4f 100644 --- a/infra/k8s/query-api.yaml +++ b/infra/k8s/query-api.yaml @@ -28,7 +28,7 @@ spec: type: RuntimeDefault containers: - name: query-api - image: ghcr.io/celesrenata/stonks-oracle/query-api:latest + image: registry.celestium.life/stonks-oracle/query-api:latest imagePullPolicy: Always ports: - containerPort: 8000 diff --git a/infra/k8s/recommendation-worker.yaml b/infra/k8s/recommendation-worker.yaml index 15d16d7..ff870fa 100644 --- a/infra/k8s/recommendation-worker.yaml +++ b/infra/k8s/recommendation-worker.yaml @@ -28,7 +28,7 @@ spec: type: RuntimeDefault containers: - name: recommendation-worker - image: ghcr.io/celesrenata/stonks-oracle/recommendation:latest + image: registry.celestium.life/stonks-oracle/recommendation:latest imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false diff --git a/infra/k8s/risk-engine.yaml b/infra/k8s/risk-engine.yaml index 1a979fc..45049ed 100644 --- a/infra/k8s/risk-engine.yaml +++ b/infra/k8s/risk-engine.yaml @@ -28,7 +28,7 @@ spec: type: RuntimeDefault containers: - name: risk-engine - image: ghcr.io/celesrenata/stonks-oracle/risk:latest + image: registry.celestium.life/stonks-oracle/risk:latest imagePullPolicy: Always ports: - containerPort: 8000 diff --git a/infra/k8s/scheduler.yaml b/infra/k8s/scheduler.yaml index b8a2d72..55bb25a 100644 --- a/infra/k8s/scheduler.yaml +++ b/infra/k8s/scheduler.yaml @@ -28,7 +28,7 @@ spec: type: RuntimeDefault containers: - name: scheduler - image: ghcr.io/celesrenata/stonks-oracle/scheduler:latest + image: registry.celestium.life/stonks-oracle/scheduler:latest imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false diff --git a/infra/k8s/symbol-registry.yaml b/infra/k8s/symbol-registry.yaml index aeed4a2..ca6988f 100644 --- a/infra/k8s/symbol-registry.yaml +++ b/infra/k8s/symbol-registry.yaml @@ -28,7 +28,7 @@ spec: type: RuntimeDefault containers: - name: symbol-registry-api - image: ghcr.io/celesrenata/stonks-oracle/symbol-registry:latest + image: registry.celestium.life/stonks-oracle/symbol-registry:latest imagePullPolicy: Always ports: - containerPort: 8000