feat: migrate CI/CD from GHCR to local Harbor registry
- Makefile: GHCR -> registry.celestium.life/stonks-oracle - GitHub Actions: login to Harbor, use HARBOR_PASSWORD secret - infra/k8s/*.yaml: all image refs -> registry.celestium.life - inttest pipeline: remove GHCR pull secret (local registry, no auth) - Steering docs: update registry/git endpoints
This commit is contained in:
+11
-13
@@ -7,8 +7,8 @@ on:
|
|||||||
branches: [main]
|
branches: [main]
|
||||||
|
|
||||||
env:
|
env:
|
||||||
REGISTRY: ghcr.io
|
REGISTRY: registry.celestium.life
|
||||||
IMAGE_BASE: ghcr.io/${{ github.repository_owner }}/stonks-oracle
|
IMAGE_BASE: registry.celestium.life/stonks-oracle
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
lint-and-test:
|
lint-and-test:
|
||||||
@@ -83,12 +83,12 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@v5
|
||||||
|
|
||||||
- name: Log in to GHCR
|
- name: Log in to Harbor
|
||||||
uses: docker/login-action@v4
|
uses: docker/login-action@v4
|
||||||
with:
|
with:
|
||||||
registry: ${{ env.REGISTRY }}
|
registry: ${{ env.REGISTRY }}
|
||||||
username: ${{ github.actor }}
|
username: admin
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.HARBOR_PASSWORD }}
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v4
|
uses: docker/setup-buildx-action@v4
|
||||||
@@ -117,12 +117,12 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@v5
|
||||||
|
|
||||||
- name: Log in to GHCR
|
- name: Log in to Harbor
|
||||||
uses: docker/login-action@v4
|
uses: docker/login-action@v4
|
||||||
with:
|
with:
|
||||||
registry: ${{ env.REGISTRY }}
|
registry: ${{ env.REGISTRY }}
|
||||||
username: ${{ github.actor }}
|
username: admin
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.HARBOR_PASSWORD }}
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v4
|
uses: docker/setup-buildx-action@v4
|
||||||
@@ -149,12 +149,12 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@v5
|
||||||
|
|
||||||
- name: Log in to GHCR
|
- name: Log in to Harbor
|
||||||
uses: docker/login-action@v4
|
uses: docker/login-action@v4
|
||||||
with:
|
with:
|
||||||
registry: ${{ env.REGISTRY }}
|
registry: ${{ env.REGISTRY }}
|
||||||
username: ${{ github.actor }}
|
username: admin
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.HARBOR_PASSWORD }}
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v4
|
uses: docker/setup-buildx-action@v4
|
||||||
@@ -217,8 +217,6 @@ jobs:
|
|||||||
kubectl cluster-info || echo "WARNING: kubectl cannot reach cluster API"
|
kubectl cluster-info || echo "WARNING: kubectl cannot reach cluster API"
|
||||||
|
|
||||||
- name: Run integration tests
|
- name: Run integration tests
|
||||||
env:
|
|
||||||
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
run: |
|
run: |
|
||||||
bash infra/inttest/run_pipeline.sh \
|
bash infra/inttest/run_pipeline.sh \
|
||||||
--image-tag ${{ github.sha }} \
|
--image-tag ${{ github.sha }} \
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ The namespace is NOT managed by Helm — it's created by `runmefirst.sh` with He
|
|||||||
- Services defined in `values.yaml` under `services:` — the deployments template iterates over them
|
- Services defined in `values.yaml` under `services:` — the deployments template iterates over them
|
||||||
- Adding a new service: add entry to `values.yaml`, add network policy if it needs ingress, add ingress if it needs external access
|
- Adding a new service: add entry to `values.yaml`, add network policy if it needs ingress, add ingress if it needs external access
|
||||||
- Dashboard uses nginx-unprivileged on port 8080 (not 80)
|
- Dashboard uses nginx-unprivileged on port 8080 (not 80)
|
||||||
- Superset uses custom image `ghcr.io/celesrenata/stonks-oracle/superset:latest` with trino + psycopg2 drivers
|
- Superset uses custom image `registry.celestium.life/stonks-oracle/superset:latest` with trino + psycopg2 drivers
|
||||||
|
|
||||||
## TLS
|
## TLS
|
||||||
- Internal services: use `ca-issuer` ClusterIssuer (local CA)
|
- Internal services: use `ca-issuer` ClusterIssuer (local CA)
|
||||||
@@ -44,9 +44,8 @@ The namespace is NOT managed by Helm — it's created by `runmefirst.sh` with He
|
|||||||
- Ollama: `ollama.ollama-service.svc.cluster.local:11434`
|
- Ollama: `ollama.ollama-service.svc.cluster.local:11434`
|
||||||
|
|
||||||
## Images
|
## Images
|
||||||
- All images from `ghcr.io/celesrenata/stonks-oracle/<service>:latest`
|
- All images from `registry.celestium.life/stonks-oracle/<service>:latest`
|
||||||
- Use `imagePullPolicy: Always`
|
- Use `imagePullPolicy: Always`
|
||||||
- Use `imagePullSecrets` referencing `ghcr-credentials`
|
|
||||||
|
|
||||||
## Labels
|
## Labels
|
||||||
- `app.kubernetes.io/part-of: stonks-oracle`
|
- `app.kubernetes.io/part-of: stonks-oracle`
|
||||||
|
|||||||
@@ -29,18 +29,20 @@ Three-layer signal aggregation engine:
|
|||||||
- Trading Engine: `https://stonks-trading.celestium.life`
|
- Trading Engine: `https://stonks-trading.celestium.life`
|
||||||
- Superset: `https://stonks-dash.celestium.life`
|
- Superset: `https://stonks-dash.celestium.life`
|
||||||
- Trino: `https://stonks-trino.celestium.life`
|
- Trino: `https://stonks-trino.celestium.life`
|
||||||
|
- Gitea: `https://git.celestium.life`
|
||||||
|
- Harbor Registry: `https://registry.celestium.life`
|
||||||
|
|
||||||
## Infrastructure
|
## Infrastructure
|
||||||
- Kubernetes cluster: 4x NixOS nodes (gremlin-1 through gremlin-4), reachable via `kubectl`, `virtctl`, `ssh root@gremlin-{1,2,3,4}`
|
- Kubernetes cluster: 4x NixOS nodes (gremlin-1 through gremlin-4), reachable via `kubectl`, `virtctl`, `ssh root@gremlin-{1,2,3,4}`
|
||||||
- NixOS configs stored at `/etc/nixos` on gremlin-1, git-pushed to other hosts
|
- NixOS configs stored at `/etc/nixos` on gremlin-1, git-pushed to other hosts
|
||||||
- Ingress: Traefik, domain `*.celestium.life`
|
- Ingress: Traefik, domain `*.celestium.life`
|
||||||
- Cert-Manager: `ca-issuer` (local CA) for internal services
|
- Cert-Manager: `ca-issuer` (local CA) for internal services
|
||||||
- Container registry: `ghcr.io/celesrenata/stonks-oracle`
|
- Container registry: `registry.celestium.life/stonks-oracle`
|
||||||
|
|
||||||
## CI/CD
|
## CI/CD
|
||||||
- GitHub Actions workflow at `.github/workflows/build.yml`
|
- GitHub Actions workflow at `.github/workflows/build.yml`
|
||||||
- Push to `main` triggers: lint → pytest → frontend vitest → build all service images + dashboard + superset → push to GHCR
|
- Push to `main` triggers: lint → pytest → frontend vitest → build all service images + dashboard + superset → push to Harbor
|
||||||
- Images tagged as `ghcr.io/celesrenata/stonks-oracle/<service>:<sha>` and `:latest`
|
- Images tagged as `registry.celestium.life/stonks-oracle/<service>:<sha>` and `:latest`
|
||||||
- Dashboard image: `frontend/Dockerfile` (multi-stage: node:24 → nginx-unprivileged on port 8080)
|
- Dashboard image: `frontend/Dockerfile` (multi-stage: node:24 → nginx-unprivileged on port 8080)
|
||||||
- Superset image: `docker/Dockerfile.superset` (apache/superset + trino + psycopg2)
|
- Superset image: `docker/Dockerfile.superset` (apache/superset + trino + psycopg2)
|
||||||
- Python service images: `docker/Dockerfile` with `SERVICE_CMD` build arg
|
- Python service images: `docker/Dockerfile` with `SERVICE_CMD` build arg
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
REPO_OWNER := celesrenata
|
REPO_OWNER := celesrenata
|
||||||
REPO_NAME := stonks-oracle
|
REPO_NAME := stonks-oracle
|
||||||
GHCR := ghcr.io/$(REPO_OWNER)/$(REPO_NAME)
|
REGISTRY := registry.celestium.life/stonks-oracle
|
||||||
SHA := $(shell git rev-parse --short HEAD 2>/dev/null || echo "dev")
|
SHA := $(shell git rev-parse --short HEAD 2>/dev/null || echo "dev")
|
||||||
|
|
||||||
SERVICES := scheduler symbol-registry ingestion parser extractor aggregation recommendation risk broker-adapter lake-publisher query-api
|
SERVICES := scheduler symbol-registry ingestion parser extractor aggregation recommendation risk broker-adapter lake-publisher query-api
|
||||||
@@ -12,7 +12,7 @@ help:
|
|||||||
@echo " lint - Run ruff linter"
|
@echo " lint - Run ruff linter"
|
||||||
@echo " test - Run pytest"
|
@echo " test - Run pytest"
|
||||||
@echo " build - Build all service images locally"
|
@echo " build - Build all service images locally"
|
||||||
@echo " push - Push all images to GHCR"
|
@echo " push - Push all images to registry"
|
||||||
@echo " deploy - Apply K8s manifests"
|
@echo " deploy - Apply K8s manifests"
|
||||||
@echo " clean - Remove local images"
|
@echo " clean - Remove local images"
|
||||||
|
|
||||||
@@ -40,26 +40,26 @@ build:
|
|||||||
echo "Building $$svc ($$cmd)..."; \
|
echo "Building $$svc ($$cmd)..."; \
|
||||||
docker build \
|
docker build \
|
||||||
--build-arg "SERVICE_CMD=$$cmd" \
|
--build-arg "SERVICE_CMD=$$cmd" \
|
||||||
-t $(GHCR)/$$svc:$(SHA) \
|
-t $(REGISTRY)/$$svc:$(SHA) \
|
||||||
-t $(GHCR)/$$svc:latest \
|
-t $(REGISTRY)/$$svc:latest \
|
||||||
-f docker/Dockerfile . || exit 1; \
|
-f docker/Dockerfile . || exit 1; \
|
||||||
done
|
done
|
||||||
@echo "Building dashboard..."
|
@echo "Building dashboard..."
|
||||||
docker build \
|
docker build \
|
||||||
-t $(GHCR)/dashboard:$(SHA) \
|
-t $(REGISTRY)/dashboard:$(SHA) \
|
||||||
-t $(GHCR)/dashboard:latest \
|
-t $(REGISTRY)/dashboard:latest \
|
||||||
-f frontend/Dockerfile frontend/ || exit 1
|
-f frontend/Dockerfile frontend/ || exit 1
|
||||||
@echo "Building superset..."
|
@echo "Building superset..."
|
||||||
docker build \
|
docker build \
|
||||||
-t $(GHCR)/superset:$(SHA) \
|
-t $(REGISTRY)/superset:$(SHA) \
|
||||||
-t $(GHCR)/superset:latest \
|
-t $(REGISTRY)/superset:latest \
|
||||||
-f docker/Dockerfile.superset docker/ || exit 1
|
-f docker/Dockerfile.superset docker/ || exit 1
|
||||||
|
|
||||||
push:
|
push:
|
||||||
@for svc in $(SERVICES); do \
|
@for svc in $(SERVICES); do \
|
||||||
echo "Pushing $$svc..."; \
|
echo "Pushing $$svc..."; \
|
||||||
docker push $(GHCR)/$$svc:$(SHA); \
|
docker push $(REGISTRY)/$$svc:$(SHA); \
|
||||||
docker push $(GHCR)/$$svc:latest; \
|
docker push $(REGISTRY)/$$svc:latest; \
|
||||||
done
|
done
|
||||||
|
|
||||||
deploy:
|
deploy:
|
||||||
@@ -70,5 +70,5 @@ deploy:
|
|||||||
|
|
||||||
clean:
|
clean:
|
||||||
@for svc in $(SERVICES); do \
|
@for svc in $(SERVICES); do \
|
||||||
docker rmi $(GHCR)/$$svc:$(SHA) $(GHCR)/$$svc:latest 2>/dev/null || true; \
|
docker rmi $(REGISTRY)/$$svc:$(SHA) $(REGISTRY)/$$svc:latest 2>/dev/null || true; \
|
||||||
done
|
done
|
||||||
|
|||||||
@@ -236,15 +236,12 @@ if ! kubectl create namespace "$NAMESPACE"; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Create GHCR image pull secret (if token available) ───────────────────────
|
# ── Create GHCR image pull secret (if token available) ───────────────────────
|
||||||
|
# NOTE: Images now served from Harbor at registry.celestium.life (no auth needed for pulls)
|
||||||
|
# This block is kept for backward compatibility but is no longer required
|
||||||
if [ -n "${GHCR_TOKEN:-}" ]; then
|
if [ -n "${GHCR_TOKEN:-}" ]; then
|
||||||
log "Creating ghcr-credentials secret ..."
|
log "GHCR_TOKEN set but images are on local Harbor — skipping GHCR secret"
|
||||||
kubectl create secret docker-registry ghcr-credentials \
|
|
||||||
--docker-server=ghcr.io \
|
|
||||||
--docker-username=celesrenata \
|
|
||||||
--docker-password="$GHCR_TOKEN" \
|
|
||||||
-n "$NAMESPACE" || true
|
|
||||||
else
|
else
|
||||||
log "GHCR_TOKEN not set — skipping image pull secret (images must be pullable without auth)"
|
log "Images served from registry.celestium.life (no pull secret needed)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Create Docker Hub pull secret (avoid rate limits) ────────────────────────
|
# ── Create Docker Hub pull secret (avoid rate limits) ────────────────────────
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: aggregation-worker
|
- name: aggregation-worker
|
||||||
image: ghcr.io/celesrenata/stonks-oracle/aggregation:latest
|
image: registry.celestium.life/stonks-oracle/aggregation:latest
|
||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
securityContext:
|
securityContext:
|
||||||
allowPrivilegeEscalation: false
|
allowPrivilegeEscalation: false
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: broker-adapter
|
- name: broker-adapter
|
||||||
image: ghcr.io/celesrenata/stonks-oracle/broker-adapter:latest
|
image: registry.celestium.life/stonks-oracle/broker-adapter:latest
|
||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
securityContext:
|
securityContext:
|
||||||
allowPrivilegeEscalation: false
|
allowPrivilegeEscalation: false
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: extractor-worker
|
- name: extractor-worker
|
||||||
image: ghcr.io/celesrenata/stonks-oracle/extractor:latest
|
image: registry.celestium.life/stonks-oracle/extractor:latest
|
||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
securityContext:
|
securityContext:
|
||||||
allowPrivilegeEscalation: false
|
allowPrivilegeEscalation: false
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: ingestion-worker
|
- name: ingestion-worker
|
||||||
image: ghcr.io/celesrenata/stonks-oracle/ingestion:latest
|
image: registry.celestium.life/stonks-oracle/ingestion:latest
|
||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
securityContext:
|
securityContext:
|
||||||
allowPrivilegeEscalation: false
|
allowPrivilegeEscalation: false
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: lake-publisher
|
- name: lake-publisher
|
||||||
image: ghcr.io/celesrenata/stonks-oracle/lake-publisher:latest
|
image: registry.celestium.life/stonks-oracle/lake-publisher:latest
|
||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
securityContext:
|
securityContext:
|
||||||
allowPrivilegeEscalation: false
|
allowPrivilegeEscalation: false
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: parser-worker
|
- name: parser-worker
|
||||||
image: ghcr.io/celesrenata/stonks-oracle/parser:latest
|
image: registry.celestium.life/stonks-oracle/parser:latest
|
||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
securityContext:
|
securityContext:
|
||||||
allowPrivilegeEscalation: false
|
allowPrivilegeEscalation: false
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: query-api
|
- name: query-api
|
||||||
image: ghcr.io/celesrenata/stonks-oracle/query-api:latest
|
image: registry.celestium.life/stonks-oracle/query-api:latest
|
||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 8000
|
- containerPort: 8000
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: recommendation-worker
|
- name: recommendation-worker
|
||||||
image: ghcr.io/celesrenata/stonks-oracle/recommendation:latest
|
image: registry.celestium.life/stonks-oracle/recommendation:latest
|
||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
securityContext:
|
securityContext:
|
||||||
allowPrivilegeEscalation: false
|
allowPrivilegeEscalation: false
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: risk-engine
|
- name: risk-engine
|
||||||
image: ghcr.io/celesrenata/stonks-oracle/risk:latest
|
image: registry.celestium.life/stonks-oracle/risk:latest
|
||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 8000
|
- containerPort: 8000
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: scheduler
|
- name: scheduler
|
||||||
image: ghcr.io/celesrenata/stonks-oracle/scheduler:latest
|
image: registry.celestium.life/stonks-oracle/scheduler:latest
|
||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
securityContext:
|
securityContext:
|
||||||
allowPrivilegeEscalation: false
|
allowPrivilegeEscalation: false
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: symbol-registry-api
|
- name: symbol-registry-api
|
||||||
image: ghcr.io/celesrenata/stonks-oracle/symbol-registry:latest
|
image: registry.celestium.life/stonks-oracle/symbol-registry:latest
|
||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 8000
|
- containerPort: 8000
|
||||||
|
|||||||
Reference in New Issue
Block a user