fix: split pipeline into 4 workflows for cross-node scheduling

Multi-workflow with local-path storage + mismatchLabelKeys anti-affinity
forces each workflow onto a different cluster node:
- test: lint + pytest + vitest (node A)
- build-1: scheduler, symbol-registry, ingestion, parser (node B)
- build-2: extractor, aggregation, recommendation, risk (node C)
- build-3: broker-adapter, lake-publisher, query-api, trading-engine, dashboard, superset (node D)
- finalize: integration-test + github mirror (any available node)
This commit is contained in:
Celes Renata
2026-04-20 03:31:50 +00:00
parent 9850dc45b1
commit c81e17f527
6 changed files with 696 additions and 789 deletions
+48
View File
@@ -0,0 +1,48 @@
# Workflow 5: Integration test and mirror
depends_on:
- build-1
- build-2
- build-3
when:
event: push
branch: main
steps:
integration-test:
image: registry.celestium.life/dockerhub-cache/alpine/k8s:1.30.2
environment:
DOCKERHUB_USER:
from_secret: docker_username
DOCKERHUB_TOKEN:
from_secret: docker_password
commands:
- bash infra/inttest/run_pipeline.sh --image-tag ${CI_COMMIT_SHA} --results-file inttest-results.json
backend_options:
kubernetes:
resources:
requests:
memory: 256Mi
cpu: 200m
limits:
memory: 512Mi
cpu: 1000m
mirror-github:
image: registry.celestium.life/dockerhub-cache/alpine/git
failure: ignore
environment:
GITHUB_SSH_KEY:
from_secret: github_ssh_key
commands:
- "[ -z \"$GITHUB_SSH_KEY\" ] && echo 'No SSH key configured, skipping mirror' && exit 0"
- mkdir -p ~/.ssh
- echo "$GITHUB_SSH_KEY" > ~/.ssh/id_ed25519
- chmod 600 ~/.ssh/id_ed25519
- ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
- git remote add github git@github.com:celesrenata/stonks-oracle.git || git remote set-url github git@github.com:celesrenata/stonks-oracle.git
- git fetch --unshallow origin || true
- git push github main --force
- git push github --tags --force
depends_on:
- integration-test