fix: split pipeline into 4 workflows for cross-node scheduling

Multi-workflow with local-path storage + mismatchLabelKeys anti-affinity forces each workflow onto a different cluster node: - test: lint + pytest + vitest (node A) - build-1: scheduler, symbol-registry, ingestion, parser (node B) - build-2: extractor, aggregation, recommendation, risk (node C) - build-3: broker-adapter, lake-publisher, query-api, trading-engine, dashboard, superset (node D) - finalize: integration-test + github mirror (any available node)
2026-04-20 03:31:50 +00:00
parent 9850dc45b1
commit c81e17f527
6 changed files with 696 additions and 789 deletions
@@ -0,0 +1,171 @@
+# Workflow 2: Build batch 1 (node B)
+depends_on:
+  - test
+
+when:
+  event: push
+  branch: main
+
+steps:
+  build-scheduler:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/scheduler
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile.scheduler
+      context: .
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-symbol-registry:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/symbol-registry
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-ingestion:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/ingestion
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=python -m services.ingestion.worker"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-parser:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/parser
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=python -m services.parser.worker"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
@@ -0,0 +1,172 @@
+# Workflow 3: Build batch 2 (node C)
+depends_on:
+  - test
+
+when:
+  event: push
+  branch: main
+
+steps:
+  build-extractor:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/extractor
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=python -m services.extractor.worker"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-aggregation:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/aggregation
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=python -m services.aggregation.worker"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-recommendation:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/recommendation
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=python -m services.recommendation.worker"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-risk:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/risk
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=uvicorn services.risk.app:app --host 0.0.0.0 --port 8000"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
@@ -0,0 +1,253 @@
+# Workflow 4: Build batch 3 (node D)
+depends_on:
+  - test
+
+when:
+  event: push
+  branch: main
+
+steps:
+  build-broker-adapter:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/broker-adapter
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=python -m services.adapters.broker_adapter"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-lake-publisher:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/lake-publisher
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=python -m services.lake_publisher.worker"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-query-api:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/query-api
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=uvicorn services.api.app:app --host 0.0.0.0 --port 8000"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-trading-engine:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/trading-engine
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=uvicorn services.trading.app:app --host 0.0.0.0 --port 8000"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-dashboard:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/dashboard
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: frontend/Dockerfile
+      context: frontend
+      no_cache: true
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-superset:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/superset
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile.superset
+      context: docker
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
@@ -0,0 +1,48 @@
+# Workflow 5: Integration test and mirror
+depends_on:
+  - build-1
+  - build-2
+  - build-3
+
+when:
+  event: push
+  branch: main
+
+steps:
+  integration-test:
+    image: registry.celestium.life/dockerhub-cache/alpine/k8s:1.30.2
+    environment:
+      DOCKERHUB_USER:
+        from_secret: docker_username
+      DOCKERHUB_TOKEN:
+        from_secret: docker_password
+    commands:
+      - bash infra/inttest/run_pipeline.sh --image-tag ${CI_COMMIT_SHA} --results-file inttest-results.json
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 256Mi
+            cpu: 200m
+          limits:
+            memory: 512Mi
+            cpu: 1000m
+
+  mirror-github:
+    image: registry.celestium.life/dockerhub-cache/alpine/git
+    failure: ignore
+    environment:
+      GITHUB_SSH_KEY:
+        from_secret: github_ssh_key
+    commands:
+      - "[ -z \"$GITHUB_SSH_KEY\" ] && echo 'No SSH key configured, skipping mirror' && exit 0"
+      - mkdir -p ~/.ssh
+      - echo "$GITHUB_SSH_KEY" > ~/.ssh/id_ed25519
+      - chmod 600 ~/.ssh/id_ed25519
+      - ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
+      - git remote add github git@github.com:celesrenata/stonks-oracle.git || git remote set-url github git@github.com:celesrenata/stonks-oracle.git
+      - git fetch --unshallow origin || true
+      - git push github main --force
+      - git push github --tags --force
+    depends_on:
+      - integration-test
@@ -0,0 +1,52 @@
+# Workflow 1: Lint and tests (node A)
+when:
+  event: [push, pull_request]
+
+steps:
+  lint-python:
+    image: registry.celestium.life/dockerhub-cache/library/python:3.12-slim
+    commands:
+      - pip install ruff==0.15.10 --quiet
+      - ruff --version
+      - ruff check --output-format=concise .
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 256Mi
+            cpu: 200m
+          limits:
+            memory: 512Mi
+            cpu: 1000m
+
+  test-python:
+    image: registry.celestium.life/dockerhub-cache/library/python:3.12-slim
+    commands:
+      - pip install -r requirements.txt --quiet
+      - pip install hypothesis --quiet
+      - pytest tests/ -x --tb=short -q --ignore=tests/integration
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: 500m
+          limits:
+            memory: 1Gi
+            cpu: 2000m
+
+  test-frontend:
+    image: registry.celestium.life/dockerhub-cache/library/node:24-slim
+    commands:
+      - cd frontend
+      - npm ci
+      - npx vitest --run
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: 500m
+          limits:
+            memory: 1Gi
+            cpu: 2000m