fix: split pipeline into 4 workflows for cross-node scheduling

Multi-workflow with local-path storage + mismatchLabelKeys anti-affinity forces each workflow onto a different cluster node: - test: lint + pytest + vitest (node A) - build-1: scheduler, symbol-registry, ingestion, parser (node B) - build-2: extractor, aggregation, recommendation, risk (node C) - build-3: broker-adapter, lake-publisher, query-api, trading-engine, dashboard, superset (node D) - finalize: integration-test + github mirror (any available node)
2026-04-20 03:31:50 +00:00
parent 9850dc45b1
commit c81e17f527
6 changed files with 696 additions and 789 deletions
@@ -0,0 +1,171 @@
+# Workflow 2: Build batch 1 (node B)
+depends_on:
+  - test
+
+when:
+  event: push
+  branch: main
+
+steps:
+  build-scheduler:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/scheduler
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile.scheduler
+      context: .
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-symbol-registry:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/symbol-registry
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-ingestion:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/ingestion
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=python -m services.ingestion.worker"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m
+
+  build-parser:
+    image: woodpeckerci/plugin-docker-buildx
+    privileged: true
+    settings:
+      repo: registry.celestium.life/stonks-oracle/parser
+      registry: registry.celestium.life
+      custom_dns: 192.168.42.1
+      buildx_flags: --driver-opt network=host
+      buildkitd_config: |
+        [registry."docker.io"]
+          mirrors = ["registry.celestium.life/v2/dockerhub-cache"]
+        [registry."ghcr.io"]
+          mirrors = ["registry.celestium.life/v2/ghcr-cache"]
+      no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life"
+      logins:
+        - registry: https://registry.celestium.life
+          username:
+            from_secret: harbor_username
+          password:
+            from_secret: harbor_password
+        - registry: https://index.docker.io/v1/
+          username:
+            from_secret: docker_username
+          password:
+            from_secret: docker_password
+      tags:
+        - ${CI_COMMIT_SHA}
+        - latest
+      dockerfile: docker/Dockerfile
+      context: .
+      build_args: "SERVICE_CMD=python -m services.parser.worker"
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1000m
+          limits:
+            memory: 2Gi
+            cpu: 4000m