From c81e17f5279cbea1f958a450d61a8c9951cfc6dc Mon Sep 17 00:00:00 2001 From: Celes Renata Date: Mon, 20 Apr 2026 03:31:50 +0000 Subject: [PATCH] fix: split pipeline into 4 workflows for cross-node scheduling Multi-workflow with local-path storage + mismatchLabelKeys anti-affinity forces each workflow onto a different cluster node: - test: lint + pytest + vitest (node A) - build-1: scheduler, symbol-registry, ingestion, parser (node B) - build-2: extractor, aggregation, recommendation, risk (node C) - build-3: broker-adapter, lake-publisher, query-api, trading-engine, dashboard, superset (node D) - finalize: integration-test + github mirror (any available node) --- .woodpecker.yml | 789 --------------------------------------- .woodpecker/build-1.yml | 171 +++++++++ .woodpecker/build-2.yml | 172 +++++++++ .woodpecker/build-3.yml | 253 +++++++++++++ .woodpecker/finalize.yml | 48 +++ .woodpecker/test.yml | 52 +++ 6 files changed, 696 insertions(+), 789 deletions(-) delete mode 100644 .woodpecker.yml create mode 100644 .woodpecker/build-1.yml create mode 100644 .woodpecker/build-2.yml create mode 100644 .woodpecker/build-3.yml create mode 100644 .woodpecker/finalize.yml create mode 100644 .woodpecker/test.yml diff --git a/.woodpecker.yml b/.woodpecker.yml deleted file mode 100644 index 977a39c..0000000 --- a/.woodpecker.yml +++ /dev/null @@ -1,789 +0,0 @@ -# Woodpecker CI pipeline for Stonks Oracle -# Images push to registry.celestium.life/stonks-oracle/: and :latest -# -# Storage: longhorn-rwx (RWX) for cross-node step scheduling -# Scheduling: podAntiAffinity spreads pipeline pods across cluster nodes - -steps: - lint-python: - image: registry.celestium.life/dockerhub-cache/library/python:3.12-slim - commands: - - pip install ruff==0.15.10 --quiet - - ruff --version - - ruff check --output-format=concise . - backend_options: - kubernetes: - resources: - requests: - memory: 256Mi - cpu: 200m - limits: - memory: 512Mi - cpu: 1000m - when: - event: [push, pull_request] - - test-python: - image: registry.celestium.life/dockerhub-cache/library/python:3.12-slim - commands: - - pip install -r requirements.txt --quiet - - pip install hypothesis --quiet - - pytest tests/ -x --tb=short -q --ignore=tests/integration - backend_options: - kubernetes: - resources: - requests: - memory: 512Mi - cpu: 500m - limits: - memory: 1Gi - cpu: 2000m - when: - event: [push, pull_request] - - test-frontend: - image: registry.celestium.life/dockerhub-cache/library/node:24-slim - commands: - - cd frontend - - npm ci - - npx vitest --run - backend_options: - kubernetes: - resources: - requests: - memory: 512Mi - cpu: 500m - limits: - memory: 1Gi - cpu: 2000m - when: - event: [push, pull_request] - - build-scheduler: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/scheduler - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: docker/Dockerfile.scheduler - context: . - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - build-symbol-registry: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/symbol-registry - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: docker/Dockerfile - context: . - build_args: "SERVICE_CMD=uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000" - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - build-ingestion: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/ingestion - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: docker/Dockerfile - context: . - build_args: "SERVICE_CMD=python -m services.ingestion.worker" - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - build-parser: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/parser - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: docker/Dockerfile - context: . - build_args: "SERVICE_CMD=python -m services.parser.worker" - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - build-extractor: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/extractor - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: docker/Dockerfile - context: . - build_args: "SERVICE_CMD=python -m services.extractor.worker" - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - build-aggregation: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/aggregation - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: docker/Dockerfile - context: . - build_args: "SERVICE_CMD=python -m services.aggregation.worker" - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - build-recommendation: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/recommendation - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: docker/Dockerfile - context: . - build_args: "SERVICE_CMD=python -m services.recommendation.worker" - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - build-risk: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/risk - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: docker/Dockerfile - context: . - build_args: "SERVICE_CMD=uvicorn services.risk.app:app --host 0.0.0.0 --port 8000" - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - build-broker-adapter: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/broker-adapter - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: docker/Dockerfile - context: . - build_args: "SERVICE_CMD=python -m services.adapters.broker_adapter" - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - build-lake-publisher: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/lake-publisher - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: docker/Dockerfile - context: . - build_args: "SERVICE_CMD=python -m services.lake_publisher.worker" - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - build-query-api: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/query-api - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: docker/Dockerfile - context: . - build_args: "SERVICE_CMD=uvicorn services.api.app:app --host 0.0.0.0 --port 8000" - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - build-trading-engine: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/trading-engine - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: docker/Dockerfile - context: . - build_args: "SERVICE_CMD=uvicorn services.trading.app:app --host 0.0.0.0 --port 8000" - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - build-dashboard: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/dashboard - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: frontend/Dockerfile - context: frontend - no_cache: true - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - build-superset: - image: woodpeckerci/plugin-docker-buildx - privileged: true - settings: - repo: registry.celestium.life/stonks-oracle/superset - registry: registry.celestium.life - custom_dns: 192.168.42.1 - buildx_flags: --driver-opt network=host - buildkitd_config: | - [registry."docker.io"] - mirrors = ["registry.celestium.life/v2/dockerhub-cache"] - [registry."ghcr.io"] - mirrors = ["registry.celestium.life/v2/ghcr-cache"] - no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" - logins: - - registry: https://registry.celestium.life - username: - from_secret: harbor_username - password: - from_secret: harbor_password - - registry: https://index.docker.io/v1/ - username: - from_secret: docker_username - password: - from_secret: docker_password - tags: - - ${CI_COMMIT_SHA} - - latest - dockerfile: docker/Dockerfile.superset - context: docker - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 4000m - when: - event: push - branch: main - depends_on: - - lint-python - - test-python - - test-frontend - - integration-test: - image: registry.celestium.life/dockerhub-cache/alpine/k8s:1.30.2 - environment: - DOCKERHUB_USER: - from_secret: docker_username - DOCKERHUB_TOKEN: - from_secret: docker_password - commands: - - bash infra/inttest/run_pipeline.sh --image-tag ${CI_COMMIT_SHA} --results-file inttest-results.json - backend_options: - kubernetes: - resources: - requests: - memory: 256Mi - cpu: 200m - limits: - memory: 512Mi - cpu: 1000m - when: - event: push - branch: main - depends_on: - - build-scheduler - - build-symbol-registry - - build-ingestion - - build-parser - - build-extractor - - build-aggregation - - build-recommendation - - build-risk - - build-broker-adapter - - build-lake-publisher - - build-query-api - - build-trading-engine - - build-dashboard - - build-superset - - mirror-github: - image: registry.celestium.life/dockerhub-cache/alpine/git - failure: ignore - environment: - GITHUB_SSH_KEY: - from_secret: github_ssh_key - commands: - - "[ -z \"$GITHUB_SSH_KEY\" ] && echo 'No SSH key configured, skipping mirror' && exit 0" - - mkdir -p ~/.ssh - - echo "$GITHUB_SSH_KEY" > ~/.ssh/id_ed25519 - - chmod 600 ~/.ssh/id_ed25519 - - ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null - - git remote add github git@github.com:celesrenata/stonks-oracle.git || git remote set-url github git@github.com:celesrenata/stonks-oracle.git - - git fetch --unshallow origin || true - - git push github main --force - - git push github --tags --force - when: - event: push - branch: main - depends_on: - - integration-test diff --git a/.woodpecker/build-1.yml b/.woodpecker/build-1.yml new file mode 100644 index 0000000..1adc719 --- /dev/null +++ b/.woodpecker/build-1.yml @@ -0,0 +1,171 @@ +# Workflow 2: Build batch 1 (node B) +depends_on: + - test + +when: + event: push + branch: main + +steps: + build-scheduler: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/scheduler + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: docker/Dockerfile.scheduler + context: . + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m + + build-symbol-registry: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/symbol-registry + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: docker/Dockerfile + context: . + build_args: "SERVICE_CMD=uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000" + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m + + build-ingestion: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/ingestion + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: docker/Dockerfile + context: . + build_args: "SERVICE_CMD=python -m services.ingestion.worker" + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m + + build-parser: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/parser + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: docker/Dockerfile + context: . + build_args: "SERVICE_CMD=python -m services.parser.worker" + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m diff --git a/.woodpecker/build-2.yml b/.woodpecker/build-2.yml new file mode 100644 index 0000000..4221d06 --- /dev/null +++ b/.woodpecker/build-2.yml @@ -0,0 +1,172 @@ +# Workflow 3: Build batch 2 (node C) +depends_on: + - test + +when: + event: push + branch: main + +steps: + build-extractor: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/extractor + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: docker/Dockerfile + context: . + build_args: "SERVICE_CMD=python -m services.extractor.worker" + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m + + build-aggregation: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/aggregation + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: docker/Dockerfile + context: . + build_args: "SERVICE_CMD=python -m services.aggregation.worker" + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m + + build-recommendation: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/recommendation + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: docker/Dockerfile + context: . + build_args: "SERVICE_CMD=python -m services.recommendation.worker" + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m + + build-risk: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/risk + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: docker/Dockerfile + context: . + build_args: "SERVICE_CMD=uvicorn services.risk.app:app --host 0.0.0.0 --port 8000" + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m diff --git a/.woodpecker/build-3.yml b/.woodpecker/build-3.yml new file mode 100644 index 0000000..87f7900 --- /dev/null +++ b/.woodpecker/build-3.yml @@ -0,0 +1,253 @@ +# Workflow 4: Build batch 3 (node D) +depends_on: + - test + +when: + event: push + branch: main + +steps: + build-broker-adapter: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/broker-adapter + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: docker/Dockerfile + context: . + build_args: "SERVICE_CMD=python -m services.adapters.broker_adapter" + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m + + build-lake-publisher: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/lake-publisher + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: docker/Dockerfile + context: . + build_args: "SERVICE_CMD=python -m services.lake_publisher.worker" + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m + + build-query-api: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/query-api + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: docker/Dockerfile + context: . + build_args: "SERVICE_CMD=uvicorn services.api.app:app --host 0.0.0.0 --port 8000" + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m + + build-trading-engine: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/trading-engine + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: docker/Dockerfile + context: . + build_args: "SERVICE_CMD=uvicorn services.trading.app:app --host 0.0.0.0 --port 8000" + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m + + build-dashboard: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/dashboard + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: frontend/Dockerfile + context: frontend + no_cache: true + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m + + build-superset: + image: woodpeckerci/plugin-docker-buildx + privileged: true + settings: + repo: registry.celestium.life/stonks-oracle/superset + registry: registry.celestium.life + custom_dns: 192.168.42.1 + buildx_flags: --driver-opt network=host + buildkitd_config: | + [registry."docker.io"] + mirrors = ["registry.celestium.life/v2/dockerhub-cache"] + [registry."ghcr.io"] + mirrors = ["registry.celestium.life/v2/ghcr-cache"] + no_proxy: "10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,.local,.celestium.life" + logins: + - registry: https://registry.celestium.life + username: + from_secret: harbor_username + password: + from_secret: harbor_password + - registry: https://index.docker.io/v1/ + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${CI_COMMIT_SHA} + - latest + dockerfile: docker/Dockerfile.superset + context: docker + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 4000m diff --git a/.woodpecker/finalize.yml b/.woodpecker/finalize.yml new file mode 100644 index 0000000..afd8f26 --- /dev/null +++ b/.woodpecker/finalize.yml @@ -0,0 +1,48 @@ +# Workflow 5: Integration test and mirror +depends_on: + - build-1 + - build-2 + - build-3 + +when: + event: push + branch: main + +steps: + integration-test: + image: registry.celestium.life/dockerhub-cache/alpine/k8s:1.30.2 + environment: + DOCKERHUB_USER: + from_secret: docker_username + DOCKERHUB_TOKEN: + from_secret: docker_password + commands: + - bash infra/inttest/run_pipeline.sh --image-tag ${CI_COMMIT_SHA} --results-file inttest-results.json + backend_options: + kubernetes: + resources: + requests: + memory: 256Mi + cpu: 200m + limits: + memory: 512Mi + cpu: 1000m + + mirror-github: + image: registry.celestium.life/dockerhub-cache/alpine/git + failure: ignore + environment: + GITHUB_SSH_KEY: + from_secret: github_ssh_key + commands: + - "[ -z \"$GITHUB_SSH_KEY\" ] && echo 'No SSH key configured, skipping mirror' && exit 0" + - mkdir -p ~/.ssh + - echo "$GITHUB_SSH_KEY" > ~/.ssh/id_ed25519 + - chmod 600 ~/.ssh/id_ed25519 + - ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null + - git remote add github git@github.com:celesrenata/stonks-oracle.git || git remote set-url github git@github.com:celesrenata/stonks-oracle.git + - git fetch --unshallow origin || true + - git push github main --force + - git push github --tags --force + depends_on: + - integration-test diff --git a/.woodpecker/test.yml b/.woodpecker/test.yml new file mode 100644 index 0000000..638bc5c --- /dev/null +++ b/.woodpecker/test.yml @@ -0,0 +1,52 @@ +# Workflow 1: Lint and tests (node A) +when: + event: [push, pull_request] + +steps: + lint-python: + image: registry.celestium.life/dockerhub-cache/library/python:3.12-slim + commands: + - pip install ruff==0.15.10 --quiet + - ruff --version + - ruff check --output-format=concise . + backend_options: + kubernetes: + resources: + requests: + memory: 256Mi + cpu: 200m + limits: + memory: 512Mi + cpu: 1000m + + test-python: + image: registry.celestium.life/dockerhub-cache/library/python:3.12-slim + commands: + - pip install -r requirements.txt --quiet + - pip install hypothesis --quiet + - pytest tests/ -x --tb=short -q --ignore=tests/integration + backend_options: + kubernetes: + resources: + requests: + memory: 512Mi + cpu: 500m + limits: + memory: 1Gi + cpu: 2000m + + test-frontend: + image: registry.celestium.life/dockerhub-cache/library/node:24-slim + commands: + - cd frontend + - npm ci + - npx vitest --run + backend_options: + kubernetes: + resources: + requests: + memory: 512Mi + cpu: 500m + limits: + memory: 1Gi + cpu: 2000m