fix: split pipeline into 4 workflows for cross-node scheduling
ci/woodpecker/push/test Pipeline was successful
ci/woodpecker/push/build-2 Pipeline failed
ci/woodpecker/push/build-3 Pipeline failed
ci/woodpecker/push/build-1 Pipeline failed
ci/woodpecker/push/finalize unknown status
Build and Push / lint-and-test (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.adapters.broker_adapter name:broker-adapter]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.aggregation.worker name:aggregation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.extractor.worker name:extractor]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.ingestion.worker name:ingestion]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.lake_publisher.worker name:lake-publisher]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.parser.worker name:parser]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.recommendation.worker name:recommendation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.scheduler.app name:scheduler]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.api.app:app --host 0.0.0.0 --port 8000 name:query-api]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.risk.app:app --host 0.0.0.0 --port 8000 name:risk]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000 name:symbol-registry]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.trading.app:app --host 0.0.0.0 --port 8000 name:trading-engine]) (push) Has been cancelled
Build and Push / build-dashboard (push) Has been cancelled
Build and Push / build-superset (push) Has been cancelled
Build and Push / integration-test (push) Has been cancelled
ci/woodpecker/push/test Pipeline was successful
ci/woodpecker/push/build-2 Pipeline failed
ci/woodpecker/push/build-3 Pipeline failed
ci/woodpecker/push/build-1 Pipeline failed
ci/woodpecker/push/finalize unknown status
Build and Push / lint-and-test (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.adapters.broker_adapter name:broker-adapter]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.aggregation.worker name:aggregation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.extractor.worker name:extractor]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.ingestion.worker name:ingestion]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.lake_publisher.worker name:lake-publisher]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.parser.worker name:parser]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.recommendation.worker name:recommendation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.scheduler.app name:scheduler]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.api.app:app --host 0.0.0.0 --port 8000 name:query-api]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.risk.app:app --host 0.0.0.0 --port 8000 name:risk]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000 name:symbol-registry]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.trading.app:app --host 0.0.0.0 --port 8000 name:trading-engine]) (push) Has been cancelled
Build and Push / build-dashboard (push) Has been cancelled
Build and Push / build-superset (push) Has been cancelled
Build and Push / integration-test (push) Has been cancelled
Multi-workflow with local-path storage + mismatchLabelKeys anti-affinity forces each workflow onto a different cluster node: - test: lint + pytest + vitest (node A) - build-1: scheduler, symbol-registry, ingestion, parser (node B) - build-2: extractor, aggregation, recommendation, risk (node C) - build-3: broker-adapter, lake-publisher, query-api, trading-engine, dashboard, superset (node D) - finalize: integration-test + github mirror (any available node)
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
# Workflow 5: Integration test and mirror
|
||||
depends_on:
|
||||
- build-1
|
||||
- build-2
|
||||
- build-3
|
||||
|
||||
when:
|
||||
event: push
|
||||
branch: main
|
||||
|
||||
steps:
|
||||
integration-test:
|
||||
image: registry.celestium.life/dockerhub-cache/alpine/k8s:1.30.2
|
||||
environment:
|
||||
DOCKERHUB_USER:
|
||||
from_secret: docker_username
|
||||
DOCKERHUB_TOKEN:
|
||||
from_secret: docker_password
|
||||
commands:
|
||||
- bash infra/inttest/run_pipeline.sh --image-tag ${CI_COMMIT_SHA} --results-file inttest-results.json
|
||||
backend_options:
|
||||
kubernetes:
|
||||
resources:
|
||||
requests:
|
||||
memory: 256Mi
|
||||
cpu: 200m
|
||||
limits:
|
||||
memory: 512Mi
|
||||
cpu: 1000m
|
||||
|
||||
mirror-github:
|
||||
image: registry.celestium.life/dockerhub-cache/alpine/git
|
||||
failure: ignore
|
||||
environment:
|
||||
GITHUB_SSH_KEY:
|
||||
from_secret: github_ssh_key
|
||||
commands:
|
||||
- "[ -z \"$GITHUB_SSH_KEY\" ] && echo 'No SSH key configured, skipping mirror' && exit 0"
|
||||
- mkdir -p ~/.ssh
|
||||
- echo "$GITHUB_SSH_KEY" > ~/.ssh/id_ed25519
|
||||
- chmod 600 ~/.ssh/id_ed25519
|
||||
- ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
|
||||
- git remote add github git@github.com:celesrenata/stonks-oracle.git || git remote set-url github git@github.com:celesrenata/stonks-oracle.git
|
||||
- git fetch --unshallow origin || true
|
||||
- git push github main --force
|
||||
- git push github --tags --force
|
||||
depends_on:
|
||||
- integration-test
|
||||
Reference in New Issue
Block a user