fix: use kubectl wait for job completion detection in inttest pipeline
ci/woodpecker/push/test Pipeline was successful
ci/woodpecker/push/build-3 Pipeline was successful
ci/woodpecker/push/build-1 Pipeline was successful
ci/woodpecker/push/build-2 Pipeline was successful
ci/woodpecker/push/finalize Pipeline was successful
Build and Push / lint-and-test (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.adapters.broker_adapter name:broker-adapter]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.aggregation.worker name:aggregation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.extractor.worker name:extractor]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.ingestion.worker name:ingestion]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.lake_publisher.worker name:lake-publisher]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.parser.worker name:parser]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.recommendation.worker name:recommendation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.scheduler.app name:scheduler]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.api.app:app --host 0.0.0.0 --port 8000 name:query-api]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.risk.app:app --host 0.0.0.0 --port 8000 name:risk]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000 name:symbol-registry]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.trading.app:app --host 0.0.0.0 --port 8000 name:trading-engine]) (push) Has been cancelled
Build and Push / build-dashboard (push) Has been cancelled
Build and Push / build-superset (push) Has been cancelled
Build and Push / integration-test (push) Has been cancelled
ci/woodpecker/push/test Pipeline was successful
ci/woodpecker/push/build-3 Pipeline was successful
ci/woodpecker/push/build-1 Pipeline was successful
ci/woodpecker/push/build-2 Pipeline was successful
ci/woodpecker/push/finalize Pipeline was successful
Build and Push / lint-and-test (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.adapters.broker_adapter name:broker-adapter]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.aggregation.worker name:aggregation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.extractor.worker name:extractor]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.ingestion.worker name:ingestion]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.lake_publisher.worker name:lake-publisher]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.parser.worker name:parser]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.recommendation.worker name:recommendation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.scheduler.app name:scheduler]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.api.app:app --host 0.0.0.0 --port 8000 name:query-api]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.risk.app:app --host 0.0.0.0 --port 8000 name:risk]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000 name:symbol-registry]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.trading.app:app --host 0.0.0.0 --port 8000 name:trading-engine]) (push) Has been cancelled
Build and Push / build-dashboard (push) Has been cancelled
Build and Push / build-superset (push) Has been cancelled
Build and Push / integration-test (push) Has been cancelled
The polling loop checked conditions[0].type which missed the Complete condition when it wasn't at index 0. Switch to kubectl wait --for=condition=complete which handles condition matching reliably.
This commit is contained in:
@@ -428,24 +428,23 @@ envsubst < "$REPO_ROOT/infra/inttest/runner.yaml" \
|
||||
| kubectl apply -n "$NAMESPACE" -f -
|
||||
|
||||
log "Waiting for test runner to complete (timeout: 600s) ..."
|
||||
# Wait for either complete or failed — whichever comes first
|
||||
# Use kubectl wait which handles condition matching reliably.
|
||||
# Wait for Complete first; if that times out, check for Failed.
|
||||
JOB_DONE=false
|
||||
for i in $(seq 1 120); do
|
||||
STATUS=$(kubectl get job inttest-runner -n "$NAMESPACE" -o jsonpath='{.status.conditions[0].type}' 2>/dev/null || true)
|
||||
if [ "$STATUS" = "Complete" ]; then
|
||||
if kubectl wait --for=condition=complete job/inttest-runner -n "$NAMESPACE" --timeout=600s 2>/dev/null; then
|
||||
log "Test runner completed successfully"
|
||||
stage_end "integration_tests" "ok"
|
||||
JOB_DONE=true
|
||||
break
|
||||
elif [ "$STATUS" = "Failed" ]; then
|
||||
else
|
||||
# Check if it failed (non-zero exit) vs timed out
|
||||
FAILED=$(kubectl get job inttest-runner -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true)
|
||||
if [ "$FAILED" = "True" ]; then
|
||||
log "Test runner job reported failure (tests failed)"
|
||||
stage_fail "integration_tests"
|
||||
PIPELINE_EXIT_CODE=1
|
||||
JOB_DONE=true
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
fi
|
||||
if [ "$JOB_DONE" = false ]; then
|
||||
log "Test runner timed out after 600s"
|
||||
debug_pod_failure "inttest-runner" "app=inttest-runner"
|
||||
|
||||
Reference in New Issue
Block a user