From 422326bf83e9e64da625ba668f4ee93e364e6930 Mon Sep 17 00:00:00 2001 From: Celes Renata Date: Mon, 20 Apr 2026 04:23:19 +0000 Subject: [PATCH] fix: inttest pipeline timeout and busybox grep compatibility - Poll job status instead of kubectl wait (catches Failed condition immediately instead of waiting 600s for Complete that never comes) - Replace grep -oP (Perl regex) with POSIX grep -o (BusyBox compat) --- infra/inttest/run_pipeline.sh | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/infra/inttest/run_pipeline.sh b/infra/inttest/run_pipeline.sh index 690df8f..f08a8fd 100755 --- a/infra/inttest/run_pipeline.sh +++ b/infra/inttest/run_pipeline.sh @@ -428,15 +428,26 @@ envsubst < "$REPO_ROOT/infra/inttest/runner.yaml" \ | kubectl apply -n "$NAMESPACE" -f - log "Waiting for test runner to complete (timeout: 600s) ..." -if kubectl wait --for=condition=complete job/inttest-runner -n "$NAMESPACE" --timeout=600s; then - log "Test runner completed successfully" - stage_end "integration_tests" "ok" -else - log "Test runner failed or timed out" - # Check if the job failed vs timed out - if kubectl wait --for=condition=failed job/inttest-runner -n "$NAMESPACE" --timeout=5s 2>/dev/null; then - log "Test runner job reported failure" +# Wait for either complete or failed — whichever comes first +JOB_DONE=false +for i in $(seq 1 120); do + STATUS=$(kubectl get job inttest-runner -n "$NAMESPACE" -o jsonpath='{.status.conditions[0].type}' 2>/dev/null || true) + if [ "$STATUS" = "Complete" ]; then + log "Test runner completed successfully" + stage_end "integration_tests" "ok" + JOB_DONE=true + break + elif [ "$STATUS" = "Failed" ]; then + log "Test runner job reported failure (tests failed)" + stage_fail "integration_tests" + PIPELINE_EXIT_CODE=1 + JOB_DONE=true + break fi + sleep 5 +done +if [ "$JOB_DONE" = false ]; then + log "Test runner timed out after 600s" debug_pod_failure "inttest-runner" "app=inttest-runner" stage_fail "integration_tests" PIPELINE_EXIT_CODE=1 @@ -468,10 +479,10 @@ if [ -n "$RUNNER_POD" ]; then # Parse test counts from logs (pytest output format: "X passed, Y failed, Z errors") TEST_OUTPUT=$(kubectl logs "$RUNNER_POD" -n "$NAMESPACE" 2>/dev/null || true) if [ -n "$TEST_OUTPUT" ]; then - # Extract counts from pytest summary line like "41 passed, 2 failed, 1 error" - TESTS_PASSED=$(echo "$TEST_OUTPUT" | grep -oP '\d+(?= passed)' | tail -1 || echo "0") - TESTS_FAILED=$(echo "$TEST_OUTPUT" | grep -oP '\d+(?= failed)' | tail -1 || echo "0") - TESTS_ERRORS=$(echo "$TEST_OUTPUT" | grep -oP '\d+(?= error)' | tail -1 || echo "0") + # Extract counts from pytest summary line like "172 passed, 6 failed" + TESTS_PASSED=$(echo "$TEST_OUTPUT" | grep -o '[0-9]* passed' | tail -1 | grep -o '[0-9]*' || echo "0") + TESTS_FAILED=$(echo "$TEST_OUTPUT" | grep -o '[0-9]* failed' | tail -1 | grep -o '[0-9]*' || echo "0") + TESTS_ERRORS=$(echo "$TEST_OUTPUT" | grep -o '[0-9]* error' | tail -1 | grep -o '[0-9]*' || echo "0") TESTS_PASSED=${TESTS_PASSED:-0} TESTS_FAILED=${TESTS_FAILED:-0} TESTS_ERRORS=${TESTS_ERRORS:-0}