fix: inttest pipeline timeout and busybox grep compatibility
- Poll job status instead of kubectl wait (catches Failed condition immediately instead of waiting 600s for Complete that never comes) - Replace grep -oP (Perl regex) with POSIX grep -o (BusyBox compat)
This commit is contained in:
@@ -428,15 +428,26 @@ envsubst < "$REPO_ROOT/infra/inttest/runner.yaml" \
|
|||||||
| kubectl apply -n "$NAMESPACE" -f -
|
| kubectl apply -n "$NAMESPACE" -f -
|
||||||
|
|
||||||
log "Waiting for test runner to complete (timeout: 600s) ..."
|
log "Waiting for test runner to complete (timeout: 600s) ..."
|
||||||
if kubectl wait --for=condition=complete job/inttest-runner -n "$NAMESPACE" --timeout=600s; then
|
# Wait for either complete or failed — whichever comes first
|
||||||
log "Test runner completed successfully"
|
JOB_DONE=false
|
||||||
stage_end "integration_tests" "ok"
|
for i in $(seq 1 120); do
|
||||||
else
|
STATUS=$(kubectl get job inttest-runner -n "$NAMESPACE" -o jsonpath='{.status.conditions[0].type}' 2>/dev/null || true)
|
||||||
log "Test runner failed or timed out"
|
if [ "$STATUS" = "Complete" ]; then
|
||||||
# Check if the job failed vs timed out
|
log "Test runner completed successfully"
|
||||||
if kubectl wait --for=condition=failed job/inttest-runner -n "$NAMESPACE" --timeout=5s 2>/dev/null; then
|
stage_end "integration_tests" "ok"
|
||||||
log "Test runner job reported failure"
|
JOB_DONE=true
|
||||||
|
break
|
||||||
|
elif [ "$STATUS" = "Failed" ]; then
|
||||||
|
log "Test runner job reported failure (tests failed)"
|
||||||
|
stage_fail "integration_tests"
|
||||||
|
PIPELINE_EXIT_CODE=1
|
||||||
|
JOB_DONE=true
|
||||||
|
break
|
||||||
fi
|
fi
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
if [ "$JOB_DONE" = false ]; then
|
||||||
|
log "Test runner timed out after 600s"
|
||||||
debug_pod_failure "inttest-runner" "app=inttest-runner"
|
debug_pod_failure "inttest-runner" "app=inttest-runner"
|
||||||
stage_fail "integration_tests"
|
stage_fail "integration_tests"
|
||||||
PIPELINE_EXIT_CODE=1
|
PIPELINE_EXIT_CODE=1
|
||||||
@@ -468,10 +479,10 @@ if [ -n "$RUNNER_POD" ]; then
|
|||||||
# Parse test counts from logs (pytest output format: "X passed, Y failed, Z errors")
|
# Parse test counts from logs (pytest output format: "X passed, Y failed, Z errors")
|
||||||
TEST_OUTPUT=$(kubectl logs "$RUNNER_POD" -n "$NAMESPACE" 2>/dev/null || true)
|
TEST_OUTPUT=$(kubectl logs "$RUNNER_POD" -n "$NAMESPACE" 2>/dev/null || true)
|
||||||
if [ -n "$TEST_OUTPUT" ]; then
|
if [ -n "$TEST_OUTPUT" ]; then
|
||||||
# Extract counts from pytest summary line like "41 passed, 2 failed, 1 error"
|
# Extract counts from pytest summary line like "172 passed, 6 failed"
|
||||||
TESTS_PASSED=$(echo "$TEST_OUTPUT" | grep -oP '\d+(?= passed)' | tail -1 || echo "0")
|
TESTS_PASSED=$(echo "$TEST_OUTPUT" | grep -o '[0-9]* passed' | tail -1 | grep -o '[0-9]*' || echo "0")
|
||||||
TESTS_FAILED=$(echo "$TEST_OUTPUT" | grep -oP '\d+(?= failed)' | tail -1 || echo "0")
|
TESTS_FAILED=$(echo "$TEST_OUTPUT" | grep -o '[0-9]* failed' | tail -1 | grep -o '[0-9]*' || echo "0")
|
||||||
TESTS_ERRORS=$(echo "$TEST_OUTPUT" | grep -oP '\d+(?= error)' | tail -1 || echo "0")
|
TESTS_ERRORS=$(echo "$TEST_OUTPUT" | grep -o '[0-9]* error' | tail -1 | grep -o '[0-9]*' || echo "0")
|
||||||
TESTS_PASSED=${TESTS_PASSED:-0}
|
TESTS_PASSED=${TESTS_PASSED:-0}
|
||||||
TESTS_FAILED=${TESTS_FAILED:-0}
|
TESTS_FAILED=${TESTS_FAILED:-0}
|
||||||
TESTS_ERRORS=${TESTS_ERRORS:-0}
|
TESTS_ERRORS=${TESTS_ERRORS:-0}
|
||||||
|
|||||||
Reference in New Issue
Block a user