fix: clean up utcnow deprecation warnings, fix 12 failing tests, add CI/CD pipeline manifests
- Replace all datetime.utcnow() with datetime.now(tz=timezone.utc) across 8 files - Fix 12 failing tests to match current implementation behavior - Fix pytest_plugins in non-top-level conftest (moved to root conftest.py) - Auto-fix 189 lint issues (import sorting, unused imports) - Add CI/CD pipeline infrastructure (ARC, ArgoCD, Kargo manifests) - Add values-beta.yaml and values-paper.yaml for staged deployments - Update GitHub Actions workflow to use self-hosted-gremlin runners - Add integration-test job to CI pipeline Result: 1596 passed, 0 failed, 0 warnings
This commit is contained in:
Celes Renata
Executable
+458
@@ -0,0 +1,458 @@
|
||||
#!/bin/bash
|
||||
# Integration test pipeline — standalone orchestration script
|
||||
#
|
||||
# Deploys an ephemeral Kubernetes sandbox (postgres, redis, minio, services),
|
||||
# seeds deterministic data, runs the integration test suite, collects results,
|
||||
# and tears everything down.
|
||||
#
|
||||
# Designed to be invoked by any CI/CD system or a human developer.
|
||||
#
|
||||
# Usage: bash infra/inttest/run_pipeline.sh [OPTIONS]
|
||||
#
|
||||
# Options:
|
||||
# --image-tag TAG Docker image tag to deploy (default: latest)
|
||||
# --namespace NAME Override namespace name (default: stonks-inttest-<timestamp>)
|
||||
# --skip-teardown Leave namespace running after tests (for debugging)
|
||||
# --results-file PATH Path for JSON results output (default: inttest-results.json)
|
||||
# -h, --help Show usage
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 All tests passed
|
||||
# 1 One or more test failures
|
||||
# 2 Infrastructure setup failure
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
# ── Defaults ─────────────────────────────────────────────────────────────────
|
||||
IMAGE_TAG="latest"
|
||||
NAMESPACE="stonks-inttest-$(date +%s)"
|
||||
SKIP_TEARDOWN=false
|
||||
RESULTS_FILE="inttest-results.json"
|
||||
|
||||
# ── Stage tracking ───────────────────────────────────────────────────────────
|
||||
declare -A STAGE_START
|
||||
declare -A STAGE_DURATION
|
||||
declare -A STAGE_STATUS
|
||||
PIPELINE_EXIT_CODE=0
|
||||
PIPELINE_START=$(date +%s)
|
||||
STARTED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: bash infra/inttest/run_pipeline.sh [OPTIONS]
|
||||
|
||||
Options:
|
||||
--image-tag TAG Docker image tag to deploy (default: latest)
|
||||
--namespace NAME Override namespace name (default: stonks-inttest-<timestamp>)
|
||||
--skip-teardown Leave namespace running after tests (for debugging)
|
||||
--results-file PATH Path for JSON results output (default: inttest-results.json)
|
||||
-h, --help Show usage
|
||||
|
||||
Exit codes:
|
||||
0 All tests passed
|
||||
1 One or more test failures
|
||||
2 Infrastructure setup failure
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
log() {
|
||||
echo "[$(date -u +"%H:%M:%S")] $*"
|
||||
}
|
||||
|
||||
stage_start() {
|
||||
local name="$1"
|
||||
log "▶ Stage: $name"
|
||||
STAGE_START[$name]=$(date +%s)
|
||||
}
|
||||
|
||||
stage_end() {
|
||||
local name="$1"
|
||||
local status="${2:-ok}"
|
||||
local end_ts
|
||||
end_ts=$(date +%s)
|
||||
STAGE_DURATION[$name]=$(( end_ts - ${STAGE_START[$name]} ))
|
||||
STAGE_STATUS[$name]="$status"
|
||||
log "✓ Stage: $name completed in ${STAGE_DURATION[$name]}s (${status})"
|
||||
}
|
||||
|
||||
stage_fail() {
|
||||
local name="$1"
|
||||
local end_ts
|
||||
end_ts=$(date +%s)
|
||||
STAGE_DURATION[$name]=$(( end_ts - ${STAGE_START[$name]} ))
|
||||
STAGE_STATUS[$name]="failed"
|
||||
log "✗ Stage: $name FAILED after ${STAGE_DURATION[$name]}s"
|
||||
}
|
||||
|
||||
# ── Parse CLI args ───────────────────────────────────────────────────────────
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--image-tag)
|
||||
IMAGE_TAG="$2"
|
||||
shift 2
|
||||
;;
|
||||
--namespace)
|
||||
NAMESPACE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--skip-teardown)
|
||||
SKIP_TEARDOWN=true
|
||||
shift
|
||||
;;
|
||||
--results-file)
|
||||
RESULTS_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
echo "Run with --help for usage."
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
export NAMESPACE
|
||||
export IMAGE_TAG
|
||||
|
||||
log "Pipeline starting"
|
||||
log " Namespace: $NAMESPACE"
|
||||
log " Image tag: $IMAGE_TAG"
|
||||
log " Results: $RESULTS_FILE"
|
||||
log " Teardown: $([ "$SKIP_TEARDOWN" = true ] && echo "SKIPPED" || echo "enabled")"
|
||||
|
||||
# ── Test result tracking ─────────────────────────────────────────────────────
|
||||
TESTS_TOTAL=0
|
||||
TESTS_PASSED=0
|
||||
TESTS_FAILED=0
|
||||
TESTS_ERRORS=0
|
||||
PROFILING_JSON=""
|
||||
|
||||
# ── Write JSON results ───────────────────────────────────────────────────────
|
||||
write_results() {
|
||||
local completed_at
|
||||
completed_at=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
# Build stages JSON
|
||||
local stages_json="{"
|
||||
local first=true
|
||||
for stage_name in infra_deploy seed_data service_deploy integration_tests teardown; do
|
||||
local dur="${STAGE_DURATION[$stage_name]:-0}"
|
||||
local st="${STAGE_STATUS[$stage_name]:-skipped}"
|
||||
if [ "$first" = true ]; then
|
||||
first=false
|
||||
else
|
||||
stages_json+=","
|
||||
fi
|
||||
stages_json+="\"${stage_name}\":{\"duration_s\":${dur},\"status\":\"${st}\"}"
|
||||
done
|
||||
stages_json+="}"
|
||||
|
||||
# Build profiling section
|
||||
local profiling_section
|
||||
if [ -n "$PROFILING_JSON" ] && [ -f "$PROFILING_JSON" ]; then
|
||||
profiling_section=$(cat "$PROFILING_JSON")
|
||||
else
|
||||
profiling_section='{"endpoints":{},"slow_endpoints":[]}'
|
||||
fi
|
||||
|
||||
cat > "$RESULTS_FILE" <<RESULTS_EOF
|
||||
{
|
||||
"run_id": "${NAMESPACE}",
|
||||
"image_tag": "${IMAGE_TAG}",
|
||||
"started_at": "${STARTED_AT}",
|
||||
"completed_at": "${completed_at}",
|
||||
"exit_code": ${PIPELINE_EXIT_CODE},
|
||||
"stages": ${stages_json},
|
||||
"tests": {
|
||||
"total": ${TESTS_TOTAL},
|
||||
"passed": ${TESTS_PASSED},
|
||||
"failed": ${TESTS_FAILED},
|
||||
"errors": ${TESTS_ERRORS}
|
||||
},
|
||||
"profiling": ${profiling_section}
|
||||
}
|
||||
RESULTS_EOF
|
||||
|
||||
log "Results written to $RESULTS_FILE"
|
||||
}
|
||||
|
||||
# ── Cleanup trap ─────────────────────────────────────────────────────────────
|
||||
cleanup() {
|
||||
stage_start "teardown"
|
||||
if [ "$SKIP_TEARDOWN" = true ]; then
|
||||
log "Teardown skipped (--skip-teardown). Namespace $NAMESPACE left running."
|
||||
stage_end "teardown" "skipped"
|
||||
else
|
||||
log "Tearing down namespace $NAMESPACE ..."
|
||||
kubectl delete namespace "$NAMESPACE" --wait=false 2>/dev/null || true
|
||||
stage_end "teardown" "ok"
|
||||
fi
|
||||
write_results
|
||||
log "Pipeline finished with exit code $PIPELINE_EXIT_CODE"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Stage: Create namespace
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
stage_start "infra_deploy"
|
||||
|
||||
log "Creating namespace $NAMESPACE ..."
|
||||
if ! kubectl create namespace "$NAMESPACE"; then
|
||||
log "FATAL: Failed to create namespace $NAMESPACE"
|
||||
stage_fail "infra_deploy"
|
||||
PIPELINE_EXIT_CODE=2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# ── Create GHCR image pull secret (if token available) ───────────────────────
|
||||
if [ -n "${GHCR_TOKEN:-}" ]; then
|
||||
log "Creating ghcr-credentials secret ..."
|
||||
kubectl create secret docker-registry ghcr-credentials \
|
||||
--docker-server=ghcr.io \
|
||||
--docker-username=celesrenata \
|
||||
--docker-password="$GHCR_TOKEN" \
|
||||
-n "$NAMESPACE" || true
|
||||
else
|
||||
log "GHCR_TOKEN not set — skipping image pull secret (images must be pullable without auth)"
|
||||
fi
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Stage: Deploy infra (postgres, redis, minio)
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
log "Creating postgres-migrations ConfigMap ..."
|
||||
if ! kubectl create configmap postgres-migrations \
|
||||
--from-file="$REPO_ROOT/infra/migrations/" \
|
||||
-n "$NAMESPACE"; then
|
||||
log "FATAL: Failed to create postgres-migrations ConfigMap"
|
||||
stage_fail "infra_deploy"
|
||||
PIPELINE_EXIT_CODE=2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
log "Applying postgres manifest ..."
|
||||
envsubst < "$REPO_ROOT/infra/inttest/postgres.yaml" | kubectl apply -n "$NAMESPACE" -f -
|
||||
|
||||
log "Applying redis manifest ..."
|
||||
envsubst < "$REPO_ROOT/infra/inttest/redis.yaml" | kubectl apply -n "$NAMESPACE" -f -
|
||||
|
||||
log "Applying minio manifest ..."
|
||||
envsubst < "$REPO_ROOT/infra/inttest/minio.yaml" | kubectl apply -n "$NAMESPACE" -f -
|
||||
|
||||
log "Waiting for postgres readiness ..."
|
||||
if ! kubectl wait --for=condition=ready pod -l app=postgres -n "$NAMESPACE" --timeout=120s; then
|
||||
log "FATAL: PostgreSQL did not become ready"
|
||||
stage_fail "infra_deploy"
|
||||
PIPELINE_EXIT_CODE=2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
log "Waiting for redis readiness ..."
|
||||
if ! kubectl wait --for=condition=ready pod -l app=redis -n "$NAMESPACE" --timeout=60s; then
|
||||
log "FATAL: Redis did not become ready"
|
||||
stage_fail "infra_deploy"
|
||||
PIPELINE_EXIT_CODE=2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
log "Waiting for minio readiness ..."
|
||||
if ! kubectl wait --for=condition=ready pod -l app=minio -n "$NAMESPACE" --timeout=60s; then
|
||||
log "FATAL: MinIO did not become ready"
|
||||
stage_fail "infra_deploy"
|
||||
PIPELINE_EXIT_CODE=2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
log "Waiting for minio-bucket-init job ..."
|
||||
kubectl wait --for=condition=complete job/minio-bucket-init -n "$NAMESPACE" --timeout=60s || true
|
||||
|
||||
stage_end "infra_deploy" "ok"
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Stage: Seed data
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
stage_start "seed_data"
|
||||
|
||||
SEED_IMAGE="ghcr.io/celesrenata/stonks-oracle/query-api:${IMAGE_TAG}"
|
||||
|
||||
log "Seeding sandbox database ..."
|
||||
if ! kubectl run seed-sandbox \
|
||||
--image="$SEED_IMAGE" \
|
||||
--restart=Never \
|
||||
--rm \
|
||||
--attach \
|
||||
--namespace="$NAMESPACE" \
|
||||
--image-pull-policy=Always \
|
||||
--overrides='{
|
||||
"spec": {
|
||||
"imagePullSecrets": [{"name": "ghcr-credentials"}],
|
||||
"securityContext": {"runAsNonRoot": true, "runAsUser": 1000, "runAsGroup": 1000}
|
||||
}
|
||||
}' \
|
||||
--env="POSTGRES_HOST=postgres" \
|
||||
--env="POSTGRES_PORT=5432" \
|
||||
--env="POSTGRES_DB=stonks" \
|
||||
--env="POSTGRES_USER=stonks" \
|
||||
--env="POSTGRES_PASSWORD=inttest" \
|
||||
--env="MINIO_ENDPOINT=minio:9000" \
|
||||
--env="MINIO_SECURE=false" \
|
||||
--env="MINIO_ACCESS_KEY=minioadmin" \
|
||||
--env="MINIO_SECRET_KEY=minioadmin" \
|
||||
--command -- python -m tests.integration.seed_sandbox; then
|
||||
log "FATAL: Database seed failed"
|
||||
stage_fail "seed_data"
|
||||
PIPELINE_EXIT_CODE=2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
log "Seeding MinIO buckets ..."
|
||||
if ! kubectl run seed-minio \
|
||||
--image="$SEED_IMAGE" \
|
||||
--restart=Never \
|
||||
--rm \
|
||||
--attach \
|
||||
--namespace="$NAMESPACE" \
|
||||
--image-pull-policy=Always \
|
||||
--overrides='{
|
||||
"spec": {
|
||||
"imagePullSecrets": [{"name": "ghcr-credentials"}],
|
||||
"securityContext": {"runAsNonRoot": true, "runAsUser": 1000, "runAsGroup": 1000}
|
||||
}
|
||||
}' \
|
||||
--env="MINIO_ENDPOINT=minio:9000" \
|
||||
--env="MINIO_SECURE=false" \
|
||||
--env="MINIO_ACCESS_KEY=minioadmin" \
|
||||
--env="MINIO_SECRET_KEY=minioadmin" \
|
||||
--command -- python -m tests.integration.seed_minio; then
|
||||
log "FATAL: MinIO seed failed"
|
||||
stage_fail "seed_data"
|
||||
PIPELINE_EXIT_CODE=2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
stage_end "seed_data" "ok"
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Stage: Deploy services
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
stage_start "service_deploy"
|
||||
|
||||
log "Applying services manifest (image tag: $IMAGE_TAG) ..."
|
||||
envsubst < "$REPO_ROOT/infra/inttest/services.yaml" \
|
||||
| sed "s/:latest/:${IMAGE_TAG}/g" \
|
||||
| kubectl apply -n "$NAMESPACE" -f -
|
||||
|
||||
log "Waiting for all API services to become ready ..."
|
||||
if ! kubectl wait --for=condition=ready pod -l tier=api -n "$NAMESPACE" --timeout=120s; then
|
||||
log "FATAL: API services did not become ready"
|
||||
stage_fail "service_deploy"
|
||||
PIPELINE_EXIT_CODE=2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
stage_end "service_deploy" "ok"
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Stage: Run integration tests
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
stage_start "integration_tests"
|
||||
|
||||
log "Applying test runner job (image tag: $IMAGE_TAG) ..."
|
||||
envsubst < "$REPO_ROOT/infra/inttest/runner.yaml" \
|
||||
| sed "s/:latest/:${IMAGE_TAG}/g" \
|
||||
| kubectl apply -n "$NAMESPACE" -f -
|
||||
|
||||
log "Waiting for test runner to complete (timeout: 600s) ..."
|
||||
if kubectl wait --for=condition=complete job/inttest-runner -n "$NAMESPACE" --timeout=600s; then
|
||||
log "Test runner completed successfully"
|
||||
stage_end "integration_tests" "ok"
|
||||
else
|
||||
log "Test runner failed or timed out"
|
||||
# Check if the job failed vs timed out
|
||||
if kubectl wait --for=condition=failed job/inttest-runner -n "$NAMESPACE" --timeout=5s 2>/dev/null; then
|
||||
log "Test runner job reported failure"
|
||||
fi
|
||||
stage_fail "integration_tests"
|
||||
PIPELINE_EXIT_CODE=1
|
||||
fi
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Stage: Collect results
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
log "Collecting test results ..."
|
||||
|
||||
# Get the runner pod name
|
||||
RUNNER_POD=$(kubectl get pods -n "$NAMESPACE" -l app=inttest-runner -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
|
||||
|
||||
if [ -n "$RUNNER_POD" ]; then
|
||||
# Collect test logs
|
||||
log "Collecting test logs from $RUNNER_POD ..."
|
||||
kubectl logs "$RUNNER_POD" -n "$NAMESPACE" 2>/dev/null || true
|
||||
|
||||
# Try to copy profiling report
|
||||
PROFILING_TMP=$(mktemp /tmp/profiling-report-XXXXXX.json)
|
||||
if kubectl cp "$NAMESPACE/$RUNNER_POD:/tmp/profiling-report.json" "$PROFILING_TMP" 2>/dev/null; then
|
||||
log "Profiling report collected"
|
||||
PROFILING_JSON="$PROFILING_TMP"
|
||||
else
|
||||
log "No profiling report found (test may not have produced one)"
|
||||
rm -f "$PROFILING_TMP"
|
||||
fi
|
||||
|
||||
# Parse test counts from logs (pytest output format: "X passed, Y failed, Z errors")
|
||||
TEST_OUTPUT=$(kubectl logs "$RUNNER_POD" -n "$NAMESPACE" 2>/dev/null || true)
|
||||
if [ -n "$TEST_OUTPUT" ]; then
|
||||
# Extract counts from pytest summary line like "41 passed, 2 failed, 1 error"
|
||||
TESTS_PASSED=$(echo "$TEST_OUTPUT" | grep -oP '\d+(?= passed)' | tail -1 || echo "0")
|
||||
TESTS_FAILED=$(echo "$TEST_OUTPUT" | grep -oP '\d+(?= failed)' | tail -1 || echo "0")
|
||||
TESTS_ERRORS=$(echo "$TEST_OUTPUT" | grep -oP '\d+(?= error)' | tail -1 || echo "0")
|
||||
TESTS_PASSED=${TESTS_PASSED:-0}
|
||||
TESTS_FAILED=${TESTS_FAILED:-0}
|
||||
TESTS_ERRORS=${TESTS_ERRORS:-0}
|
||||
TESTS_TOTAL=$(( TESTS_PASSED + TESTS_FAILED + TESTS_ERRORS ))
|
||||
fi
|
||||
else
|
||||
log "Could not find runner pod — results unavailable"
|
||||
fi
|
||||
|
||||
# If tests had failures, ensure exit code reflects it
|
||||
if [ "$TESTS_FAILED" -gt 0 ] || [ "$TESTS_ERRORS" -gt 0 ]; then
|
||||
PIPELINE_EXIT_CODE=1
|
||||
fi
|
||||
|
||||
# Mark integration_tests stage if not already done
|
||||
if [ -z "${STAGE_STATUS[integration_tests]:-}" ]; then
|
||||
if [ "$PIPELINE_EXIT_CODE" -eq 0 ]; then
|
||||
stage_end "integration_tests" "ok"
|
||||
else
|
||||
stage_fail "integration_tests"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Summary
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
PIPELINE_END=$(date +%s)
|
||||
PIPELINE_DURATION=$(( PIPELINE_END - PIPELINE_START ))
|
||||
|
||||
echo ""
|
||||
log "═══════════════════════════════════════════════════"
|
||||
log " Pipeline Summary"
|
||||
log "═══════════════════════════════════════════════════"
|
||||
log " Namespace: $NAMESPACE"
|
||||
log " Image tag: $IMAGE_TAG"
|
||||
log " Duration: ${PIPELINE_DURATION}s"
|
||||
log " Tests: ${TESTS_PASSED} passed, ${TESTS_FAILED} failed, ${TESTS_ERRORS} errors"
|
||||
log " Exit code: $PIPELINE_EXIT_CODE"
|
||||
log "═══════════════════════════════════════════════════"
|
||||
echo ""
|
||||
|
||||
# Teardown + results writing handled by the EXIT trap
|
||||
exit "$PIPELINE_EXIT_CODE"
|
||||
Reference in New Issue
Block a user