ci: fix runmelast hanging, add proxy CA injection, clean SA/CRD leftovers on redeploy

pipelines/runmefirst.sh

@@ -9,18 +9,6 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "$SCRIPT_DIR"
 
-# -------------------------------------------------------
-# 0. Tear down ARC infrastructure (if present)
-# -------------------------------------------------------
-echo "--- Step 0: Tearing down ARC infrastructure ---"
-helm uninstall arc-runner-set --namespace arc-system || true
-helm uninstall arc --namespace arc-system || true
-kubectl delete clusterrolebinding arc-runner-rbac --ignore-not-found
-kubectl delete pv pipeline-arc-pv --ignore-not-found
-kubectl delete namespace arc-system --ignore-not-found --wait=false
-echo "  ✓ ARC teardown complete"
-echo ""
-
 # -------------------------------------------------------
 # 1. Create namespaces
 # -------------------------------------------------------
@@ -31,6 +19,21 @@ for ns in woodpecker argocd kargo stonks-beta stonks-paper; do
 done
 echo ""
 
+# -------------------------------------------------------
+# 1b. Ensure proxy-ca-cert ConfigMap exists in pipeline namespaces
+# -------------------------------------------------------
+echo "--- Step 1b: Ensuring proxy CA cert ConfigMap ---"
+CA_CERT_PATH="/home/celes/nixos-goblin-1-2-3/home.crt"
+for ns in woodpecker argocd kargo; do
+  if ! kubectl get configmap proxy-ca-cert -n "$ns" > /dev/null 2>&1; then
+    kubectl create configmap proxy-ca-cert --from-file=ca.crt="$CA_CERT_PATH" -n "$ns"
+    echo "  ✓ proxy-ca-cert created in $ns"
+  else
+    echo "  ✓ proxy-ca-cert already exists in $ns"
+  fi
+done
+echo ""
+
 # -------------------------------------------------------
 # 2. Apply NFS PersistentVolumes
 # -------------------------------------------------------
@@ -87,6 +90,12 @@ echo ""
 # 6. Install ArgoCD via Helm
 # -------------------------------------------------------
 echo "--- Step 6: Installing ArgoCD ---"
+# Clean up leftover ArgoCD CRDs from previous installs (they have resource-policy: keep)
+kubectl delete crd applications.argoproj.io applicationsets.argoproj.io appprojects.argoproj.io \
+  --ignore-not-found > /dev/null 2>&1 || true
+kubectl delete sa --all -n argocd --ignore-not-found > /dev/null 2>&1 || true
+kubectl delete role --all -n argocd --ignore-not-found > /dev/null 2>&1 || true
+kubectl delete rolebinding --all -n argocd --ignore-not-found > /dev/null 2>&1 || true
 helm repo add argo https://argoproj.github.io/argo-helm || true
 helm repo update
 helm upgrade --install argocd argo/argo-cd \
@@ -107,6 +116,15 @@ echo ""
 # 7. Install Kargo via Helm
 # -------------------------------------------------------
 echo "--- Step 7: Installing Kargo ---"
+# Clean up leftover Kargo CRDs from previous installs (they have resource-policy: keep)
+kubectl delete crd freights.kargo.akuity.io projects.kargo.akuity.io stages.kargo.akuity.io \
+  warehouses.kargo.akuity.io promotions.kargo.akuity.io promotiontasks.kargo.akuity.io \
+  clusterpromotiontasks.kargo.akuity.io projectconfigs.kargo.akuity.io \
+  clusterconfigs.kargo.akuity.io --ignore-not-found > /dev/null 2>&1 || true
+# Also clean up any leftover SAs/roles from previous installs that block Helm
+kubectl delete sa --all -n kargo --ignore-not-found > /dev/null 2>&1 || true
+kubectl delete role --all -n kargo --ignore-not-found > /dev/null 2>&1 || true
+kubectl delete rolebinding --all -n kargo --ignore-not-found > /dev/null 2>&1 || true
 helm upgrade --install kargo oci://ghcr.io/akuity/kargo-charts/kargo \
   --namespace kargo \
   --values kargo/values.yaml \

pipelines/runmelast.sh

@@ -3,8 +3,7 @@ set -euo pipefail
 
 # runmelast.sh — Pipeline infrastructure teardown
 # Removes: Kargo → ArgoCD → Woodpecker (reverse install order)
-# Preserves: NFS PVs, NFS data, git-server namespace (Gitea + registry),
+# Preserves: namespaces, PVCs, TLS certs, NFS PVs, git-server, application namespaces
-#            stonks-oracle namespace, stonks-beta, stonks-paper
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "$SCRIPT_DIR"
@@ -12,11 +11,11 @@ cd "$SCRIPT_DIR"
 echo "=== Pipeline Infrastructure Teardown ==="
 echo ""
 echo "This will remove Kargo, ArgoCD, and Woodpecker CI."
-echo "Preserved: NFS PVs, NFS data, git-server (Gitea + registry), application namespaces."
+echo "Preserved: namespaces, PVCs, TLS certs, NFS PVs, git-server, application namespaces."
 echo ""
 
 # -------------------------------------------------------
-# 1. Remove Kargo resources + Helm release
+# 1. Remove Kargo resources + Helm release + CRDs
 # -------------------------------------------------------
 echo "--- Step 1: Removing Kargo ---"
 kubectl delete -f kargo/stages/live.yaml --ignore-not-found || true
@@ -27,11 +26,15 @@ kubectl delete -f kargo/warehouse.yaml --ignore-not-found || true
 kubectl delete -f kargo/project-config.yaml --ignore-not-found || true
 kubectl delete -f kargo/project.yaml --ignore-not-found || true
 helm uninstall kargo --namespace kargo || true
+kubectl delete crd freights.kargo.akuity.io projects.kargo.akuity.io stages.kargo.akuity.io \
+  warehouses.kargo.akuity.io promotions.kargo.akuity.io promotiontasks.kargo.akuity.io \
+  clusterpromotiontasks.kargo.akuity.io projectconfigs.kargo.akuity.io \
+  clusterconfigs.kargo.akuity.io --ignore-not-found > /dev/null 2>&1 || true
 echo "  ✓ Kargo removed"
 echo ""
 
 # -------------------------------------------------------
-# 2. Remove ArgoCD resources + Helm release
+# 2. Remove ArgoCD resources + Helm release + CRDs
 # -------------------------------------------------------
 echo "--- Step 2: Removing ArgoCD ---"
 kubectl delete -f argocd/apps/stonks-live.yaml --ignore-not-found || true
@@ -39,6 +42,8 @@ kubectl delete -f argocd/apps/stonks-paper.yaml --ignore-not-found || true
 kubectl delete -f argocd/apps/stonks-beta.yaml --ignore-not-found || true
 kubectl delete -f argocd/repo-secret.yaml --ignore-not-found || true
 helm uninstall argocd --namespace argocd || true
+kubectl delete crd applications.argoproj.io applicationsets.argoproj.io appprojects.argoproj.io \
+  --ignore-not-found > /dev/null 2>&1 || true
 echo "  ✓ ArgoCD removed"
 echo ""
 
@@ -51,27 +56,17 @@ helm uninstall woodpecker --namespace woodpecker || true
 echo "  ✓ Woodpecker CI removed"
 echo ""
 
-# -------------------------------------------------------
+# NOTE: Namespaces are intentionally NOT deleted.
-# 4. Delete namespaces (pipeline infra only)
+# PVCs, TLS certs, and proxy-ca-cert ConfigMaps survive for fast redeploy.
-# -------------------------------------------------------
+# runmefirst.sh uses idempotent creates (--dry-run=client | kubectl apply).
-echo "--- Step 4: Deleting pipeline namespaces ---"
-for ns in woodpecker argocd kargo; do
-  kubectl delete namespace "$ns" --ignore-not-found || true
-  echo "  ✓ namespace/$ns deleted"
-done
-echo ""
-
-# NOTE: The following are intentionally NOT deleted:
-# - NFS PersistentVolumes (pipeline-argocd-pv, pipeline-kargo-pv, pipeline-woodpecker-pv)
-# - NFS data at nfs://192.168.42.8:/volume1/Kubernetes/pipelines/
-# - git-server namespace (Gitea + local registry)
-# - stonks-oracle namespace (production workloads)
-# - stonks-beta namespace (beta workloads)
-# - stonks-paper namespace (paper trading workloads)
 
 echo "=== Pipeline Infrastructure Teardown Complete ==="
 echo ""
 echo "Preserved:"
-echo "  - NFS PVs and data (survives cluster rebuild)"
+echo "  - Namespaces (woodpecker, argocd, kargo)"
+echo "  - PVCs (Woodpecker DB, agent configs)"
+echo "  - TLS certificates (Let's Encrypt rate limited)"
+echo "  - NFS PVs and data"
+echo "  - proxy-ca-cert ConfigMaps"
 echo "  - git-server namespace (Gitea + registry)"
 echo "  - Application namespaces (stonks-oracle, stonks-beta, stonks-paper)"

pipelines/woodpecker/values.yaml

@@ -6,6 +6,9 @@
 server:
   enabled: true
 
+  podAnnotations:
+    celestium.life/inject-ca: "true"
+
   env:
     WOODPECKER_HOST: "https://stonks-ci.celestium.life"
     WOODPECKER_SERVER_ADDR: "0.0.0.0:8000"
@@ -33,9 +36,9 @@ server:
         hosts:
           - stonks-ci.celestium.life
     annotations:
-      cert-manager.io/cluster-issuer: ca-issuer
+      cert-manager.io/cluster-issuer: celestium-le-production
 
-  # NFS-backed persistent volume for SQLite database and build data
+  # Persistent volume for SQLite database and build data
   persistentVolume:
     enabled: true
     size: 5Gi
@@ -46,9 +49,13 @@ agent:
   enabled: true
   replicaCount: 2
 
+  # No proxy CA injection for agents — they only talk to the server internally
+  # Pipeline step pods spawned by the agent inherit the node's proxy config
+
   env:
     WOODPECKER_SERVER: "woodpecker-server:9000"
     WOODPECKER_BACKEND: kubernetes
     WOODPECKER_BACKEND_K8S_NAMESPACE: woodpecker
     WOODPECKER_BACKEND_K8S_VOLUME_SIZE: 5Gi
     WOODPECKER_BACKEND_K8S_STORAGE_RWX: "false"
+    WOODPECKER_MAX_WORKFLOWS: "16"