From 00a6485e70bdb1f9b98210fae2d6a4dda76ddbbe Mon Sep 17 00:00:00 2001 From: Celes Renata Date: Sun, 19 Apr 2026 05:14:13 +0000 Subject: [PATCH] ci: sync esnixi changes - CA download, dockerhub auth, local-path storage, proxy exclusions, pod annotations --- pipelines/argocd/values.yaml | 4 + pipelines/fix-webhook.sh | 31 +++++++ pipelines/gitea/setup.sh | 2 +- pipelines/home.crt | 24 ++++++ pipelines/kargo/values.yaml | 2 + pipelines/runmefirst.sh | 96 ++++++++++++++-------- pipelines/runmelast.sh | 2 +- pipelines/woodpecker/kyverno-proxy-ca.yaml | 10 +++ pipelines/woodpecker/values.yaml | 7 +- 9 files changed, 143 insertions(+), 35 deletions(-) create mode 100644 pipelines/fix-webhook.sh create mode 100644 pipelines/home.crt diff --git a/pipelines/argocd/values.yaml b/pipelines/argocd/values.yaml index 62b071f..61f5592 100644 --- a/pipelines/argocd/values.yaml +++ b/pipelines/argocd/values.yaml @@ -2,6 +2,10 @@ # Chart: argo/argo-cd # Namespace: argocd +global: + podAnnotations: + celestium.life/inject-ca: "true" + # Disable dex (not needed) dex: enabled: false diff --git a/pipelines/fix-webhook.sh b/pipelines/fix-webhook.sh new file mode 100644 index 0000000..51c8d32 --- /dev/null +++ b/pipelines/fix-webhook.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +# fix-webhook.sh — Fix Woodpecker webhook to use internal cluster URL +# Run this after activating the repo in the Woodpecker UI + +GITEA_AUTH="Authorization: Basic $(echo -n 'admin:St0nks0racl3!' | base64)" +GITEA_API="http://10.1.1.12:30300/api/v1" + +HOOK_ID=$(curl -s -H "$GITEA_AUTH" "$GITEA_API/repos/admin/stonks-oracle/hooks" | python3 -c ' +import sys, json +hooks = json.loads(sys.stdin.read()) +for h in hooks: + if "woodpecker" in h["config"]["url"] or "hook" in h["config"]["url"]: + print(h["id"]) + break +' 2>/dev/null || echo "") + +if [ -z "$HOOK_ID" ]; then + echo "No Woodpecker webhook found. Activate the repo in Woodpecker first." + exit 1 +fi + +TOKEN=$(curl -s -H "$GITEA_AUTH" "$GITEA_API/repos/admin/stonks-oracle/hooks/$HOOK_ID" | \ + python3 -c 'import sys,json; print(json.loads(sys.stdin.read())["config"]["url"].split("access_token=")[1])') + +curl -s -X PATCH -H "$GITEA_AUTH" -H "Content-Type: application/json" \ + "$GITEA_API/repos/admin/stonks-oracle/hooks/$HOOK_ID" \ + -d "{\"config\":{\"url\":\"http://woodpecker-server.woodpecker.svc.cluster.local:80/api/hook?access_token=${TOKEN}\",\"content_type\":\"json\"},\"active\":true}" > /dev/null + +echo "✓ Webhook fixed to internal URL" diff --git a/pipelines/gitea/setup.sh b/pipelines/gitea/setup.sh index ce8f744..643a57a 100755 --- a/pipelines/gitea/setup.sh +++ b/pipelines/gitea/setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail # pipelines/gitea/setup.sh — Automate Gitea initial setup via REST API diff --git a/pipelines/home.crt b/pipelines/home.crt new file mode 100644 index 0000000..f905741 --- /dev/null +++ b/pipelines/home.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIIEDDCCAvSgAwIBAgIBADANBgkqhkiG9w0BAQsFADBiMRQwEgYDVQQDEwtpbnRl +cm5hbC1jYTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xDzANBgNV +BAcTBlJlbnRvbjEXMBUGA1UEChMOQ2VsZXN0aXVtLmxpZmUwHhcNMjAxMDMwMDgz +MTI1WhcNMzAxMDI4MDgzMTI1WjBiMRQwEgYDVQQDEwtpbnRlcm5hbC1jYTELMAkG +A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xDzANBgNVBAcTBlJlbnRvbjEX +MBUGA1UEChMOQ2VsZXN0aXVtLmxpZmUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQC5GNM9gr9qlaAsThoEf7h3nSrBoG8Rvhp63cfp5P1pcvFsSk4TY8St +Gqkd1rZI0GIe9hxMGd2ky7oT/4zgNtDzGWaY8GAOnWYxoMb/hzjZOh3Ft2yzbQRQ +vamcGREEHqaRa+WQuzMI43+heRKL6iZdljGiOEen6azh0jNN41xX1FqU0t7tH76H +5S16/otOT8Z2hjBlkN+DMZ1nBefGAjJMjtXTLFDPI6jFJcQdMRN6yOjk2LJ8FCTP +NOEvDYI06DXiG5X+AOm+VJ/1CG7bmCbzaEML2pPMY0sndrIpkcTTJY8v4Z+EX4Km +cVwog0OHQ7prt3IHA5nqgGpJdzi6KUKlAgMBAAGjgcwwgckwHQYDVR0OBBYEFC9b +adjJsJxislghGmqyppofq+TVMIGMBgNVHSMEgYQwgYGAFC9badjJsJxislghGmqy +ppofq+TVoWakZDBiMRQwEgYDVQQDEwtpbnRlcm5hbC1jYTELMAkGA1UEBhMCVVMx +EzARBgNVBAgTCldhc2hpbmd0b24xDzANBgNVBAcTBlJlbnRvbjEXMBUGA1UEChMO +Q2VsZXN0aXVtLmxpZmWCAQAwDAYDVR0TBAUwAwEB/zALBgNVHQ8EBAMCAQYwDQYJ +KoZIhvcNAQELBQADggEBAGtnVZX2UvedrrAak7tdTGfbqjQtR+3vfxV2QLrQ12/G +tsoQuYWNyfvzEvzs3OQZKSHKy9Gfy9XjpCDGKFMBVDr/IfP8tzbvi4bKhaydKV2C +PwHdz5vXgdqxUKUlebNYVzZ1k0vvx+cci1bT316odleIX0AjOfAU8Vn/wtkfwB2g +QxUV11S/WX9qN6wWNMhCgdloSBvUm6Nmce+EW9R69/sWlI99LnjnP1VGRRE3SDo+ +3EH5roEKYHuL3XWi+WUL10yYbolciKk2r7fVrXOnGHY2V9NYSgYdPDfAwkw+UKid +CXWvpQmv6HxIZUDkw5IkVAqfuwBEAHfmnuo6WT9Lu8c= +-----END CERTIFICATE----- \ No newline at end of file diff --git a/pipelines/kargo/values.yaml b/pipelines/kargo/values.yaml index 80709a8..903c18a 100644 --- a/pipelines/kargo/values.yaml +++ b/pipelines/kargo/values.yaml @@ -5,6 +5,8 @@ api: enabled: true host: stonks-kargo.celestium.life + podAnnotations: + celestium.life/inject-ca: "true" tls: enabled: false ingress: diff --git a/pipelines/runmefirst.sh b/pipelines/runmefirst.sh index 4950f0b..70ccf73 100755 --- a/pipelines/runmefirst.sh +++ b/pipelines/runmefirst.sh @@ -1,14 +1,16 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail # runmefirst.sh — Full CI/CD pipeline infrastructure install # Installs: Gitea config → Woodpecker CI → ArgoCD → Kargo -# Tears down ARC first (if present) # Persists state on NFS volumes at nfs://192.168.42.8:/volume1/Kubernetes/pipelines SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$SCRIPT_DIR" +GITEA_AUTH="Authorization: Basic $(echo -n 'admin:St0nks0racl3!' | base64)" +GITEA_API="http://10.1.1.12:30300/api/v1" + # ------------------------------------------------------- # 1. Create namespaces # ------------------------------------------------------- @@ -20,10 +22,11 @@ done echo "" # ------------------------------------------------------- -# 1b. Ensure proxy-ca-cert ConfigMap exists in pipeline namespaces +# 2. Ensure proxy-ca-cert ConfigMap + Kyverno policies # ------------------------------------------------------- -echo "--- Step 1b: Ensuring proxy CA cert ConfigMap ---" -CA_CERT_PATH="/home/celes/nixos-goblin-1-2-3/home.crt" +echo "--- Step 2: Proxy CA cert and Kyverno policies ---" +CA_CERT_PATH="${SCRIPT_DIR}/home.crt" +curl -sf http://192.168.42.1/home.crt -o "$CA_CERT_PATH" for ns in woodpecker argocd kargo; do if ! kubectl get configmap proxy-ca-cert -n "$ns" > /dev/null 2>&1; then kubectl create configmap proxy-ca-cert --from-file=ca.crt="$CA_CERT_PATH" -n "$ns" @@ -32,12 +35,23 @@ for ns in woodpecker argocd kargo; do echo " ✓ proxy-ca-cert already exists in $ns" fi done +# Apply Kyverno policy BEFORE Woodpecker install so pods get injected on creation +kubectl apply -f woodpecker/kyverno-proxy-ca.yaml +echo " ✓ Kyverno woodpecker-proxy-ca policy applied" +# Docker Hub auth for builder pods (avoids rate limits) +if ! kubectl get secret dockerhub-config -n woodpecker > /dev/null 2>&1; then + kubectl create secret generic dockerhub-config -n woodpecker \ + --from-literal=config.json='{"auths":{"https://index.docker.io/v1/":{"auth":"'"$(echo -n 'celesrenata:dckr_pat_rDJs5PbzGll_jyFyL9_NGEk_bJI' | base64)"'"}}}' + echo " ✓ dockerhub-config secret created" +else + echo " ✓ dockerhub-config secret already exists" +fi echo "" # ------------------------------------------------------- -# 2. Apply NFS PersistentVolumes +# 3. Apply NFS PersistentVolumes # ------------------------------------------------------- -echo "--- Step 2: Applying NFS PersistentVolumes ---" +echo "--- Step 3: Applying NFS PersistentVolumes ---" kubectl apply -f pvs/argocd-pv.yaml kubectl apply -f pvs/kargo-pv.yaml kubectl apply -f pvs/woodpecker-pv.yaml @@ -45,45 +59,39 @@ echo " ✓ PVs applied" echo "" # ------------------------------------------------------- -# 3. Configure Gitea (admin user, repo, webhook config) +# 4. Configure Gitea (admin user, repo, webhook config) # ------------------------------------------------------- -echo "--- Step 3: Configuring Gitea ---" +echo "--- Step 4: Configuring Gitea ---" bash gitea/setup.sh echo " ✓ Gitea configured" -# Ensure Gitea allows webhook delivery to local/cluster addresses GITEA_POD=$(kubectl get pods -n git-server -l app=gitea -o jsonpath='{.items[0].metadata.name}') if ! kubectl exec -n git-server "$GITEA_POD" -- grep -q '\[webhook\]' /data/gitea/conf/app.ini 2>/dev/null; then kubectl exec -n git-server "$GITEA_POD" -- sh -c 'printf "\n[webhook]\nALLOWED_HOST_LIST = *\nSKIP_TLS_VERIFY = true\n" >> /data/gitea/conf/app.ini' kubectl rollout restart deployment/gitea -n git-server kubectl rollout status deployment/gitea -n git-server --timeout=60s - echo " ✓ Gitea webhook config added (ALLOWED_HOST_LIST=*)" + echo " ✓ Gitea webhook config added" else echo " ✓ Gitea webhook config already present" fi echo "" # ------------------------------------------------------- -# 4. Install Woodpecker CI via Helm +# 5. Install Woodpecker CI via Helm # ------------------------------------------------------- -echo "--- Step 4: Installing Woodpecker CI ---" +echo "--- Step 5: Installing Woodpecker CI ---" -# Check if Woodpecker is already installed (upgrade vs fresh install) WOODPECKER_EXISTS=$(helm list -n woodpecker -q 2>/dev/null | grep -c woodpecker || true) if [ "${WOODPECKER_EXISTS:-0}" -gt 0 ]; then - # Upgrade — don't touch OAuth2 credentials, Woodpecker DB already has them echo " Woodpecker already installed — upgrading (preserving OAuth2 grants)..." helm upgrade woodpecker oci://ghcr.io/woodpecker-ci/helm/woodpecker \ --namespace woodpecker \ --values woodpecker/values.yaml \ --wait --timeout 5m else - # Fresh install — need fresh OAuth2 credentials from Gitea - echo " Fresh Woodpecker install — creating fresh OAuth2 app..." - # Delete any existing OAuth2 app in Gitea (stale from previous install) - GITEA_AUTH="Authorization: Basic $(echo -n 'admin:St0nks0racl3!' | base64)" - GITEA_API="http://10.1.1.12:30300/api/v1" + echo " Fresh Woodpecker install..." + # Delete stale OAuth2 app in Gitea (if any) EXISTING_APP_ID=$(curl -s -H "$GITEA_AUTH" "$GITEA_API/user/applications/oauth2" | python3 -c ' import sys, json for app in json.loads(sys.stdin.read()): @@ -102,6 +110,7 @@ for app in json.loads(sys.stdin.read()): GITEA_CLIENT_ID=$(echo "$OAUTH2_RESP" | python3 -c "import sys,json; print(json.load(sys.stdin)['client_id'])") GITEA_CLIENT_SECRET=$(echo "$OAUTH2_RESP" | python3 -c "import sys,json; print(json.load(sys.stdin)['client_secret'])") echo " ✓ OAuth2 app created (client_id: $GITEA_CLIENT_ID)" + helm install woodpecker oci://ghcr.io/woodpecker-ci/helm/woodpecker \ --namespace woodpecker \ --values woodpecker/values.yaml \ @@ -109,16 +118,44 @@ for app in json.loads(sys.stdin.read()): --set server.env.WOODPECKER_GITEA_SECRET="${GITEA_CLIENT_SECRET}" \ --wait --timeout 5m fi -echo " ✓ Woodpecker CI installed" -echo "" + +# Apply agent RBAC (grants cluster-admin to default + woodpecker-agent SAs) +kubectl apply -f woodpecker/agent-rbac.yaml +echo " ✓ Woodpecker CI installed + RBAC applied" # ------------------------------------------------------- -# 5. Apply Woodpecker agent RBAC +# 5b. Activate repo in Woodpecker (if fresh install) # ------------------------------------------------------- -echo "--- Step 5: Applying Woodpecker agent RBAC and Kyverno policy ---" -kubectl apply -f woodpecker/agent-rbac.yaml -kubectl apply -f woodpecker/kyverno-proxy-ca.yaml -echo " ✓ Agent RBAC and Kyverno proxy CA policy applied" +if [ "${WOODPECKER_EXISTS:-0}" -eq 0 ]; then + echo " Activating repo in Woodpecker..." + # Wait for server to be ready + kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=server -n woodpecker --timeout=60s > /dev/null 2>&1 + + # Port-forward to Woodpecker API + kubectl port-forward -n woodpecker svc/woodpecker-server 18080:80 > /dev/null 2>&1 & + PF_PID=$! + sleep 5 + + # Login via OAuth2 to get a user token — use the Gitea token approach + # Create a Gitea personal access token for API bootstrap + GITEA_TOKEN_RESP=$(curl -s -X POST "$GITEA_API/users/admin/tokens" \ + -H "$GITEA_AUTH" -H "Content-Type: application/json" \ + -d '{"name":"woodpecker-bootstrap-'"$(date +%s)"'","scopes":["all"]}') + GITEA_PAT=$(echo "$GITEA_TOKEN_RESP" | python3 -c "import sys,json; print(json.load(sys.stdin)['sha1'])") + + # Use Woodpecker's OAuth2 flow to get a session — this is complex via CLI + # Instead, just tell the user to activate via UI on first install + echo " ⚠ First install: please log in to https://stonks-ci.celestium.life" + echo " 1. Activate the admin/stonks-oracle repo" + echo " 2. Mark it as trusted (Settings → General → Trusted)" + echo " 3. Add 'github_ssh_key' secret (Settings → Secrets)" + + kill $PF_PID 2>/dev/null || true + wait $PF_PID 2>/dev/null || true + + # Fix webhook to internal URL after user activates + echo " After activating, run: bash pipelines/fix-webhook.sh" +fi echo "" # ------------------------------------------------------- @@ -127,7 +164,6 @@ echo "" echo "--- Step 6: Installing ArgoCD ---" ARGOCD_EXISTS=$(helm list -n argocd -q 2>/dev/null | grep -c argocd || true) if [ "${ARGOCD_EXISTS:-0}" -eq 0 ]; then - # Fresh install — clean up leftover CRDs and SAs kubectl delete crd applications.argoproj.io applicationsets.argoproj.io appprojects.argoproj.io \ --ignore-not-found --timeout=30s > /dev/null 2>&1 || true kubectl delete sa --all -n argocd --ignore-not-found --timeout=10s > /dev/null 2>&1 || true @@ -142,7 +178,6 @@ helm upgrade --install argocd argo/argo-cd \ --wait --timeout 5m echo " ✓ ArgoCD installed" -# Apply repo secret and Applications kubectl apply -f argocd/repo-secret.yaml kubectl apply -f argocd/apps/stonks-beta.yaml kubectl apply -f argocd/apps/stonks-paper.yaml @@ -156,7 +191,6 @@ echo "" echo "--- Step 7: Installing Kargo ---" KARGO_EXISTS=$(helm list -n kargo -q 2>/dev/null | grep -c kargo || true) if [ "${KARGO_EXISTS:-0}" -eq 0 ]; then - # Fresh install — clean up leftover CRDs and SAs from previous installs kubectl delete crd freights.kargo.akuity.io projects.kargo.akuity.io stages.kargo.akuity.io \ warehouses.kargo.akuity.io promotions.kargo.akuity.io promotiontasks.kargo.akuity.io \ clusterpromotiontasks.kargo.akuity.io projectconfigs.kargo.akuity.io \ @@ -171,7 +205,6 @@ helm upgrade --install kargo oci://ghcr.io/akuity/kargo-charts/kargo \ --timeout 5m || true # Kargo chart bug: controller deployment references SA 'kargo-controller' but chart doesn't create it kubectl create serviceaccount kargo-controller -n kargo 2>/dev/null || true -# Wait for controller to stabilize echo " Waiting for kargo-controller..." for i in $(seq 1 24); do if kubectl get pods -n kargo -l app.kubernetes.io/component=controller -o jsonpath='{.items[0].status.containerStatuses[0].ready}' 2>/dev/null | grep -q true; then @@ -182,7 +215,6 @@ for i in $(seq 1 24); do done echo " ✓ Kargo installed" -# Apply Kargo resources kubectl apply -f kargo/project.yaml kubectl apply -f kargo/project-config.yaml kubectl apply -f kargo/warehouse.yaml diff --git a/pipelines/runmelast.sh b/pipelines/runmelast.sh index a2b8676..167cac0 100755 --- a/pipelines/runmelast.sh +++ b/pipelines/runmelast.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail # runmelast.sh — Pipeline infrastructure teardown diff --git a/pipelines/woodpecker/kyverno-proxy-ca.yaml b/pipelines/woodpecker/kyverno-proxy-ca.yaml index a660f16..8acdc66 100644 --- a/pipelines/woodpecker/kyverno-proxy-ca.yaml +++ b/pipelines/woodpecker/kyverno-proxy-ca.yaml @@ -16,6 +16,16 @@ spec: - Pod namespaces: - woodpecker + exclude: + any: + - resources: + selector: + matchLabels: + app.kubernetes.io/name: server + - resources: + selector: + matchLabels: + app.kubernetes.io/name: agent mutate: patchStrategicMerge: spec: diff --git a/pipelines/woodpecker/values.yaml b/pipelines/woodpecker/values.yaml index 46edda4..8f49f5d 100644 --- a/pipelines/woodpecker/values.yaml +++ b/pipelines/woodpecker/values.yaml @@ -48,7 +48,10 @@ agent: enabled: true replicaCount: 2 - # CA injection handled by woodpecker-proxy-ca Kyverno policy (matches all pods in namespace) + # Agents must NOT have proxy/CA injection — they communicate with server via gRPC + # and the proxy blocks port 9000. Builder pods get injection via Kyverno policy + # matching WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS. + podAnnotations: {} env: WOODPECKER_SERVER: "woodpecker-server:9000" @@ -56,4 +59,6 @@ agent: WOODPECKER_BACKEND_K8S_NAMESPACE: woodpecker WOODPECKER_BACKEND_K8S_VOLUME_SIZE: 5Gi WOODPECKER_BACKEND_K8S_STORAGE_RWX: "false" + WOODPECKER_BACKEND_K8S_STORAGE_CLASS: local-path + WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS: "celestium.life/inject-ca:true" WOODPECKER_MAX_WORKFLOWS: "16"