stonks-oracle/scripts/backup.sh

#!/usr/bin/env bash
# Stonks Oracle Backup — PostgreSQL + MinIO → NFS
# Usage: bash scripts/backup.sh
set -euo pipefail

NAMESPACE="stonks-oracle"
NFS_SERVER="192.168.42.8"
NFS_PATH="/volume1/Kubernetes/stonks"
BACKUP_NAME="stonks-backup-$(date +%Y%m%d-%H%M%S)"
JOB_NAME="stonks-backup"

echo "=== Stonks Oracle Backup: ${BACKUP_NAME} ==="

# Clean up any previous backup job
kubectl delete job ${JOB_NAME} -n ${NAMESPACE} --ignore-not-found=true 2>/dev/null || true

cat <<EOJOB | kubectl apply -f -
apiVersion: batch/v1
kind: Job
metadata:
  name: ${JOB_NAME}
  namespace: ${NAMESPACE}
spec:
  ttlSecondsAfterFinished: 300
  backoffLimit: 0
  template:
    spec:
      restartPolicy: Never
      volumes:
        - name: nfs-backup
          nfs:
            server: ${NFS_SERVER}
            path: ${NFS_PATH}
      containers:
        - name: backup
          image: postgres:18-alpine
          volumeMounts:
            - name: nfs-backup
              mountPath: /backup
          envFrom:
            - configMapRef:
                name: stonks-config
            - secretRef:
                name: stonks-core-secrets
          env:
            - name: MINIO_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: stonks-core-secrets
                  key: MINIO_ACCESS_KEY
            - name: MINIO_SECRET_KEY
              valueFrom:
                secretKeyRef:
                  name: stonks-core-secrets
                  key: MINIO_SECRET_KEY
          command: ["sh", "-c"]
          args:
            - |
              set -e
              apk add --no-cache curl ca-certificates

              STAMP="${BACKUP_NAME}"
              DIR="/backup/\${STAMP}"
              mkdir -p "\${DIR}/minio"

              echo "[1/3] Backing up PostgreSQL..."
              PGPASSWORD="\${POSTGRES_PASSWORD}" pg_dump \
                -h "\${POSTGRES_HOST}" -p "\${POSTGRES_PORT}" \
                -U "\${POSTGRES_USER}" -d "\${POSTGRES_DB}" \
                --no-owner --no-acl -Fc \
                -f "\${DIR}/stonks.pgdump"
              echo "  pg_dump done: \$(du -h "\${DIR}/stonks.pgdump" | cut -f1)"

              echo "[2/3] Backing up MinIO buckets..."
              # Install mc (MinIO client)
              curl -sL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
              chmod +x /usr/local/bin/mc
              mc alias set backup "http://\${MINIO_ENDPOINT}" "\${MINIO_ACCESS_KEY}" "\${MINIO_SECRET_KEY}" --api S3v4 2>/dev/null

              for bucket in stonks-raw-market stonks-raw-news stonks-raw-filings stonks-normalized stonks-llm-prompts stonks-llm-results stonks-lakehouse stonks-audit; do
                if mc ls "backup/\${bucket}" >/dev/null 2>&1; then
                  COUNT=\$(mc ls --recursive "backup/\${bucket}" 2>/dev/null | wc -l)
                  if [ "\${COUNT}" -gt 0 ]; then
                    echo "  \${bucket}: \${COUNT} objects, mirroring..."
                    mc mirror "backup/\${bucket}" "\${DIR}/minio/\${bucket}/" 2>&1 | tail -1
                    SIZE=\$(du -sh "\${DIR}/minio/\${bucket}/" 2>/dev/null | cut -f1)
                    echo "  \${bucket}: done (\${SIZE})"
                  else
                    echo "  \${bucket}: empty, skipping"
                  fi
                else
                  echo "  \${bucket}: not found, skipping"
                fi
              done

              echo ""
              echo "=== Backup Summary ==="
              echo "  PostgreSQL: \$(du -h "\${DIR}/stonks.pgdump" | cut -f1)"
              MINIO_TOTAL=\$(du -sh "\${DIR}/minio/" 2>/dev/null | cut -f1)
              echo "  MinIO total: \${MINIO_TOTAL}"
              for bucket in stonks-raw-market stonks-raw-news stonks-raw-filings stonks-normalized stonks-llm-prompts stonks-llm-results stonks-lakehouse stonks-audit; do
                if [ -d "\${DIR}/minio/\${bucket}" ]; then
                  BSIZE=\$(du -sh "\${DIR}/minio/\${bucket}/" 2>/dev/null | cut -f1)
                  BCOUNT=\$(find "\${DIR}/minio/\${bucket}/" -type f 2>/dev/null | wc -l)
                  echo "    \${bucket}: \${BSIZE} (\${BCOUNT} files)"
                fi
              done
              TOTAL=\$(du -sh "\${DIR}/" 2>/dev/null | cut -f1)
              echo "  Total backup size: \${TOTAL}"

              echo "[3/3] Writing manifest..."
              cat > "\${DIR}/manifest.json" <<EOF
              {
                "backup_name": "\${STAMP}",
                "created_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
                "pg_dump": "stonks.pgdump",
                "minio_buckets": ["stonks-raw-market","stonks-raw-news","stonks-raw-filings","stonks-normalized","stonks-llm-prompts","stonks-llm-results","stonks-lakehouse","stonks-audit"]
              }
              EOF

              # Symlink latest
              ln -sfn "\${STAMP}" /backup/latest
              echo "=== Backup complete: \${DIR} ==="
              ls -lh "\${DIR}/"
EOJOB

echo "Waiting for backup job to complete..."
kubectl wait --for=condition=complete job/${JOB_NAME} -n ${NAMESPACE} --timeout=600s 2>&1 || {
  echo "Backup job failed or timed out. Logs:"
  kubectl logs job/${JOB_NAME} -n ${NAMESPACE} 2>&1 | tail -20
  exit 1
}

echo ""
kubectl logs job/${JOB_NAME} -n ${NAMESPACE} 2>&1 | tail -10
echo ""
echo "=== Backup ${BACKUP_NAME} complete ==="