phase 17: add backup/restore scripts — PostgreSQL + MinIO → NFS

2026-04-12 14:21:59 -07:00
parent 6e2f174b19
commit 3f5b4adcec
2 changed files with 234 additions and 0 deletions
@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+# Stonks Oracle Backup — PostgreSQL + MinIO → NFS
+# Usage: bash scripts/backup.sh
+set -euo pipefail
+
+NAMESPACE="stonks-oracle"
+NFS_SERVER="192.168.42.8"
+NFS_PATH="/volume1/Kubernetes/stonks"
+BACKUP_NAME="stonks-backup-$(date +%Y%m%d-%H%M%S)"
+JOB_NAME="stonks-backup"
+
+echo "=== Stonks Oracle Backup: ${BACKUP_NAME} ==="
+
+# Clean up any previous backup job
+kubectl delete job ${JOB_NAME} -n ${NAMESPACE} --ignore-not-found=true 2>/dev/null || true
+
+cat <<EOJOB | kubectl apply -f -
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ${JOB_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  ttlSecondsAfterFinished: 300
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      volumes:
+        - name: nfs-backup
+          nfs:
+            server: ${NFS_SERVER}
+            path: ${NFS_PATH}
+      containers:
+        - name: backup
+          image: alpine:3.20
+          volumeMounts:
+            - name: nfs-backup
+              mountPath: /backup
+          envFrom:
+            - configMapRef:
+                name: stonks-config
+            - secretRef:
+                name: stonks-core-secrets
+          env:
+            - name: MINIO_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: stonks-core-secrets
+                  key: MINIO_ACCESS_KEY
+            - name: MINIO_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: stonks-core-secrets
+                  key: MINIO_SECRET_KEY
+          command: ["sh", "-c"]
+          args:
+            - |
+              set -e
+              apk add --no-cache postgresql16-client curl ca-certificates
+
+              STAMP="${BACKUP_NAME}"
+              DIR="/backup/\${STAMP}"
+              mkdir -p "\${DIR}/minio"
+
+              echo "[1/3] Backing up PostgreSQL..."
+              PGPASSWORD="\${POSTGRES_PASSWORD}" pg_dump \
+                -h "\${POSTGRES_HOST}" -p "\${POSTGRES_PORT}" \
+                -U "\${POSTGRES_USER}" -d "\${POSTGRES_DB}" \
+                --no-owner --no-acl -Fc \
+                -f "\${DIR}/stonks.pgdump"
+              echo "  pg_dump done: \$(du -h "\${DIR}/stonks.pgdump" | cut -f1)"
+
+              echo "[2/3] Backing up MinIO buckets..."
+              # Install mc (MinIO client)
+              curl -sL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
+              chmod +x /usr/local/bin/mc
+              mc alias set backup "http://\${MINIO_ENDPOINT}" "\${MINIO_ACCESS_KEY}" "\${MINIO_SECRET_KEY}" --api S3v4 2>/dev/null
+
+              for bucket in stonks-raw-market stonks-raw-news stonks-raw-filings stonks-normalized stonks-llm-prompts stonks-llm-results stonks-lakehouse stonks-audit; do
+                if mc ls "backup/\${bucket}" >/dev/null 2>&1; then
+                  echo "  Mirroring \${bucket}..."
+                  mc mirror --quiet "backup/\${bucket}" "\${DIR}/minio/\${bucket}/" 2>/dev/null || echo "  (empty or error)"
+                else
+                  echo "  Bucket \${bucket} not found, skipping"
+                fi
+              done
+
+              echo "[3/3] Writing manifest..."
+              cat > "\${DIR}/manifest.json" <<EOF
+              {
+                "backup_name": "\${STAMP}",
+                "created_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
+                "pg_dump": "stonks.pgdump",
+                "minio_buckets": ["stonks-raw-market","stonks-raw-news","stonks-raw-filings","stonks-normalized","stonks-llm-prompts","stonks-llm-results","stonks-lakehouse","stonks-audit"]
+              }
+              EOF
+
+              # Symlink latest
+              ln -sfn "\${STAMP}" /backup/latest
+              echo "=== Backup complete: \${DIR} ==="
+              ls -lh "\${DIR}/"
+EOJOB
+
+echo "Waiting for backup job to complete..."
+kubectl wait --for=condition=complete job/${JOB_NAME} -n ${NAMESPACE} --timeout=600s 2>&1 || {
+  echo "Backup job failed or timed out. Logs:"
+  kubectl logs job/${JOB_NAME} -n ${NAMESPACE} 2>&1 | tail -20
+  exit 1
+}
+
+echo ""
+kubectl logs job/${JOB_NAME} -n ${NAMESPACE} 2>&1 | tail -10
+echo ""
+echo "=== Backup ${BACKUP_NAME} complete ==="
@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+# Stonks Oracle Restore — NFS → PostgreSQL + MinIO
+# Usage: bash scripts/restore.sh [backup_name]
+#   If no backup_name given, restores from "latest" symlink.
+set -euo pipefail
+
+NAMESPACE="stonks-oracle"
+NFS_SERVER="192.168.42.8"
+NFS_PATH="/volume1/Kubernetes/stonks"
+BACKUP_NAME="${1:-latest}"
+JOB_NAME="stonks-restore"
+
+echo "=== Stonks Oracle Restore: ${BACKUP_NAME} ==="
+echo "WARNING: This will DROP and recreate the stonks database."
+echo "Press Ctrl+C within 5 seconds to abort..."
+sleep 5
+
+# Clean up any previous restore job
+kubectl delete job ${JOB_NAME} -n ${NAMESPACE} --ignore-not-found=true 2>/dev/null || true
+
+cat <<EOJOB | kubectl apply -f -
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ${JOB_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  ttlSecondsAfterFinished: 300
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      volumes:
+        - name: nfs-backup
+          nfs:
+            server: ${NFS_SERVER}
+            path: ${NFS_PATH}
+      containers:
+        - name: restore
+          image: alpine:3.20
+          volumeMounts:
+            - name: nfs-backup
+              mountPath: /backup
+          envFrom:
+            - configMapRef:
+                name: stonks-config
+            - secretRef:
+                name: stonks-core-secrets
+          env:
+            - name: MINIO_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: stonks-core-secrets
+                  key: MINIO_ACCESS_KEY
+            - name: MINIO_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: stonks-core-secrets
+                  key: MINIO_SECRET_KEY
+          command: ["sh", "-c"]
+          args:
+            - |
+              set -e
+              apk add --no-cache postgresql16-client curl ca-certificates
+
+              DIR="/backup/${BACKUP_NAME}"
+              if [ ! -f "\${DIR}/stonks.pgdump" ]; then
+                echo "ERROR: No backup found at \${DIR}/stonks.pgdump"
+                ls -la /backup/ 2>/dev/null || true
+                exit 1
+              fi
+
+              echo "Restoring from: \${DIR}"
+              cat "\${DIR}/manifest.json" 2>/dev/null || true
+
+              echo ""
+              echo "[1/3] Restoring PostgreSQL..."
+              # Drop and recreate all tables by restoring with --clean
+              PGPASSWORD="\${POSTGRES_PASSWORD}" pg_restore \
+                -h "\${POSTGRES_HOST}" -p "\${POSTGRES_PORT}" \
+                -U "\${POSTGRES_USER}" -d "\${POSTGRES_DB}" \
+                --clean --if-exists --no-owner --no-acl \
+                "\${DIR}/stonks.pgdump" 2>&1 || echo "  (some drop errors are expected on first restore)"
+              echo "  PostgreSQL restored"
+
+              echo "[2/3] Restoring MinIO buckets..."
+              curl -sL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
+              chmod +x /usr/local/bin/mc
+              mc alias set backup "http://\${MINIO_ENDPOINT}" "\${MINIO_ACCESS_KEY}" "\${MINIO_SECRET_KEY}" --api S3v4 2>/dev/null
+
+              for bucket in stonks-raw-market stonks-raw-news stonks-raw-filings stonks-normalized stonks-llm-prompts stonks-llm-results stonks-lakehouse stonks-audit; do
+                if [ -d "\${DIR}/minio/\${bucket}" ]; then
+                  echo "  Restoring \${bucket}..."
+                  mc mb --ignore-existing "backup/\${bucket}" 2>/dev/null || true
+                  mc mirror --overwrite --quiet "\${DIR}/minio/\${bucket}/" "backup/\${bucket}" 2>/dev/null || echo "  (empty or error)"
+                fi
+              done
+
+              echo "[3/3] Verifying..."
+              PGPASSWORD="\${POSTGRES_PASSWORD}" psql \
+                -h "\${POSTGRES_HOST}" -p "\${POSTGRES_PORT}" \
+                -U "\${POSTGRES_USER}" -d "\${POSTGRES_DB}" \
+                -c "SELECT 'companies=' || count(*) FROM companies UNION ALL SELECT 'documents=' || count(*) FROM documents UNION ALL SELECT 'intelligence=' || count(*) FROM document_intelligence;"
+
+              echo "=== Restore complete ==="
+EOJOB
+
+echo "Waiting for restore job to complete..."
+kubectl wait --for=condition=complete job/${JOB_NAME} -n ${NAMESPACE} --timeout=600s 2>&1 || {
+  echo "Restore job failed or timed out. Logs:"
+  kubectl logs job/${JOB_NAME} -n ${NAMESPACE} 2>&1 | tail -20
+  exit 1
+}
+
+echo ""
+kubectl logs job/${JOB_NAME} -n ${NAMESPACE} 2>&1 | tail -15
+echo ""
+echo "=== Restore from ${BACKUP_NAME} complete ==="
+echo "You may want to restart services: kubectl rollout restart deployment -n ${NAMESPACE} --all"