stonks-oracle/scripts/restore.sh

#!/usr/bin/env bash
# Stonks Oracle Restore — NFS → PostgreSQL + MinIO
# Usage: bash scripts/restore.sh [backup_name]
#   If no backup_name given, restores from "latest" symlink.
set -euo pipefail

NAMESPACE="stonks-oracle"
NFS_SERVER="192.168.42.8"
NFS_PATH="/volume1/Kubernetes/stonks"
BACKUP_NAME="${1:-latest}"
JOB_NAME="stonks-restore"

echo "=== Stonks Oracle Restore: ${BACKUP_NAME} ==="
echo "WARNING: This will DROP and recreate the stonks database."
echo "Press Ctrl+C within 5 seconds to abort..."
sleep 5

# Clean up any previous restore job
kubectl delete job ${JOB_NAME} -n ${NAMESPACE} --ignore-not-found=true 2>/dev/null || true

cat <<EOJOB | kubectl apply -f -
apiVersion: batch/v1
kind: Job
metadata:
  name: ${JOB_NAME}
  namespace: ${NAMESPACE}
spec:
  ttlSecondsAfterFinished: 300
  backoffLimit: 0
  template:
    spec:
      restartPolicy: Never
      volumes:
        - name: nfs-backup
          nfs:
            server: ${NFS_SERVER}
            path: ${NFS_PATH}
      containers:
        - name: restore
          image: alpine:3.20
          volumeMounts:
            - name: nfs-backup
              mountPath: /backup
          envFrom:
            - configMapRef:
                name: stonks-config
            - secretRef:
                name: stonks-core-secrets
          env:
            - name: MINIO_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: stonks-core-secrets
                  key: MINIO_ACCESS_KEY
            - name: MINIO_SECRET_KEY
              valueFrom:
                secretKeyRef:
                  name: stonks-core-secrets
                  key: MINIO_SECRET_KEY
          command: ["sh", "-c"]
          args:
            - |
              set -e
              apk add --no-cache postgresql16-client curl ca-certificates

              DIR="/backup/${BACKUP_NAME}"
              if [ ! -f "\${DIR}/stonks.pgdump" ]; then
                echo "ERROR: No backup found at \${DIR}/stonks.pgdump"
                ls -la /backup/ 2>/dev/null || true
                exit 1
              fi

              echo "Restoring from: \${DIR}"
              cat "\${DIR}/manifest.json" 2>/dev/null || true

              echo ""
              echo "[1/3] Restoring PostgreSQL..."
              # Drop and recreate all tables by restoring with --clean
              PGPASSWORD="\${POSTGRES_PASSWORD}" pg_restore \
                -h "\${POSTGRES_HOST}" -p "\${POSTGRES_PORT}" \
                -U "\${POSTGRES_USER}" -d "\${POSTGRES_DB}" \
                --clean --if-exists --no-owner --no-acl \
                "\${DIR}/stonks.pgdump" 2>&1 || echo "  (some drop errors are expected on first restore)"
              echo "  PostgreSQL restored"

              echo "[2/3] Restoring MinIO buckets..."
              curl -sL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
              chmod +x /usr/local/bin/mc
              mc alias set backup "http://\${MINIO_ENDPOINT}" "\${MINIO_ACCESS_KEY}" "\${MINIO_SECRET_KEY}" --api S3v4 2>/dev/null

              for bucket in stonks-raw-market stonks-raw-news stonks-raw-filings stonks-normalized stonks-llm-prompts stonks-llm-results stonks-lakehouse stonks-audit; do
                if [ -d "\${DIR}/minio/\${bucket}" ]; then
                  echo "  Restoring \${bucket}..."
                  mc mb --ignore-existing "backup/\${bucket}" 2>/dev/null || true
                  mc mirror --overwrite --quiet "\${DIR}/minio/\${bucket}/" "backup/\${bucket}" 2>/dev/null || echo "  (empty or error)"
                fi
              done

              echo "[3/3] Verifying..."
              PGPASSWORD="\${POSTGRES_PASSWORD}" psql \
                -h "\${POSTGRES_HOST}" -p "\${POSTGRES_PORT}" \
                -U "\${POSTGRES_USER}" -d "\${POSTGRES_DB}" \
                -c "SELECT 'companies=' || count(*) FROM companies UNION ALL SELECT 'documents=' || count(*) FROM documents UNION ALL SELECT 'intelligence=' || count(*) FROM document_intelligence;"

              echo "=== Restore complete ==="
EOJOB

echo "Waiting for restore job to complete..."
kubectl wait --for=condition=complete job/${JOB_NAME} -n ${NAMESPACE} --timeout=600s 2>&1 || {
  echo "Restore job failed or timed out. Logs:"
  kubectl logs job/${JOB_NAME} -n ${NAMESPACE} 2>&1 | tail -20
  exit 1
}

echo ""
kubectl logs job/${JOB_NAME} -n ${NAMESPACE} 2>&1 | tail -15
echo ""
echo "=== Restore from ${BACKUP_NAME} complete ==="
echo "You may want to restart services: kubectl rollout restart deployment -n ${NAMESPACE} --all"