phase 17: add backup/restore scripts — PostgreSQL + MinIO → NFS
This commit is contained in:
Celes Renata
Executable
+115
@@ -0,0 +1,115 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Stonks Oracle Backup — PostgreSQL + MinIO → NFS
|
||||||
|
# Usage: bash scripts/backup.sh
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
NAMESPACE="stonks-oracle"
|
||||||
|
NFS_SERVER="192.168.42.8"
|
||||||
|
NFS_PATH="/volume1/Kubernetes/stonks"
|
||||||
|
BACKUP_NAME="stonks-backup-$(date +%Y%m%d-%H%M%S)"
|
||||||
|
JOB_NAME="stonks-backup"
|
||||||
|
|
||||||
|
echo "=== Stonks Oracle Backup: ${BACKUP_NAME} ==="
|
||||||
|
|
||||||
|
# Clean up any previous backup job
|
||||||
|
kubectl delete job ${JOB_NAME} -n ${NAMESPACE} --ignore-not-found=true 2>/dev/null || true
|
||||||
|
|
||||||
|
cat <<EOJOB | kubectl apply -f -
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: ${JOB_NAME}
|
||||||
|
namespace: ${NAMESPACE}
|
||||||
|
spec:
|
||||||
|
ttlSecondsAfterFinished: 300
|
||||||
|
backoffLimit: 0
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
volumes:
|
||||||
|
- name: nfs-backup
|
||||||
|
nfs:
|
||||||
|
server: ${NFS_SERVER}
|
||||||
|
path: ${NFS_PATH}
|
||||||
|
containers:
|
||||||
|
- name: backup
|
||||||
|
image: alpine:3.20
|
||||||
|
volumeMounts:
|
||||||
|
- name: nfs-backup
|
||||||
|
mountPath: /backup
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: stonks-config
|
||||||
|
- secretRef:
|
||||||
|
name: stonks-core-secrets
|
||||||
|
env:
|
||||||
|
- name: MINIO_ACCESS_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: stonks-core-secrets
|
||||||
|
key: MINIO_ACCESS_KEY
|
||||||
|
- name: MINIO_SECRET_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: stonks-core-secrets
|
||||||
|
key: MINIO_SECRET_KEY
|
||||||
|
command: ["sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
set -e
|
||||||
|
apk add --no-cache postgresql16-client curl ca-certificates
|
||||||
|
|
||||||
|
STAMP="${BACKUP_NAME}"
|
||||||
|
DIR="/backup/\${STAMP}"
|
||||||
|
mkdir -p "\${DIR}/minio"
|
||||||
|
|
||||||
|
echo "[1/3] Backing up PostgreSQL..."
|
||||||
|
PGPASSWORD="\${POSTGRES_PASSWORD}" pg_dump \
|
||||||
|
-h "\${POSTGRES_HOST}" -p "\${POSTGRES_PORT}" \
|
||||||
|
-U "\${POSTGRES_USER}" -d "\${POSTGRES_DB}" \
|
||||||
|
--no-owner --no-acl -Fc \
|
||||||
|
-f "\${DIR}/stonks.pgdump"
|
||||||
|
echo " pg_dump done: \$(du -h "\${DIR}/stonks.pgdump" | cut -f1)"
|
||||||
|
|
||||||
|
echo "[2/3] Backing up MinIO buckets..."
|
||||||
|
# Install mc (MinIO client)
|
||||||
|
curl -sL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
|
||||||
|
chmod +x /usr/local/bin/mc
|
||||||
|
mc alias set backup "http://\${MINIO_ENDPOINT}" "\${MINIO_ACCESS_KEY}" "\${MINIO_SECRET_KEY}" --api S3v4 2>/dev/null
|
||||||
|
|
||||||
|
for bucket in stonks-raw-market stonks-raw-news stonks-raw-filings stonks-normalized stonks-llm-prompts stonks-llm-results stonks-lakehouse stonks-audit; do
|
||||||
|
if mc ls "backup/\${bucket}" >/dev/null 2>&1; then
|
||||||
|
echo " Mirroring \${bucket}..."
|
||||||
|
mc mirror --quiet "backup/\${bucket}" "\${DIR}/minio/\${bucket}/" 2>/dev/null || echo " (empty or error)"
|
||||||
|
else
|
||||||
|
echo " Bucket \${bucket} not found, skipping"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "[3/3] Writing manifest..."
|
||||||
|
cat > "\${DIR}/manifest.json" <<EOF
|
||||||
|
{
|
||||||
|
"backup_name": "\${STAMP}",
|
||||||
|
"created_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||||
|
"pg_dump": "stonks.pgdump",
|
||||||
|
"minio_buckets": ["stonks-raw-market","stonks-raw-news","stonks-raw-filings","stonks-normalized","stonks-llm-prompts","stonks-llm-results","stonks-lakehouse","stonks-audit"]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# Symlink latest
|
||||||
|
ln -sfn "\${STAMP}" /backup/latest
|
||||||
|
echo "=== Backup complete: \${DIR} ==="
|
||||||
|
ls -lh "\${DIR}/"
|
||||||
|
EOJOB
|
||||||
|
|
||||||
|
echo "Waiting for backup job to complete..."
|
||||||
|
kubectl wait --for=condition=complete job/${JOB_NAME} -n ${NAMESPACE} --timeout=600s 2>&1 || {
|
||||||
|
echo "Backup job failed or timed out. Logs:"
|
||||||
|
kubectl logs job/${JOB_NAME} -n ${NAMESPACE} 2>&1 | tail -20
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
kubectl logs job/${JOB_NAME} -n ${NAMESPACE} 2>&1 | tail -10
|
||||||
|
echo ""
|
||||||
|
echo "=== Backup ${BACKUP_NAME} complete ==="
|
||||||
Executable
+119
@@ -0,0 +1,119 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Stonks Oracle Restore — NFS → PostgreSQL + MinIO
|
||||||
|
# Usage: bash scripts/restore.sh [backup_name]
|
||||||
|
# If no backup_name given, restores from "latest" symlink.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
NAMESPACE="stonks-oracle"
|
||||||
|
NFS_SERVER="192.168.42.8"
|
||||||
|
NFS_PATH="/volume1/Kubernetes/stonks"
|
||||||
|
BACKUP_NAME="${1:-latest}"
|
||||||
|
JOB_NAME="stonks-restore"
|
||||||
|
|
||||||
|
echo "=== Stonks Oracle Restore: ${BACKUP_NAME} ==="
|
||||||
|
echo "WARNING: This will DROP and recreate the stonks database."
|
||||||
|
echo "Press Ctrl+C within 5 seconds to abort..."
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
# Clean up any previous restore job
|
||||||
|
kubectl delete job ${JOB_NAME} -n ${NAMESPACE} --ignore-not-found=true 2>/dev/null || true
|
||||||
|
|
||||||
|
cat <<EOJOB | kubectl apply -f -
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: ${JOB_NAME}
|
||||||
|
namespace: ${NAMESPACE}
|
||||||
|
spec:
|
||||||
|
ttlSecondsAfterFinished: 300
|
||||||
|
backoffLimit: 0
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
volumes:
|
||||||
|
- name: nfs-backup
|
||||||
|
nfs:
|
||||||
|
server: ${NFS_SERVER}
|
||||||
|
path: ${NFS_PATH}
|
||||||
|
containers:
|
||||||
|
- name: restore
|
||||||
|
image: alpine:3.20
|
||||||
|
volumeMounts:
|
||||||
|
- name: nfs-backup
|
||||||
|
mountPath: /backup
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: stonks-config
|
||||||
|
- secretRef:
|
||||||
|
name: stonks-core-secrets
|
||||||
|
env:
|
||||||
|
- name: MINIO_ACCESS_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: stonks-core-secrets
|
||||||
|
key: MINIO_ACCESS_KEY
|
||||||
|
- name: MINIO_SECRET_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: stonks-core-secrets
|
||||||
|
key: MINIO_SECRET_KEY
|
||||||
|
command: ["sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
set -e
|
||||||
|
apk add --no-cache postgresql16-client curl ca-certificates
|
||||||
|
|
||||||
|
DIR="/backup/${BACKUP_NAME}"
|
||||||
|
if [ ! -f "\${DIR}/stonks.pgdump" ]; then
|
||||||
|
echo "ERROR: No backup found at \${DIR}/stonks.pgdump"
|
||||||
|
ls -la /backup/ 2>/dev/null || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Restoring from: \${DIR}"
|
||||||
|
cat "\${DIR}/manifest.json" 2>/dev/null || true
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "[1/3] Restoring PostgreSQL..."
|
||||||
|
# Drop and recreate all tables by restoring with --clean
|
||||||
|
PGPASSWORD="\${POSTGRES_PASSWORD}" pg_restore \
|
||||||
|
-h "\${POSTGRES_HOST}" -p "\${POSTGRES_PORT}" \
|
||||||
|
-U "\${POSTGRES_USER}" -d "\${POSTGRES_DB}" \
|
||||||
|
--clean --if-exists --no-owner --no-acl \
|
||||||
|
"\${DIR}/stonks.pgdump" 2>&1 || echo " (some drop errors are expected on first restore)"
|
||||||
|
echo " PostgreSQL restored"
|
||||||
|
|
||||||
|
echo "[2/3] Restoring MinIO buckets..."
|
||||||
|
curl -sL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
|
||||||
|
chmod +x /usr/local/bin/mc
|
||||||
|
mc alias set backup "http://\${MINIO_ENDPOINT}" "\${MINIO_ACCESS_KEY}" "\${MINIO_SECRET_KEY}" --api S3v4 2>/dev/null
|
||||||
|
|
||||||
|
for bucket in stonks-raw-market stonks-raw-news stonks-raw-filings stonks-normalized stonks-llm-prompts stonks-llm-results stonks-lakehouse stonks-audit; do
|
||||||
|
if [ -d "\${DIR}/minio/\${bucket}" ]; then
|
||||||
|
echo " Restoring \${bucket}..."
|
||||||
|
mc mb --ignore-existing "backup/\${bucket}" 2>/dev/null || true
|
||||||
|
mc mirror --overwrite --quiet "\${DIR}/minio/\${bucket}/" "backup/\${bucket}" 2>/dev/null || echo " (empty or error)"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "[3/3] Verifying..."
|
||||||
|
PGPASSWORD="\${POSTGRES_PASSWORD}" psql \
|
||||||
|
-h "\${POSTGRES_HOST}" -p "\${POSTGRES_PORT}" \
|
||||||
|
-U "\${POSTGRES_USER}" -d "\${POSTGRES_DB}" \
|
||||||
|
-c "SELECT 'companies=' || count(*) FROM companies UNION ALL SELECT 'documents=' || count(*) FROM documents UNION ALL SELECT 'intelligence=' || count(*) FROM document_intelligence;"
|
||||||
|
|
||||||
|
echo "=== Restore complete ==="
|
||||||
|
EOJOB
|
||||||
|
|
||||||
|
echo "Waiting for restore job to complete..."
|
||||||
|
kubectl wait --for=condition=complete job/${JOB_NAME} -n ${NAMESPACE} --timeout=600s 2>&1 || {
|
||||||
|
echo "Restore job failed or timed out. Logs:"
|
||||||
|
kubectl logs job/${JOB_NAME} -n ${NAMESPACE} 2>&1 | tail -20
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
kubectl logs job/${JOB_NAME} -n ${NAMESPACE} 2>&1 | tail -15
|
||||||
|
echo ""
|
||||||
|
echo "=== Restore from ${BACKUP_NAME} complete ==="
|
||||||
|
echo "You may want to restart services: kubectl rollout restart deployment -n ${NAMESPACE} --all"
|
||||||
Reference in New Issue
Block a user