diff --git a/scripts/factory-reset.sh b/scripts/factory-reset.sh new file mode 100755 index 0000000..6173dab --- /dev/null +++ b/scripts/factory-reset.sh @@ -0,0 +1,360 @@ +#!/usr/bin/env bash +set -euo pipefail + +# factory-reset.sh — Factory reset a Stonks Oracle stage +# +# Drops and recreates the database, flushes Redis keys, empties S3 buckets, +# re-runs migrations, and re-seeds the symbol registry. +# +# Usage: +# bash scripts/factory-reset.sh [--component ] [--yes] [--verbose] +# +# Stages: +# production — stonks-oracle namespace, DB=stonks, Redis DB=0, buckets=stonks-* +# paper — stonks-paper namespace, DB=stonks_paper, Redis DB=2, buckets=paper-stonks-* +# beta — stonks-beta namespace, DB=stonks_beta, Redis DB=1, buckets=beta-stonks-* +# +# Components (optional, default: all): +# all — Full factory reset (DB + S3 + Redis) +# db — Database only (drop/recreate + migrations + seed) +# s3 — S3 buckets only (empty all stage buckets) +# redis — Redis only (flush stage keys) +# computed — Computed data only (trends, recommendations, orders, positions) +# +# Examples: +# bash scripts/factory-reset.sh beta # Full reset of beta +# bash scripts/factory-reset.sh production --component db # DB-only reset of production +# bash scripts/factory-reset.sh paper --component computed # Clear computed data in paper +# bash scripts/factory-reset.sh beta --verbose # Full reset with per-object output +# +# Requirements: +# - kubectl access to the cluster +# - mc (MinIO client) configured with alias "stonks" + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +STAGE="${1:-}" +COMPONENT="all" +AUTO_YES=false +VERBOSE=false + +if [[ -z "$STAGE" ]]; then + echo "Usage: bash scripts/factory-reset.sh [--component ] [--yes] [--verbose]" + echo "Stages: production, paper, beta" + echo "Components: all, db, s3, redis, computed" + echo "Flags: --yes Skip confirmation prompt" + echo " --verbose Show detailed per-object output" + exit 1 +fi + +shift +while [[ $# -gt 0 ]]; do + case $1 in + --component) COMPONENT="$2"; shift 2 ;; + --yes|-y) AUTO_YES=true; shift ;; + --verbose|-v) VERBOSE=true; shift ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +# Map stage to namespace, DB name, Redis DB, and bucket prefix +case "$STAGE" in + production|prod) + NAMESPACE="stonks-oracle" + DB_NAME="stonks" + REDIS_DB=0 + BUCKET_PREFIX="stonks-" + REDIS_KEY_PREFIX="stonks:" + DEPLOY_STAGE="" + ;; + paper) + NAMESPACE="stonks-paper" + DB_NAME="stonks_paper" + REDIS_DB=2 + BUCKET_PREFIX="paper-stonks-" + REDIS_KEY_PREFIX="stonks:paper:" + DEPLOY_STAGE="paper" + ;; + beta) + NAMESPACE="stonks-beta" + DB_NAME="stonks_beta" + REDIS_DB=1 + BUCKET_PREFIX="beta-stonks-" + REDIS_KEY_PREFIX="stonks:beta:" + DEPLOY_STAGE="beta" + ;; + *) + echo "ERROR: Unknown stage '$STAGE'. Use: production, paper, beta" + exit 1 + ;; +esac + +PG_POD="postgresql-1" +PG_NS="postgresql-service" +PG_USER="postgres" +REDIS_HOST="redis-master.redis-service.svc.cluster.local" +REDIS_PORT="6379" +REDIS_PASSWORD="PSCh4ng3me!" +MC_ALIAS="stonks" + +# S3 bucket suffixes +BUCKET_SUFFIXES=( + "audit" + "lakehouse" + "llm-prompts" + "llm-results" + "normalized" + "raw-filings" + "raw-market" + "raw-news" +) + +echo "============================================" +echo " Stonks Oracle Factory Reset" +echo "============================================" +echo " Stage: $STAGE" +echo " Namespace: $NAMESPACE" +echo " Database: $DB_NAME" +echo " Redis DB: $REDIS_DB" +echo " Buckets: ${BUCKET_PREFIX}*" +echo " Component: $COMPONENT" +if [[ "$VERBOSE" == true ]]; then +echo " Verbose: ON" +fi +echo "============================================" +echo "" + +# Safety confirmation +if [[ "$AUTO_YES" == true ]]; then + echo "⚠️ --yes flag set, skipping confirmation" +else + read -rp "⚠️ This will DESTROY data. Type '$STAGE' to confirm: " confirm + if [[ "$confirm" != "$STAGE" ]]; then + echo "Aborted." + exit 1 + fi +fi +echo "" + +# --------------------------------------------------------------------------- +# Helper: scale down all deployments in the namespace +# --------------------------------------------------------------------------- +scale_down() { + echo "--- Scaling down $NAMESPACE deployments ---" + local deployments + deployments=$(kubectl get deployments -n "$NAMESPACE" -o name 2>/dev/null || true) + if [[ -n "$deployments" ]]; then + local count + count=$(echo "$deployments" | wc -l) + if [[ "$VERBOSE" == true ]]; then + echo "$deployments" | xargs -I{} kubectl scale {} --replicas=0 -n "$NAMESPACE" 2>/dev/null || true + else + echo "$deployments" | xargs -I{} kubectl scale {} --replicas=0 -n "$NAMESPACE" &>/dev/null || true + fi + echo " Waiting for $count deployments to terminate..." + kubectl wait --for=delete pod --all -n "$NAMESPACE" --timeout=60s 2>/dev/null || true + fi + echo " ✓ All deployments scaled to 0" +} + +# --------------------------------------------------------------------------- +# Helper: scale up all deployments in the namespace +# --------------------------------------------------------------------------- +scale_up() { + echo "--- Scaling up $NAMESPACE deployments ---" + # ArgoCD will auto-heal and restore replica counts, just trigger a sync + if kubectl get application -n argocd 2>/dev/null | grep -q "$NAMESPACE"; then + echo " ArgoCD will restore replicas via self-heal" + else + echo " Manually restoring replicas..." + local deployments + deployments=$(kubectl get deployments -n "$NAMESPACE" -o name 2>/dev/null || true) + if [[ "$VERBOSE" == true ]]; then + echo "$deployments" | xargs -I{} kubectl scale {} --replicas=1 -n "$NAMESPACE" 2>/dev/null || true + else + echo "$deployments" | xargs -I{} kubectl scale {} --replicas=1 -n "$NAMESPACE" &>/dev/null || true + fi + fi + echo " ✓ Scale-up triggered" + + # Wait for the scheduler pod (which runs migrations via init containers) + # to be fully ready before other services start querying the DB. + echo " Waiting for scheduler pod (runs migrations)..." + kubectl rollout restart deployment/scheduler -n "$NAMESPACE" 2>/dev/null || true + kubectl rollout status deployment/scheduler -n "$NAMESPACE" --timeout=120s 2>/dev/null || true + echo " ✓ Scheduler ready (migrations applied)" +} + +# --------------------------------------------------------------------------- +# Reset: Database +# --------------------------------------------------------------------------- +reset_db() { + echo "--- Resetting database: $DB_NAME ---" + + # Terminate active connections + if [[ "$VERBOSE" == true ]]; then + kubectl exec -n "$PG_NS" "$PG_POD" -c postgres -- psql -U "$PG_USER" -c \ + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$DB_NAME' AND pid <> pg_backend_pid();" \ + 2>/dev/null || true + else + kubectl exec -n "$PG_NS" "$PG_POD" -c postgres -- psql -U "$PG_USER" -tAc \ + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$DB_NAME' AND pid <> pg_backend_pid();" \ + &>/dev/null || true + fi + + # Drop and recreate + kubectl exec -n "$PG_NS" "$PG_POD" -c postgres -- psql -U "$PG_USER" -c \ + "DROP DATABASE IF EXISTS $DB_NAME;" + kubectl exec -n "$PG_NS" "$PG_POD" -c postgres -- psql -U "$PG_USER" -c \ + "CREATE DATABASE $DB_NAME OWNER stonks;" + echo " ✓ Database recreated" + + # Run migrations + local migrations + migrations=($(ls infra/migrations/*.sql | sort)) + local count=${#migrations[@]} + echo " Running $count migrations..." + for migration in "${migrations[@]}"; do + if [[ "$VERBOSE" == true ]]; then + echo " Applying $(basename "$migration")..." + fi + kubectl exec -n "$PG_NS" "$PG_POD" -c postgres -i -- psql -U stonks -d "$DB_NAME" < "$migration" 2>/dev/null || true + done + echo " ✓ Migrations applied ($count files)" + + # Seed symbol registry + echo " Seeding symbol registry..." + # Wait for at least one pod to be ready + scale_up + sleep 10 + local scheduler_pod + scheduler_pod=$(kubectl get pods -n "$NAMESPACE" -l app=scheduler -o name 2>/dev/null | head -1) + if [[ -n "$scheduler_pod" ]]; then + kubectl exec -n "$NAMESPACE" "$scheduler_pod" -c scheduler -- \ + python -m services.symbol_registry.seed 2>/dev/null && echo " ✓ Seeded" || echo " ⚠ Seed failed (will retry on next restart)" + else + echo " ⚠ No scheduler pod available — seed will run on next deployment" + fi +} + +# --------------------------------------------------------------------------- +# Reset: Computed data only (trends, recommendations, orders, positions) +# --------------------------------------------------------------------------- +reset_computed() { + echo "--- Clearing computed data in $DB_NAME ---" + kubectl exec -n "$PG_NS" "$PG_POD" -c postgres -- psql -U "$PG_USER" -d "$DB_NAME" -c " + -- Order matters due to FK constraints + DELETE FROM recommendation_evidence; + DELETE FROM risk_evaluations; + DELETE FROM order_events; + DELETE FROM orders; + DELETE FROM trading_decisions; + DELETE FROM positions; + DELETE FROM portfolio_snapshots; + DELETE FROM reserve_pool_ledger; + DELETE FROM risk_tier_history; + DELETE FROM circuit_breaker_events; + DELETE FROM notifications; + DELETE FROM recommendations; + DELETE FROM trend_evidence; + DELETE FROM trend_projections; + DELETE FROM trend_history; + DELETE FROM trend_windows; + DELETE FROM backtest_trades; + DELETE FROM backtest_runs; + DELETE FROM position_stop_levels; + " 2>/dev/null + echo " ✓ Computed data cleared" +} + +# --------------------------------------------------------------------------- +# Reset: S3 buckets +# --------------------------------------------------------------------------- +reset_s3() { + echo "--- Emptying S3 buckets: ${BUCKET_PREFIX}* ---" + for suffix in "${BUCKET_SUFFIXES[@]}"; do + local bucket="${BUCKET_PREFIX}${suffix}" + if mc ls "${MC_ALIAS}/${bucket}" &>/dev/null; then + echo -n " Emptying ${bucket}..." + if [[ "$VERBOSE" == true ]]; then + echo "" + mc rm --recursive --force "${MC_ALIAS}/${bucket}/" 2>/dev/null || true + else + local removed + removed=$(mc rm --recursive --force "${MC_ALIAS}/${bucket}/" 2>/dev/null | wc -l || echo "0") + echo " ${removed} objects removed" + fi + echo " ✓ ${bucket} emptied" + else + echo " ⚠ ${bucket} not found (skipping)" + fi + done + echo " ✓ All S3 buckets emptied" +} + +# --------------------------------------------------------------------------- +# Reset: Redis +# --------------------------------------------------------------------------- +reset_redis() { + echo "--- Flushing Redis DB $REDIS_DB ---" + kubectl exec -n "$NAMESPACE" deployment/scheduler -c scheduler -- python -c " +import redis +r = redis.from_url('redis://:${REDIS_PASSWORD}@${REDIS_HOST}:${REDIS_PORT}/${REDIS_DB}') +keys = r.keys('stonks:*') +if keys: + r.delete(*keys) + print(f' Deleted {len(keys)} keys') +else: + print(' No keys to delete') +" 2>/dev/null || { + # Fallback: flush the entire Redis DB if no scheduler pod + echo " Falling back to FLUSHDB..." + kubectl exec -n redis-service redis-master-0 -- redis-cli -a "$REDIS_PASSWORD" -n "$REDIS_DB" FLUSHDB 2>/dev/null || true + } + echo " ✓ Redis flushed" +} + +# --------------------------------------------------------------------------- +# Execute based on component selection +# --------------------------------------------------------------------------- + +case "$COMPONENT" in + all) + scale_down + reset_db + reset_s3 + reset_redis + scale_up + ;; + db) + scale_down + reset_db + scale_up + ;; + s3) + reset_s3 + ;; + redis) + reset_redis + ;; + computed) + reset_computed + ;; + *) + echo "ERROR: Unknown component '$COMPONENT'. Use: all, db, s3, redis, computed" + exit 1 + ;; +esac + +echo "" +echo "============================================" +echo " Factory reset complete: $STAGE / $COMPONENT" +echo "============================================" +echo "" +echo "Next steps:" +echo " - ArgoCD will auto-restore pod replicas" +echo " - Migrations and seed run automatically on scheduler init" +echo " - Ingestion will begin on the next scheduler cycle (~15s)" +echo " - First aggregation will run within ~15 minutes"