phase 16: custom superset image with trino driver, fix security context

.github/workflows/build.yml

@@ -120,3 +120,35 @@ jobs:
             ${{ env.IMAGE_BASE }}/dashboard:latest
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
+  build-superset:
+    needs: lint-and-test
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push superset
+        uses: docker/build-push-action@v6
+        with:
+          context: docker
+          file: docker/Dockerfile.superset
+          push: true
+          tags: |
+            ${{ env.IMAGE_BASE }}/superset:${{ github.sha }}
+            ${{ env.IMAGE_BASE }}/superset:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

Makefile

@@ -49,6 +49,11 @@ build:
 		-t $(GHCR)/dashboard:$(SHA) \
 		-t $(GHCR)/dashboard:latest \
 		-f frontend/Dockerfile frontend/ || exit 1
+	@echo "Building superset..."
+	docker build \
+		-t $(GHCR)/superset:$(SHA) \
+		-t $(GHCR)/superset:latest \
+		-f docker/Dockerfile.superset docker/ || exit 1
 
 push:
 	@for svc in $(SERVICES); do \

docker/Dockerfile.superset

@@ -0,0 +1,6 @@
+# Custom Superset image with Trino + Redis drivers
+FROM apache/superset:latest
+
+USER root
+RUN pip install --no-cache-dir trino[sqlalchemy] redis
+USER superset

infra/helm/stonks-oracle/templates/namespace.yaml

@@ -1,7 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: {{ .Release.Namespace }}
-  labels:
-    {{- include "stonks.labels" . | nindent 4 }}
-    kubernetes.io/metadata.name: {{ .Release.Namespace }}

infra/helm/stonks-oracle/templates/superset.yaml

@@ -25,10 +25,12 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       securityContext:
-        {{- include "stonks.podSecurityContext" . | nindent 8 }}
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
       containers:
         - name: superset
-          image: apache/superset:latest
+          image: {{ .Values.image.registry }}/superset:{{ .Values.image.tag }}
           ports:
             - containerPort: 8088
           securityContext:

infra/migrations/014_retention_policies.sql

@@ -23,7 +23,8 @@ INSERT INTO retention_policies (bucket_name, artifact_class, retention_days, arc
     ('stonks-llm-prompts', 'default', 365, FALSE),
     ('stonks-llm-results', 'default', 365, FALSE),
     ('stonks-lakehouse',   'default', 730, FALSE),
-    ('stonks-audit',       'default', 730, FALSE);
+    ('stonks-audit',       'default', 730, FALSE)
+ON CONFLICT (bucket_name, artifact_class) DO NOTHING;
 
 -- Track retention cleanup runs for observability
 CREATE TABLE retention_runs (

runmefirst.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+NAMESPACE="stonks-oracle"
+REPO_DIR="$HOME/sources/celesrenata/stonks-oracle"
+CHART_DIR="$REPO_DIR/infra/helm/stonks-oracle"
+MIGRATIONS_DIR="$REPO_DIR/infra/migrations"
+
+# --- Secrets ---
+GHCR_TOKEN=$(cat /run/secrets/github_token)
+MINIO_ACCESS_KEY="AKIA6V7J3N9B5P0D2YQH"
+MINIO_SECRET_KEY='8fG3!v2rJ7$wN@9mLpQ6zXbC4tKdPqW1'
+PG_PASSWORD='St0nks0racl3!'
+REDIS_PASSWORD='PSCh4ng3me!'
+
+echo "=== Stonks Oracle Deployment ==="
+
+# --- 1. Ensure namespace exists with correct labels ---
+echo "[1/4] Ensuring namespace $NAMESPACE exists..."
+if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
+  kubectl create namespace "$NAMESPACE"
+fi
+# Label it so Helm doesn't complain about ownership
+kubectl label namespace "$NAMESPACE" app.kubernetes.io/managed-by=Helm --overwrite
+kubectl annotate namespace "$NAMESPACE" meta.helm.sh/release-name=stonks-oracle meta.helm.sh/release-namespace=stonks-oracle --overwrite
+
+# --- 2. Create PostgreSQL user and database ---
+echo "[2/4] Setting up PostgreSQL database and user..."
+kubectl exec -i -n postgresql-service postgresql-1 -c postgres -- psql -U postgres <<EOF
+DO \$\$
+BEGIN
+  IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'stonks') THEN
+    CREATE USER stonks WITH PASSWORD '$PG_PASSWORD';
+  ELSE
+    ALTER USER stonks WITH PASSWORD '$PG_PASSWORD';
+  END IF;
+END
+\$\$;
+
+SELECT 'CREATE DATABASE stonks OWNER stonks'
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'stonks')\gexec
+
+GRANT ALL PRIVILEGES ON DATABASE stonks TO stonks;
+EOF
+
+# --- 3. Run migrations ---
+echo "[3/4] Running database migrations..."
+for f in $(ls "$MIGRATIONS_DIR"/*.sql | sort); do
+  echo "  -> $(basename "$f")"
+  kubectl exec -i -n postgresql-service postgresql-1 -c postgres -- psql -U postgres -d stonks < "$f" 2>&1 | grep -v "already exists" || true
+done
+
+# Grant permissions
+kubectl exec -i -n postgresql-service postgresql-1 -c postgres -- psql -U postgres -d stonks <<EOF
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO stonks;
+GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO stonks;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO stonks;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO stonks;
+EOF
+
+# --- 4. Helm deploy ---
+echo "[4/4] Deploying via Helm..."
+helm upgrade --install stonks-oracle "$CHART_DIR" \
+  --namespace "$NAMESPACE" \
+  --set "ghcrAuth.password=$GHCR_TOKEN" \
+  --set "secrets.core.POSTGRES_PASSWORD=$PG_PASSWORD" \
+  --set "secrets.core.MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \
+  --set "secrets.core.MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \
+  --set "secrets.core.REDIS_PASSWORD=$REDIS_PASSWORD"
+
+# --- Rolling restart to pick up secrets ---
+echo "Rolling restart..."
+for dep in $(kubectl get deployments -n "$NAMESPACE" -o name); do
+  kubectl rollout restart -n "$NAMESPACE" "$dep"
+done
+
+echo ""
+echo "=== Deployment complete ==="
+echo "Waiting for pods..."
+sleep 10
+kubectl get pods -n "$NAMESPACE" -o custom-columns='NAME:.metadata.name,READY:.status.containerStatuses[0].ready,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount'
+echo ""
+echo "Ingress endpoints:"
+kubectl get ingress -n "$NAMESPACE" -o custom-columns='HOST:.spec.rules[0].host,ADDRESS:.status.loadBalancer.ingress[0].ip'

runmelast.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Reverse of runmefirst.sh
+# Removes the Stonks Oracle deployment but preserves the PostgreSQL
+# database/user, MinIO buckets, Redis data, and Ollama.
+#
+# Usage: bash runmelast.sh
+
+set -uo pipefail
+
+NAMESPACE="stonks-oracle"
+
+echo "=== Stonks Oracle Teardown ==="
+
+# --- 1. Uninstall Helm release ---
+echo "[1/2] Uninstalling Helm release..."
+helm uninstall stonks-oracle -n "$NAMESPACE" --wait 2>/dev/null \
+  && echo "  Helm release removed." \
+  || echo "  No Helm release found or already removed, continuing."
+
+# --- 2. Clean up any stragglers ---
+echo "[2/2] Cleaning up remaining resources in $NAMESPACE..."
+kubectl delete deployments --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+kubectl delete statefulsets --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+kubectl delete services --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+kubectl delete ingresses --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+kubectl delete configmaps --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+kubectl delete secrets --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+kubectl delete networkpolicies --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+kubectl delete pvc --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+kubectl delete jobs --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+
+# NOTE: namespace is kept intact so Helm labels persist for clean redeploy
+
+echo ""
+echo "=== Teardown complete ==="
+echo ""
+echo "Preserved (untouched):"
+echo "  - Namespace $NAMESPACE (kept for Helm label compatibility)"
+echo "  - PostgreSQL database 'stonks' and user 'stonks' in postgresql-service"
+echo "  - MinIO buckets in minio-service"
+echo "  - Redis data in redis-service"
+echo "  - Ollama in ollama-service"
+echo ""
+echo "To redeploy: bash ~/sources/kube/stonks-oracle/runmefirst.sh"