phase 16: custom superset image with trino driver, fix security context
This commit is contained in:
Celes Renata
@@ -120,3 +120,35 @@ jobs:
|
||||
${{ env.IMAGE_BASE }}/dashboard:latest
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
build-superset:
|
||||
needs: lint-and-test
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build and push superset
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: docker
|
||||
file: docker/Dockerfile.superset
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.IMAGE_BASE }}/superset:${{ github.sha }}
|
||||
${{ env.IMAGE_BASE }}/superset:latest
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
@@ -49,6 +49,11 @@ build:
|
||||
-t $(GHCR)/dashboard:$(SHA) \
|
||||
-t $(GHCR)/dashboard:latest \
|
||||
-f frontend/Dockerfile frontend/ || exit 1
|
||||
@echo "Building superset..."
|
||||
docker build \
|
||||
-t $(GHCR)/superset:$(SHA) \
|
||||
-t $(GHCR)/superset:latest \
|
||||
-f docker/Dockerfile.superset docker/ || exit 1
|
||||
|
||||
push:
|
||||
@for svc in $(SERVICES); do \
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# Custom Superset image with Trino + Redis drivers
|
||||
FROM apache/superset:latest
|
||||
|
||||
USER root
|
||||
RUN pip install --no-cache-dir trino[sqlalchemy] redis
|
||||
USER superset
|
||||
@@ -1,7 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "stonks.labels" . | nindent 4 }}
|
||||
kubernetes.io/metadata.name: {{ .Release.Namespace }}
|
||||
@@ -25,10 +25,12 @@ spec:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
{{- include "stonks.podSecurityContext" . | nindent 8 }}
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: superset
|
||||
image: apache/superset:latest
|
||||
image: {{ .Values.image.registry }}/superset:{{ .Values.image.tag }}
|
||||
ports:
|
||||
- containerPort: 8088
|
||||
securityContext:
|
||||
|
||||
@@ -23,7 +23,8 @@ INSERT INTO retention_policies (bucket_name, artifact_class, retention_days, arc
|
||||
('stonks-llm-prompts', 'default', 365, FALSE),
|
||||
('stonks-llm-results', 'default', 365, FALSE),
|
||||
('stonks-lakehouse', 'default', 730, FALSE),
|
||||
('stonks-audit', 'default', 730, FALSE);
|
||||
('stonks-audit', 'default', 730, FALSE)
|
||||
ON CONFLICT (bucket_name, artifact_class) DO NOTHING;
|
||||
|
||||
-- Track retention cleanup runs for observability
|
||||
CREATE TABLE retention_runs (
|
||||
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
NAMESPACE="stonks-oracle"
|
||||
REPO_DIR="$HOME/sources/celesrenata/stonks-oracle"
|
||||
CHART_DIR="$REPO_DIR/infra/helm/stonks-oracle"
|
||||
MIGRATIONS_DIR="$REPO_DIR/infra/migrations"
|
||||
|
||||
# --- Secrets ---
|
||||
GHCR_TOKEN=$(cat /run/secrets/github_token)
|
||||
MINIO_ACCESS_KEY="AKIA6V7J3N9B5P0D2YQH"
|
||||
MINIO_SECRET_KEY='8fG3!v2rJ7$wN@9mLpQ6zXbC4tKdPqW1'
|
||||
PG_PASSWORD='St0nks0racl3!'
|
||||
REDIS_PASSWORD='PSCh4ng3me!'
|
||||
|
||||
echo "=== Stonks Oracle Deployment ==="
|
||||
|
||||
# --- 1. Ensure namespace exists with correct labels ---
|
||||
echo "[1/4] Ensuring namespace $NAMESPACE exists..."
|
||||
if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
|
||||
kubectl create namespace "$NAMESPACE"
|
||||
fi
|
||||
# Label it so Helm doesn't complain about ownership
|
||||
kubectl label namespace "$NAMESPACE" app.kubernetes.io/managed-by=Helm --overwrite
|
||||
kubectl annotate namespace "$NAMESPACE" meta.helm.sh/release-name=stonks-oracle meta.helm.sh/release-namespace=stonks-oracle --overwrite
|
||||
|
||||
# --- 2. Create PostgreSQL user and database ---
|
||||
echo "[2/4] Setting up PostgreSQL database and user..."
|
||||
kubectl exec -i -n postgresql-service postgresql-1 -c postgres -- psql -U postgres <<EOF
|
||||
DO \$\$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'stonks') THEN
|
||||
CREATE USER stonks WITH PASSWORD '$PG_PASSWORD';
|
||||
ELSE
|
||||
ALTER USER stonks WITH PASSWORD '$PG_PASSWORD';
|
||||
END IF;
|
||||
END
|
||||
\$\$;
|
||||
|
||||
SELECT 'CREATE DATABASE stonks OWNER stonks'
|
||||
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'stonks')\gexec
|
||||
|
||||
GRANT ALL PRIVILEGES ON DATABASE stonks TO stonks;
|
||||
EOF
|
||||
|
||||
# --- 3. Run migrations ---
|
||||
echo "[3/4] Running database migrations..."
|
||||
for f in $(ls "$MIGRATIONS_DIR"/*.sql | sort); do
|
||||
echo " -> $(basename "$f")"
|
||||
kubectl exec -i -n postgresql-service postgresql-1 -c postgres -- psql -U postgres -d stonks < "$f" 2>&1 | grep -v "already exists" || true
|
||||
done
|
||||
|
||||
# Grant permissions
|
||||
kubectl exec -i -n postgresql-service postgresql-1 -c postgres -- psql -U postgres -d stonks <<EOF
|
||||
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO stonks;
|
||||
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO stonks;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO stonks;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO stonks;
|
||||
EOF
|
||||
|
||||
# --- 4. Helm deploy ---
|
||||
echo "[4/4] Deploying via Helm..."
|
||||
helm upgrade --install stonks-oracle "$CHART_DIR" \
|
||||
--namespace "$NAMESPACE" \
|
||||
--set "ghcrAuth.password=$GHCR_TOKEN" \
|
||||
--set "secrets.core.POSTGRES_PASSWORD=$PG_PASSWORD" \
|
||||
--set "secrets.core.MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \
|
||||
--set "secrets.core.MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \
|
||||
--set "secrets.core.REDIS_PASSWORD=$REDIS_PASSWORD"
|
||||
|
||||
# --- Rolling restart to pick up secrets ---
|
||||
echo "Rolling restart..."
|
||||
for dep in $(kubectl get deployments -n "$NAMESPACE" -o name); do
|
||||
kubectl rollout restart -n "$NAMESPACE" "$dep"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Deployment complete ==="
|
||||
echo "Waiting for pods..."
|
||||
sleep 10
|
||||
kubectl get pods -n "$NAMESPACE" -o custom-columns='NAME:.metadata.name,READY:.status.containerStatuses[0].ready,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount'
|
||||
echo ""
|
||||
echo "Ingress endpoints:"
|
||||
kubectl get ingress -n "$NAMESPACE" -o custom-columns='HOST:.spec.rules[0].host,ADDRESS:.status.loadBalancer.ingress[0].ip'
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# Reverse of runmefirst.sh
|
||||
# Removes the Stonks Oracle deployment but preserves the PostgreSQL
|
||||
# database/user, MinIO buckets, Redis data, and Ollama.
|
||||
#
|
||||
# Usage: bash runmelast.sh
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
NAMESPACE="stonks-oracle"
|
||||
|
||||
echo "=== Stonks Oracle Teardown ==="
|
||||
|
||||
# --- 1. Uninstall Helm release ---
|
||||
echo "[1/2] Uninstalling Helm release..."
|
||||
helm uninstall stonks-oracle -n "$NAMESPACE" --wait 2>/dev/null \
|
||||
&& echo " Helm release removed." \
|
||||
|| echo " No Helm release found or already removed, continuing."
|
||||
|
||||
# --- 2. Clean up any stragglers ---
|
||||
echo "[2/2] Cleaning up remaining resources in $NAMESPACE..."
|
||||
kubectl delete deployments --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||
kubectl delete statefulsets --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||
kubectl delete services --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||
kubectl delete ingresses --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||
kubectl delete configmaps --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||
kubectl delete secrets --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||
kubectl delete networkpolicies --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||
kubectl delete pvc --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||
kubectl delete jobs --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||
|
||||
# NOTE: namespace is kept intact so Helm labels persist for clean redeploy
|
||||
|
||||
echo ""
|
||||
echo "=== Teardown complete ==="
|
||||
echo ""
|
||||
echo "Preserved (untouched):"
|
||||
echo " - Namespace $NAMESPACE (kept for Helm label compatibility)"
|
||||
echo " - PostgreSQL database 'stonks' and user 'stonks' in postgresql-service"
|
||||
echo " - MinIO buckets in minio-service"
|
||||
echo " - Redis data in redis-service"
|
||||
echo " - Ollama in ollama-service"
|
||||
echo ""
|
||||
echo "To redeploy: bash ~/sources/kube/stonks-oracle/runmefirst.sh"
|
||||
Reference in New Issue
Block a user