diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2a9380b..4a6edb4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -120,3 +120,35 @@ jobs: ${{ env.IMAGE_BASE }}/dashboard:latest cache-from: type=gha cache-to: type=gha,mode=max + + build-superset: + needs: lint-and-test + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push superset + uses: docker/build-push-action@v6 + with: + context: docker + file: docker/Dockerfile.superset + push: true + tags: | + ${{ env.IMAGE_BASE }}/superset:${{ github.sha }} + ${{ env.IMAGE_BASE }}/superset:latest + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Makefile b/Makefile index aa3c174..de95de6 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,11 @@ build: -t $(GHCR)/dashboard:$(SHA) \ -t $(GHCR)/dashboard:latest \ -f frontend/Dockerfile frontend/ || exit 1 + @echo "Building superset..." + docker build \ + -t $(GHCR)/superset:$(SHA) \ + -t $(GHCR)/superset:latest \ + -f docker/Dockerfile.superset docker/ || exit 1 push: @for svc in $(SERVICES); do \ diff --git a/docker/Dockerfile.superset b/docker/Dockerfile.superset new file mode 100644 index 0000000..9712321 --- /dev/null +++ b/docker/Dockerfile.superset @@ -0,0 +1,6 @@ +# Custom Superset image with Trino + Redis drivers +FROM apache/superset:latest + +USER root +RUN pip install --no-cache-dir trino[sqlalchemy] redis +USER superset diff --git a/infra/helm/stonks-oracle/templates/namespace.yaml b/infra/helm/stonks-oracle/templates/namespace.yaml deleted file mode 100644 index f94a748..0000000 --- a/infra/helm/stonks-oracle/templates/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: {{ .Release.Namespace }} - labels: - {{- include "stonks.labels" . | nindent 4 }} - kubernetes.io/metadata.name: {{ .Release.Namespace }} diff --git a/infra/helm/stonks-oracle/templates/superset.yaml b/infra/helm/stonks-oracle/templates/superset.yaml index 78c2267..6830c22 100644 --- a/infra/helm/stonks-oracle/templates/superset.yaml +++ b/infra/helm/stonks-oracle/templates/superset.yaml @@ -25,10 +25,12 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} securityContext: - {{- include "stonks.podSecurityContext" . | nindent 8 }} + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault containers: - name: superset - image: apache/superset:latest + image: {{ .Values.image.registry }}/superset:{{ .Values.image.tag }} ports: - containerPort: 8088 securityContext: diff --git a/infra/migrations/014_retention_policies.sql b/infra/migrations/014_retention_policies.sql index eaec6af..a0f4b69 100644 --- a/infra/migrations/014_retention_policies.sql +++ b/infra/migrations/014_retention_policies.sql @@ -23,7 +23,8 @@ INSERT INTO retention_policies (bucket_name, artifact_class, retention_days, arc ('stonks-llm-prompts', 'default', 365, FALSE), ('stonks-llm-results', 'default', 365, FALSE), ('stonks-lakehouse', 'default', 730, FALSE), - ('stonks-audit', 'default', 730, FALSE); + ('stonks-audit', 'default', 730, FALSE) +ON CONFLICT (bucket_name, artifact_class) DO NOTHING; -- Track retention cleanup runs for observability CREATE TABLE retention_runs ( diff --git a/runmefirst.sh b/runmefirst.sh new file mode 100755 index 0000000..76653ae --- /dev/null +++ b/runmefirst.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +set -euo pipefail + +NAMESPACE="stonks-oracle" +REPO_DIR="$HOME/sources/celesrenata/stonks-oracle" +CHART_DIR="$REPO_DIR/infra/helm/stonks-oracle" +MIGRATIONS_DIR="$REPO_DIR/infra/migrations" + +# --- Secrets --- +GHCR_TOKEN=$(cat /run/secrets/github_token) +MINIO_ACCESS_KEY="AKIA6V7J3N9B5P0D2YQH" +MINIO_SECRET_KEY='8fG3!v2rJ7$wN@9mLpQ6zXbC4tKdPqW1' +PG_PASSWORD='St0nks0racl3!' +REDIS_PASSWORD='PSCh4ng3me!' + +echo "=== Stonks Oracle Deployment ===" + +# --- 1. Ensure namespace exists with correct labels --- +echo "[1/4] Ensuring namespace $NAMESPACE exists..." +if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then + kubectl create namespace "$NAMESPACE" +fi +# Label it so Helm doesn't complain about ownership +kubectl label namespace "$NAMESPACE" app.kubernetes.io/managed-by=Helm --overwrite +kubectl annotate namespace "$NAMESPACE" meta.helm.sh/release-name=stonks-oracle meta.helm.sh/release-namespace=stonks-oracle --overwrite + +# --- 2. Create PostgreSQL user and database --- +echo "[2/4] Setting up PostgreSQL database and user..." +kubectl exec -i -n postgresql-service postgresql-1 -c postgres -- psql -U postgres < $(basename "$f")" + kubectl exec -i -n postgresql-service postgresql-1 -c postgres -- psql -U postgres -d stonks < "$f" 2>&1 | grep -v "already exists" || true +done + +# Grant permissions +kubectl exec -i -n postgresql-service postgresql-1 -c postgres -- psql -U postgres -d stonks </dev/null \ + && echo " Helm release removed." \ + || echo " No Helm release found or already removed, continuing." + +# --- 2. Clean up any stragglers --- +echo "[2/2] Cleaning up remaining resources in $NAMESPACE..." +kubectl delete deployments --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true +kubectl delete statefulsets --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true +kubectl delete services --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true +kubectl delete ingresses --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true +kubectl delete configmaps --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true +kubectl delete secrets --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true +kubectl delete networkpolicies --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true +kubectl delete pvc --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true +kubectl delete jobs --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true + +# NOTE: namespace is kept intact so Helm labels persist for clean redeploy + +echo "" +echo "=== Teardown complete ===" +echo "" +echo "Preserved (untouched):" +echo " - Namespace $NAMESPACE (kept for Helm label compatibility)" +echo " - PostgreSQL database 'stonks' and user 'stonks' in postgresql-service" +echo " - MinIO buckets in minio-service" +echo " - Redis data in redis-service" +echo " - Ollama in ollama-service" +echo "" +echo "To redeploy: bash ~/sources/kube/stonks-oracle/runmefirst.sh"