phase 15: helm chart for stonks-oracle deployment
This commit is contained in:
Celes Renata
@@ -142,22 +142,22 @@
|
||||
- Commit all current changes with message `phase 14-15: docker build validation and helm deployment`
|
||||
- Push to main branch
|
||||
- _Requirements: N1_
|
||||
- [ ] 15.2 Validate GitHub Actions workflow builds containers
|
||||
- [x] 15.2 Validate GitHub Actions workflow builds containers
|
||||
- Monitor the GitHub Actions run to confirm lint-and-test and build-services jobs succeed
|
||||
- Fix any CI failures and re-push if needed
|
||||
- _Requirements: N1_
|
||||
- [ ] 15.3 Create Helm chart for stonks-oracle deployment
|
||||
- [x] 15.3 Create Helm chart for stonks-oracle deployment
|
||||
- Create `infra/helm/stonks-oracle/Chart.yaml` with chart metadata
|
||||
- Create `infra/helm/stonks-oracle/values.yaml` with configurable image tags, replica counts, resource limits, and environment references
|
||||
- Create Helm templates for all deployments, services, configmap, secrets, ingress, and network policies from existing K8s manifests
|
||||
- Add imagePullSecrets configuration for GHCR private registry access
|
||||
- Add a template for a Kubernetes Secret of type `kubernetes.io/dockerconfigjson` for GHCR authentication
|
||||
- _Requirements: N1, 8.2_
|
||||
- [ ] 15.4 Configure GHCR image pull authentication on the cluster
|
||||
- [x] 15.4 Configure GHCR image pull authentication on the cluster
|
||||
- Create a `docker-registry` secret in the `stonks-oracle` namespace with GHCR credentials (using a GitHub PAT or deploy key)
|
||||
- Reference the imagePullSecret in all deployment specs via the Helm values
|
||||
- _Requirements: 8.2, N1_
|
||||
- [ ] 15.5 Deploy stonks-oracle to the cluster via Helm
|
||||
- [-] 15.5 Deploy stonks-oracle to the cluster via Helm
|
||||
- Run `helm install` or `helm upgrade --install` targeting the `stonks-oracle` namespace
|
||||
- Verify all pods reach Running/Ready state
|
||||
- Verify services and ingress endpoints are reachable
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: v2
|
||||
name: stonks-oracle
|
||||
description: AI market intelligence and paper-trading platform
|
||||
type: application
|
||||
version: 0.1.0
|
||||
appVersion: "1.0.0"
|
||||
maintainers:
|
||||
- name: Celes Renata
|
||||
email: celes@frameshift.net
|
||||
@@ -0,0 +1,15 @@
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "stonks.labels" -}}
|
||||
app.kubernetes.io/part-of: stonks-oracle
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Full image path for a service
|
||||
*/}}
|
||||
{{- define "stonks.image" -}}
|
||||
{{ $.Values.imageRegistry }}/{{ .image }}:{{ $.Values.imageTag }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: stonks-config
|
||||
namespace: {{ .Values.namespace }}
|
||||
labels:
|
||||
{{- include "stonks.labels" . | nindent 4 }}
|
||||
data:
|
||||
{{- range $key, $val := .Values.config }}
|
||||
{{ $key }}: {{ $val | quote }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,87 @@
|
||||
{{- range $name, $svc := .Values.services }}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
namespace: {{ $.Values.namespace }}
|
||||
labels:
|
||||
app: {{ $name }}
|
||||
{{- include "stonks.labels" $ | nindent 4 }}
|
||||
stonks-oracle/tier: {{ $svc.tier }}
|
||||
spec:
|
||||
replicas: {{ $svc.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ $name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ $name }}
|
||||
stonks-oracle/tier: {{ $svc.tier }}
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
{{- if $.Values.imagePullSecrets.enabled }}
|
||||
imagePullSecrets:
|
||||
- name: {{ $.Values.imagePullSecrets.name }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: {{ $name }}
|
||||
image: {{ $.Values.imageRegistry }}/{{ $svc.image }}:{{ $.Values.imageTag }}
|
||||
imagePullPolicy: {{ $.Values.imagePullPolicy }}
|
||||
{{- if $svc.port }}
|
||||
ports:
|
||||
- containerPort: {{ $svc.port }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: stonks-config
|
||||
{{- range $svc.secrets }}
|
||||
- secretRef:
|
||||
name: {{ . }}
|
||||
{{- end }}
|
||||
resources:
|
||||
requests:
|
||||
cpu: {{ $svc.resources.requests.cpu }}
|
||||
memory: {{ $svc.resources.requests.memory }}
|
||||
limits:
|
||||
cpu: {{ $svc.resources.limits.cpu | quote }}
|
||||
memory: {{ $svc.resources.limits.memory }}
|
||||
{{- if $svc.probes }}
|
||||
{{- if $svc.probes.readiness }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: {{ $svc.probes.readiness.path }}
|
||||
port: {{ $svc.probes.readiness.port }}
|
||||
initialDelaySeconds: {{ $svc.probes.readiness.initialDelay }}
|
||||
periodSeconds: {{ $svc.probes.readiness.period }}
|
||||
{{- end }}
|
||||
{{- if $svc.probes.liveness }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: {{ $svc.probes.liveness.path }}
|
||||
port: {{ $svc.probes.liveness.port }}
|
||||
initialDelaySeconds: {{ $svc.probes.liveness.initialDelay }}
|
||||
periodSeconds: {{ $svc.probes.liveness.period }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir:
|
||||
sizeLimit: 10Mi
|
||||
{{- end }}
|
||||
@@ -0,0 +1,12 @@
|
||||
{{- if and .Values.imagePullSecrets.enabled .Values.imagePullSecrets.createSecret }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ .Values.imagePullSecrets.name }}
|
||||
namespace: {{ .Values.namespace }}
|
||||
labels:
|
||||
{{- include "stonks.labels" . | nindent 4 }}
|
||||
type: kubernetes.io/dockerconfigjson
|
||||
data:
|
||||
.dockerconfigjson: {{ .Values.imagePullSecrets.dockerconfigjson }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,140 @@
|
||||
{{- if .Values.hiveMetastore.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: hive-metastore
|
||||
namespace: {{ .Values.namespace }}
|
||||
labels:
|
||||
app: hive-metastore
|
||||
{{- include "stonks.labels" . | nindent 4 }}
|
||||
stonks-oracle/tier: analytics
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: hive-metastore
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: hive-metastore
|
||||
stonks-oracle/tier: analytics
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
{{- if .Values.imagePullSecrets.enabled }}
|
||||
imagePullSecrets:
|
||||
- name: {{ .Values.imagePullSecrets.name }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
- name: hive-config-init
|
||||
image: busybox:1.36
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
command: ["sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
cat > /hive-config/core-site.xml <<EOF
|
||||
<?xml version="1.0"?>
|
||||
<configuration>
|
||||
<property><name>fs.s3a.endpoint</name><value>http://minio.minio-service.svc.cluster.local:80</value></property>
|
||||
<property><name>fs.s3a.access.key</name><value>${MINIO_ACCESS_KEY}</value></property>
|
||||
<property><name>fs.s3a.secret.key</name><value>${MINIO_SECRET_KEY}</value></property>
|
||||
<property><name>fs.s3a.path.style.access</name><value>true</value></property>
|
||||
<property><name>fs.s3a.impl</name><value>org.apache.hadoop.fs.s3a.S3AFileSystem</value></property>
|
||||
<property><name>fs.s3a.connection.ssl.enabled</name><value>false</value></property>
|
||||
</configuration>
|
||||
EOF
|
||||
cat > /hive-config/metastore-site.xml <<EOF
|
||||
<?xml version="1.0"?>
|
||||
<configuration>
|
||||
<property><name>metastore.thrift.uris</name><value>thrift://0.0.0.0:9083</value></property>
|
||||
<property><name>metastore.task.threads.always</name><value>org.apache.hadoop.hive.metastore.events.EventCleanerTask</value></property>
|
||||
<property><name>metastore.expression.proxy</name><value>org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy</value></property>
|
||||
<property><name>javax.jdo.option.ConnectionDriverName</name><value>org.apache.derby.jdbc.EmbeddedDriver</value></property>
|
||||
<property><name>javax.jdo.option.ConnectionURL</name><value>jdbc:derby:/opt/hive/data/metastore_db;create=true</value></property>
|
||||
<property><name>metastore.warehouse.dir</name><value>s3a://stonks-lakehouse/warehouse</value></property>
|
||||
</configuration>
|
||||
EOF
|
||||
env:
|
||||
- name: MINIO_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stonks-core-secrets
|
||||
key: MINIO_ACCESS_KEY
|
||||
- name: MINIO_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stonks-core-secrets
|
||||
key: MINIO_SECRET_KEY
|
||||
volumeMounts:
|
||||
- name: hive-config
|
||||
mountPath: /hive-config
|
||||
containers:
|
||||
- name: hive-metastore
|
||||
image: apache/hive:4.0.0
|
||||
ports:
|
||||
- containerPort: 9083
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
env:
|
||||
- name: SERVICE_NAME
|
||||
value: metastore
|
||||
- name: DB_DRIVER
|
||||
value: derby
|
||||
volumeMounts:
|
||||
- name: hive-data
|
||||
mountPath: /opt/hive/data
|
||||
- name: hive-config
|
||||
mountPath: /opt/hive/conf/core-site.xml
|
||||
subPath: core-site.xml
|
||||
- name: hive-config
|
||||
mountPath: /opt/hive/conf/metastore-site.xml
|
||||
subPath: metastore-site.xml
|
||||
resources:
|
||||
requests:
|
||||
cpu: {{ .Values.hiveMetastore.resources.requests.cpu }}
|
||||
memory: {{ .Values.hiveMetastore.resources.requests.memory }}
|
||||
limits:
|
||||
cpu: {{ .Values.hiveMetastore.resources.limits.cpu | quote }}
|
||||
memory: {{ .Values.hiveMetastore.resources.limits.memory }}
|
||||
volumes:
|
||||
- name: hive-data
|
||||
persistentVolumeClaim:
|
||||
claimName: hive-metastore-data
|
||||
- name: hive-config
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: hive-metastore
|
||||
namespace: {{ .Values.namespace }}
|
||||
spec:
|
||||
selector:
|
||||
app: hive-metastore
|
||||
ports:
|
||||
- port: 9083
|
||||
targetPort: 9083
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: hive-metastore-data
|
||||
namespace: {{ .Values.namespace }}
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.hiveMetastore.storageSize }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,29 @@
|
||||
{{- if .Values.ingress.enabled }}
|
||||
{{- range $key, $ing := .Values.ingress.hosts }}
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: stonks-{{ $key }}
|
||||
namespace: {{ $.Values.namespace }}
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: {{ $.Values.ingress.clusterIssuer }}
|
||||
spec:
|
||||
ingressClassName: {{ $.Values.ingress.className }}
|
||||
tls:
|
||||
- hosts:
|
||||
- {{ $ing.host }}
|
||||
secretName: stonks-{{ $key }}-tls
|
||||
rules:
|
||||
- host: {{ $ing.host }}
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ $ing.service }}
|
||||
port:
|
||||
number: {{ $ing.port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: {{ .Values.namespace }}
|
||||
labels:
|
||||
{{- include "stonks.labels" . | nindent 4 }}
|
||||
kubernetes.io/metadata.name: {{ .Values.namespace }}
|
||||
@@ -0,0 +1,164 @@
|
||||
{{- if .Values.networkPolicies.enabled }}
|
||||
# Default deny all ingress
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny-ingress
|
||||
namespace: {{ .Values.namespace }}
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
---
|
||||
# Query API: accept from Traefik ingress
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-query-api-ingress
|
||||
namespace: {{ .Values.namespace }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: query-api
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8000
|
||||
---
|
||||
# Symbol Registry API: accept from Traefik ingress
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-symbol-registry-ingress
|
||||
namespace: {{ .Values.namespace }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: symbol-registry-api
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8000
|
||||
---
|
||||
# Risk Engine: accept from broker-adapter and query-api
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-risk-engine-ingress
|
||||
namespace: {{ .Values.namespace }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: risk-engine
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: broker-adapter
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: query-api
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8000
|
||||
---
|
||||
# Superset: accept from Traefik ingress
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-superset-ingress
|
||||
namespace: {{ .Values.namespace }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: superset
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8088
|
||||
---
|
||||
# Trino: accept from Superset, query-api, and Traefik
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-trino-ingress
|
||||
namespace: {{ .Values.namespace }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: trino
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: superset
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: query-api
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8080
|
||||
---
|
||||
# Hive Metastore: accept from Trino and lake-publisher
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-hive-metastore-ingress
|
||||
namespace: {{ .Values.namespace }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: hive-metastore
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: trino
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: lake-publisher
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 9083
|
||||
---
|
||||
# Broker adapter: isolated — no inbound
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: deny-broker-adapter-ingress
|
||||
namespace: {{ .Values.namespace }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: broker-adapter
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress: []
|
||||
{{- end }}
|
||||
@@ -0,0 +1,51 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: stonks-core-secrets
|
||||
namespace: {{ .Values.namespace }}
|
||||
labels:
|
||||
{{- include "stonks.labels" . | nindent 4 }}
|
||||
type: Opaque
|
||||
stringData:
|
||||
{{- range $key, $val := .Values.secrets.core }}
|
||||
{{ $key }}: {{ $val | quote }}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: stonks-broker-secrets
|
||||
namespace: {{ .Values.namespace }}
|
||||
labels:
|
||||
{{- include "stonks.labels" . | nindent 4 }}
|
||||
type: Opaque
|
||||
stringData:
|
||||
{{- range $key, $val := .Values.secrets.broker }}
|
||||
{{ $key }}: {{ $val | quote }}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: stonks-market-secrets
|
||||
namespace: {{ .Values.namespace }}
|
||||
labels:
|
||||
{{- include "stonks.labels" . | nindent 4 }}
|
||||
type: Opaque
|
||||
stringData:
|
||||
{{- range $key, $val := .Values.secrets.market }}
|
||||
{{ $key }}: {{ $val | quote }}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: stonks-dashboard-secrets
|
||||
namespace: {{ .Values.namespace }}
|
||||
labels:
|
||||
{{- include "stonks.labels" . | nindent 4 }}
|
||||
type: Opaque
|
||||
stringData:
|
||||
{{- range $key, $val := .Values.secrets.dashboard }}
|
||||
{{ $key }}: {{ $val | quote }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,16 @@
|
||||
{{- range $name, $svc := .Values.services }}
|
||||
{{- if $svc.port }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
namespace: {{ $.Values.namespace }}
|
||||
spec:
|
||||
selector:
|
||||
app: {{ $name }}
|
||||
ports:
|
||||
- port: {{ $svc.port }}
|
||||
targetPort: {{ $svc.port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,145 @@
|
||||
{{- if .Values.superset.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: superset
|
||||
namespace: {{ .Values.namespace }}
|
||||
labels:
|
||||
app: superset
|
||||
{{- include "stonks.labels" . | nindent 4 }}
|
||||
stonks-oracle/tier: dashboard
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: superset
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: superset
|
||||
stonks-oracle/tier: dashboard
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
{{- if .Values.imagePullSecrets.enabled }}
|
||||
imagePullSecrets:
|
||||
- name: {{ .Values.imagePullSecrets.name }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: superset
|
||||
image: apache/superset:latest
|
||||
ports:
|
||||
- containerPort: 8088
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
env:
|
||||
- name: SUPERSET_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stonks-dashboard-secrets
|
||||
key: SUPERSET_SECRET_KEY
|
||||
- name: ADMIN_USERNAME
|
||||
value: admin
|
||||
- name: ADMIN_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stonks-dashboard-secrets
|
||||
key: SUPERSET_ADMIN_PASSWORD
|
||||
- name: ADMIN_EMAIL
|
||||
value: admin@stonks.local
|
||||
volumeMounts:
|
||||
- name: superset-home
|
||||
mountPath: /app/superset_home
|
||||
- name: superset-config
|
||||
mountPath: /app/pythonpath/superset_config.py
|
||||
subPath: superset_config.py
|
||||
resources:
|
||||
requests:
|
||||
cpu: {{ .Values.superset.resources.requests.cpu }}
|
||||
memory: {{ .Values.superset.resources.requests.memory }}
|
||||
limits:
|
||||
cpu: {{ .Values.superset.resources.limits.cpu | quote }}
|
||||
memory: {{ .Values.superset.resources.limits.memory }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8088
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 15
|
||||
volumes:
|
||||
- name: superset-home
|
||||
persistentVolumeClaim:
|
||||
claimName: superset-data
|
||||
- name: superset-config
|
||||
configMap:
|
||||
name: superset-config
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: superset
|
||||
namespace: {{ .Values.namespace }}
|
||||
spec:
|
||||
selector:
|
||||
app: superset
|
||||
ports:
|
||||
- port: 8088
|
||||
targetPort: 8088
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: superset-data
|
||||
namespace: {{ .Values.namespace }}
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.superset.storageSize }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: superset-config
|
||||
namespace: {{ .Values.namespace }}
|
||||
data:
|
||||
superset_config.py: |
|
||||
import os
|
||||
SECRET_KEY = os.getenv("SUPERSET_SECRET_KEY", "stonks-dev-secret-key-change-me")
|
||||
SQLALCHEMY_DATABASE_URI = "trino://trino@trino.{{ .Values.namespace }}.svc.cluster.local:8080/lakehouse/stonks"
|
||||
FEATURE_FLAGS = {"ENABLE_TEMPLATE_PROCESSING": True}
|
||||
CACHE_CONFIG = {
|
||||
"CACHE_TYPE": "RedisCache",
|
||||
"CACHE_DEFAULT_TIMEOUT": 300,
|
||||
"CACHE_KEY_PREFIX": "superset_",
|
||||
"CACHE_REDIS_HOST": os.getenv("REDIS_HOST", "redis-master.redis-service.svc.cluster.local"),
|
||||
"CACHE_REDIS_PORT": int(os.getenv("REDIS_PORT", "6379")),
|
||||
"CACHE_REDIS_DB": 1,
|
||||
}
|
||||
PUBLIC_ROLE_LIKE = None
|
||||
SESSION_COOKIE_HTTPONLY = True
|
||||
SESSION_COOKIE_SECURE = True
|
||||
SESSION_COOKIE_SAMESITE = "Lax"
|
||||
TALISMAN_ENABLED = True
|
||||
TALISMAN_CONFIG = {
|
||||
"content_security_policy": {
|
||||
"default-src": ["'self'"],
|
||||
"img-src": ["'self'", "data:"],
|
||||
"style-src": ["'self'", "'unsafe-inline'"],
|
||||
"script-src": ["'self'", "'unsafe-inline'", "'unsafe-eval'"],
|
||||
},
|
||||
"force_https": False,
|
||||
}
|
||||
PREVENT_UNSAFE_DB_CONNECTIONS = True
|
||||
ROW_LIMIT = 50000
|
||||
SQL_MAX_ROW = 100000
|
||||
{{- end }}
|
||||
@@ -0,0 +1,122 @@
|
||||
{{- if .Values.trino.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: trino
|
||||
namespace: {{ .Values.namespace }}
|
||||
labels:
|
||||
app: trino
|
||||
{{- include "stonks.labels" . | nindent 4 }}
|
||||
stonks-oracle/tier: analytics
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: trino
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: trino
|
||||
stonks-oracle/tier: analytics
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
{{- if .Values.imagePullSecrets.enabled }}
|
||||
imagePullSecrets:
|
||||
- name: {{ .Values.imagePullSecrets.name }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
- name: catalog-init
|
||||
image: busybox:1.36
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
command: ["sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
cat > /catalog/iceberg.properties <<EOF
|
||||
connector.name=iceberg
|
||||
iceberg.catalog.type=hive_metastore
|
||||
hive.metastore.uri=thrift://hive-metastore.{{ $.Values.namespace }}.svc.cluster.local:9083
|
||||
hive.s3.endpoint=http://minio.minio-service.svc.cluster.local:80
|
||||
hive.s3.path-style-access=true
|
||||
hive.s3.aws-access-key=${MINIO_ACCESS_KEY}
|
||||
hive.s3.aws-secret-key=${MINIO_SECRET_KEY}
|
||||
fs.native-s3.enabled=true
|
||||
s3.endpoint=http://minio.minio-service.svc.cluster.local:80
|
||||
s3.path-style-access=true
|
||||
s3.aws-access-key=${MINIO_ACCESS_KEY}
|
||||
s3.aws-secret-key=${MINIO_SECRET_KEY}
|
||||
EOF
|
||||
cat > /catalog/lakehouse.properties <<EOF
|
||||
connector.name=hive
|
||||
hive.metastore.uri=thrift://hive-metastore.{{ $.Values.namespace }}.svc.cluster.local:9083
|
||||
hive.s3.endpoint=http://minio.minio-service.svc.cluster.local:80
|
||||
hive.s3.path-style-access=true
|
||||
hive.s3.aws-access-key=${MINIO_ACCESS_KEY}
|
||||
hive.s3.aws-secret-key=${MINIO_SECRET_KEY}
|
||||
hive.non-managed-table-writes-enabled=true
|
||||
hive.s3select-pushdown.enabled=true
|
||||
EOF
|
||||
env:
|
||||
- name: MINIO_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stonks-core-secrets
|
||||
key: MINIO_ACCESS_KEY
|
||||
- name: MINIO_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stonks-core-secrets
|
||||
key: MINIO_SECRET_KEY
|
||||
volumeMounts:
|
||||
- name: catalog-config
|
||||
mountPath: /catalog
|
||||
containers:
|
||||
- name: trino
|
||||
image: trinodb/trino:latest
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumeMounts:
|
||||
- name: catalog-config
|
||||
mountPath: /etc/trino/catalog
|
||||
resources:
|
||||
requests:
|
||||
cpu: {{ .Values.trino.resources.requests.cpu }}
|
||||
memory: {{ .Values.trino.resources.requests.memory }}
|
||||
limits:
|
||||
cpu: {{ .Values.trino.resources.limits.cpu | quote }}
|
||||
memory: {{ .Values.trino.resources.limits.memory }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /v1/info
|
||||
port: 8080
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
volumes:
|
||||
- name: catalog-config
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: trino
|
||||
namespace: {{ .Values.namespace }}
|
||||
spec:
|
||||
selector:
|
||||
app: trino
|
||||
ports:
|
||||
- port: 8080
|
||||
targetPort: 8080
|
||||
{{- end }}
|
||||
@@ -0,0 +1,243 @@
|
||||
## Stonks Oracle Helm Values
|
||||
|
||||
namespace: stonks-oracle
|
||||
|
||||
imageRegistry: ghcr.io/celesrenata/stonks-oracle
|
||||
imageTag: latest
|
||||
imagePullPolicy: Always
|
||||
|
||||
## GHCR image pull secret
|
||||
imagePullSecrets:
|
||||
enabled: true
|
||||
name: ghcr-credentials
|
||||
# Set to true to have Helm manage the secret. If false, create it
|
||||
# manually with: kubectl create secret docker-registry ghcr-credentials ...
|
||||
createSecret: false
|
||||
# Base64-encoded .dockerconfigjson — only used when createSecret is true
|
||||
dockerconfigjson: ""
|
||||
|
||||
## ── Services ──────────────────────────────────────────────────────────
|
||||
## Each service maps to a Deployment. Services with `port` also get a
|
||||
## ClusterIP Service resource. `secrets` lists which Secret names to
|
||||
## mount via envFrom.
|
||||
|
||||
services:
|
||||
scheduler:
|
||||
image: scheduler
|
||||
replicas: 1
|
||||
tier: orchestration
|
||||
secrets: [stonks-core-secrets]
|
||||
resources:
|
||||
requests: { cpu: 50m, memory: 64Mi }
|
||||
limits: { cpu: 200m, memory: 128Mi }
|
||||
|
||||
symbol-registry-api:
|
||||
image: symbol-registry
|
||||
replicas: 1
|
||||
tier: api
|
||||
port: 8000
|
||||
secrets: [stonks-core-secrets]
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 128Mi }
|
||||
limits: { cpu: 500m, memory: 256Mi }
|
||||
probes:
|
||||
readiness: { path: /docs, port: 8000, initialDelay: 5, period: 10 }
|
||||
liveness: { path: /docs, port: 8000, initialDelay: 10, period: 30 }
|
||||
|
||||
ingestion-worker:
|
||||
image: ingestion
|
||||
replicas: 2
|
||||
tier: ingestion
|
||||
secrets: [stonks-core-secrets, stonks-market-secrets]
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 128Mi }
|
||||
limits: { cpu: 500m, memory: 256Mi }
|
||||
|
||||
parser-worker:
|
||||
image: parser
|
||||
replicas: 2
|
||||
tier: processing
|
||||
secrets: [stonks-core-secrets]
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 128Mi }
|
||||
limits: { cpu: 500m, memory: 256Mi }
|
||||
|
||||
extractor-worker:
|
||||
image: extractor
|
||||
replicas: 1
|
||||
tier: processing
|
||||
secrets: [stonks-core-secrets]
|
||||
resources:
|
||||
requests: { cpu: 200m, memory: 256Mi }
|
||||
limits: { cpu: "1", memory: 512Mi }
|
||||
|
||||
aggregation-worker:
|
||||
image: aggregation
|
||||
replicas: 1
|
||||
tier: processing
|
||||
secrets: [stonks-core-secrets]
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 128Mi }
|
||||
limits: { cpu: 500m, memory: 256Mi }
|
||||
|
||||
recommendation-worker:
|
||||
image: recommendation
|
||||
replicas: 1
|
||||
tier: processing
|
||||
secrets: [stonks-core-secrets]
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 128Mi }
|
||||
limits: { cpu: 500m, memory: 256Mi }
|
||||
|
||||
risk-engine:
|
||||
image: risk
|
||||
replicas: 1
|
||||
tier: trading
|
||||
port: 8000
|
||||
secrets: [stonks-core-secrets, stonks-broker-secrets]
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 128Mi }
|
||||
limits: { cpu: 500m, memory: 256Mi }
|
||||
|
||||
broker-adapter:
|
||||
image: broker-adapter
|
||||
replicas: 1
|
||||
tier: trading
|
||||
secrets: [stonks-core-secrets, stonks-broker-secrets]
|
||||
resources:
|
||||
requests: { cpu: 50m, memory: 64Mi }
|
||||
limits: { cpu: 200m, memory: 128Mi }
|
||||
|
||||
lake-publisher:
|
||||
image: lake-publisher
|
||||
replicas: 1
|
||||
tier: analytics
|
||||
secrets: [stonks-core-secrets]
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 128Mi }
|
||||
limits: { cpu: 500m, memory: 256Mi }
|
||||
|
||||
query-api:
|
||||
image: query-api
|
||||
replicas: 1
|
||||
tier: api
|
||||
port: 8000
|
||||
secrets: [stonks-core-secrets]
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 128Mi }
|
||||
limits: { cpu: 500m, memory: 256Mi }
|
||||
probes:
|
||||
readiness: { path: /docs, port: 8000, initialDelay: 5, period: 10 }
|
||||
|
||||
## ── ConfigMap data ────────────────────────────────────────────────────
|
||||
config:
|
||||
POSTGRES_HOST: "postgresql-rw.postgresql-service.svc.cluster.local"
|
||||
POSTGRES_PORT: "5432"
|
||||
POSTGRES_DB: "stonks"
|
||||
POSTGRES_USER: "stonks"
|
||||
REDIS_HOST: "redis-master.redis-service.svc.cluster.local"
|
||||
REDIS_PORT: "6379"
|
||||
REDIS_DB: "0"
|
||||
MINIO_ENDPOINT: "minio.minio-service.svc.cluster.local:80"
|
||||
MINIO_SECURE: "false"
|
||||
OLLAMA_BASE_URL: "http://ollama.ollama-service.svc.cluster.local:11434"
|
||||
OLLAMA_MODEL: "llama3.1:8b"
|
||||
OLLAMA_TIMEOUT: "120"
|
||||
OLLAMA_MAX_RETRIES: "2"
|
||||
OLLAMA_RETRY_BASE_DELAY: "1.0"
|
||||
OLLAMA_RETRY_MAX_DELAY: "10.0"
|
||||
OLLAMA_RETRY_BACKOFF_MULTIPLIER: "2.0"
|
||||
TRINO_HOST: "trino.stonks-oracle.svc.cluster.local"
|
||||
TRINO_PORT: "8080"
|
||||
TRINO_CATALOG: "lakehouse"
|
||||
TRINO_SCHEMA: "stonks"
|
||||
TRINO_ICEBERG_CATALOG: "iceberg"
|
||||
BROKER_MODE: "paper"
|
||||
BROKER_PROVIDER: "alpaca"
|
||||
MARKET_DATA_BASE_URL: "https://api.polygon.io"
|
||||
MARKET_DATA_PROVIDER: "polygon"
|
||||
RETENTION_RAW_MARKET_DAYS: "90"
|
||||
RETENTION_RAW_NEWS_DAYS: "180"
|
||||
RETENTION_RAW_FILINGS_DAYS: "365"
|
||||
RETENTION_NORMALIZED_DAYS: "180"
|
||||
RETENTION_LLM_PROMPTS_DAYS: "365"
|
||||
RETENTION_LLM_RESULTS_DAYS: "365"
|
||||
RETENTION_LAKEHOUSE_DAYS: "730"
|
||||
RETENTION_AUDIT_DAYS: "730"
|
||||
RETENTION_CLEANUP_INTERVAL_HOURS: "24"
|
||||
RETENTION_BATCH_SIZE: "1000"
|
||||
LOG_LEVEL: "INFO"
|
||||
JSON_LOGS: "true"
|
||||
ALERT_SOURCE_FAILURE_THRESHOLD: "3"
|
||||
ALERT_SOURCE_FAILURE_WINDOW_HOURS: "6"
|
||||
ALERT_SCHEMA_FAILURE_RATE_THRESHOLD: "0.3"
|
||||
ALERT_SCHEMA_FAILURE_WINDOW_HOURS: "1"
|
||||
ALERT_LAKE_LAG_THRESHOLD_MINUTES: "60"
|
||||
ALERT_BROKER_ERROR_THRESHOLD: "3"
|
||||
ALERT_BROKER_ERROR_WINDOW_HOURS: "1"
|
||||
ALERT_CHECK_INTERVAL_SECONDS: "120"
|
||||
|
||||
## ── Secrets ───────────────────────────────────────────────────────────
|
||||
## Values here are placeholders. Override via --set or a secrets file.
|
||||
secrets:
|
||||
core:
|
||||
POSTGRES_PASSWORD: "REPLACE_ME"
|
||||
MINIO_ACCESS_KEY: "REPLACE_ME"
|
||||
MINIO_SECRET_KEY: "REPLACE_ME"
|
||||
REDIS_PASSWORD: ""
|
||||
broker:
|
||||
BROKER_API_KEY: "REPLACE_ME"
|
||||
BROKER_API_SECRET: "REPLACE_ME"
|
||||
BROKER_BASE_URL: "https://paper-api.alpaca.markets"
|
||||
market:
|
||||
MARKET_DATA_API_KEY: "REPLACE_ME"
|
||||
dashboard:
|
||||
SUPERSET_SECRET_KEY: "REPLACE_ME"
|
||||
SUPERSET_ADMIN_PASSWORD: "REPLACE_ME"
|
||||
|
||||
## ── Ingress ───────────────────────────────────────────────────────────
|
||||
ingress:
|
||||
enabled: true
|
||||
className: traefik
|
||||
clusterIssuer: ca-issuer
|
||||
hosts:
|
||||
query-api:
|
||||
host: stonks-api.celestium.life
|
||||
service: query-api
|
||||
port: 8000
|
||||
registry:
|
||||
host: stonks-registry.celestium.life
|
||||
service: symbol-registry-api
|
||||
port: 8000
|
||||
superset:
|
||||
host: stonks-dash.celestium.life
|
||||
service: superset
|
||||
port: 8088
|
||||
trino:
|
||||
host: stonks-trino.celestium.life
|
||||
service: trino
|
||||
port: 8080
|
||||
|
||||
## ── Analytics stack ───────────────────────────────────────────────────
|
||||
trino:
|
||||
enabled: true
|
||||
resources:
|
||||
requests: { cpu: 500m, memory: 1Gi }
|
||||
limits: { cpu: "2", memory: 4Gi }
|
||||
|
||||
hiveMetastore:
|
||||
enabled: true
|
||||
storageSize: 1Gi
|
||||
resources:
|
||||
requests: { cpu: 200m, memory: 512Mi }
|
||||
limits: { cpu: "1", memory: 1Gi }
|
||||
|
||||
superset:
|
||||
enabled: true
|
||||
storageSize: 2Gi
|
||||
resources:
|
||||
requests: { cpu: 200m, memory: 512Mi }
|
||||
limits: { cpu: "1", memory: 2Gi }
|
||||
|
||||
networkPolicies:
|
||||
enabled: true
|
||||
Reference in New Issue
Block a user