admin/stonks-oracle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: risk engine command points to services.risk.app, redis password, window quoting

Browse Source
This commit is contained in:
Celes Renata
2026-04-11 14:15:47 -07:00
parent 1c3a5f8285
commit 273a86e2bd
17 changed files with 237 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build.yml
+1 -1
View File
@@ -55,7 +55,7 @@ jobs:
- name: recommendation - name: recommendation
cmd: "python -m services.recommendation.worker" cmd: "python -m services.recommendation.worker"
- name: risk - name: risk
cmd: "uvicorn services.risk.engine:app --host 0.0.0.0 --port 8000" cmd: "uvicorn services.risk.app:app --host 0.0.0.0 --port 8000"
- name: broker-adapter - name: broker-adapter
cmd: "python -m services.adapters.broker_adapter" cmd: "python -m services.adapters.broker_adapter"
- name: lake-publisher - name: lake-publisher
.kiro/specs/stonks-oracle/tasks.md
+1 -1
View File
@@ -146,7 +146,7 @@
- Monitor the GitHub Actions run to confirm lint-and-test and build-services jobs succeed - Monitor the GitHub Actions run to confirm lint-and-test and build-services jobs succeed
- Fix any CI failures and re-push if needed - Fix any CI failures and re-push if needed
- _Requirements: N1_ - _Requirements: N1_
- [x] 15.3 Create Helm chart for stonks-oracle deployment - [-] 15.3 Create Helm chart for stonks-oracle deployment
- Create `infra/helm/stonks-oracle/Chart.yaml` with chart metadata - Create `infra/helm/stonks-oracle/Chart.yaml` with chart metadata
- Create `infra/helm/stonks-oracle/values.yaml` with configurable image tags, replica counts, resource limits, and environment references - Create `infra/helm/stonks-oracle/values.yaml` with configurable image tags, replica counts, resource limits, and environment references
- Create Helm templates for all deployments, services, configmap, secrets, ingress, and network policies from existing K8s manifests - Create Helm templates for all deployments, services, configmap, secrets, ingress, and network policies from existing K8s manifests
infra/helm/stonks-oracle/Chart.yaml
-3
View File
@@ -4,6 +4,3 @@ description: AI market intelligence and paper-trading platform
type: application type: application
version: 0.1.0 version: 0.1.0
appVersion: "1.0.0" appVersion: "1.0.0"
maintainers:
- name: Celes Renata
email: celes@frameshift.net
infra/helm/stonks-oracle/templates/_helpers.tpl
+18 -3
View File
@@ -8,8 +8,23 @@ helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
{{- end }} {{- end }}
{{/* {{/*
Full image path for a service Pod security context — shared across all custom service pods
*/}} */}}
{{- define "stonks.image" -}} {{- define "stonks.podSecurityContext" -}}
{{ $.Values.imageRegistry }}/{{ .image }}:{{ $.Values.imageTag }} runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
{{- end }}
{{/*
Container security context
*/}}
{{- define "stonks.containerSecurityContext" -}}
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
{{- end }} {{- end }}
infra/helm/stonks-oracle/templates/configmap.yaml
+1 -1
View File
@@ -2,7 +2,7 @@ apiVersion: v1
kind: ConfigMap kind: ConfigMap
metadata: metadata:
name: stonks-config name: stonks-config
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
labels: labels:
{{- include "stonks.labels" . | nindent 4 }} {{- include "stonks.labels" . | nindent 4 }}
data: data:
infra/helm/stonks-oracle/templates/deployments.yaml
+15 -27
View File
@@ -1,50 +1,43 @@
{{- $root := . -}}
{{- range $name, $svc := .Values.services }} {{- range $name, $svc := .Values.services }}
--- ---
apiVersion: apps/v1 apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
name: {{ $name }} name: {{ $svc.image }}
namespace: {{ $.Values.namespace }} namespace: {{ $root.Release.Namespace }}
labels: labels:
app: {{ $name }} app: {{ $svc.image }}
{{- include "stonks.labels" $ | nindent 4 }} {{- include "stonks.labels" $root | nindent 4 }}
stonks-oracle/tier: {{ $svc.tier }} stonks-oracle/tier: {{ $svc.tier }}
spec: spec:
replicas: {{ $svc.replicas }} replicas: {{ $svc.replicas }}
selector: selector:
matchLabels: matchLabels:
app: {{ $name }} app: {{ $svc.image }}
template: template:
metadata: metadata:
labels: labels:
app: {{ $name }} app: {{ $svc.image }}
stonks-oracle/tier: {{ $svc.tier }} stonks-oracle/tier: {{ $svc.tier }}
spec: spec:
automountServiceAccountToken: false automountServiceAccountToken: false
{{- if $.Values.imagePullSecrets.enabled }} {{- with $root.Values.imagePullSecrets }}
imagePullSecrets: imagePullSecrets:
- name: {{ $.Values.imagePullSecrets.name }} {{- toYaml . | nindent 8 }}
{{- end }} {{- end }}
securityContext: securityContext:
runAsNonRoot: true {{- include "stonks.podSecurityContext" $root | nindent 8 }}
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers: containers:
- name: {{ $name }} - name: {{ $svc.image }}
image: {{ $.Values.imageRegistry }}/{{ $svc.image }}:{{ $.Values.imageTag }} image: {{ $root.Values.image.registry }}/{{ $svc.image }}:{{ $root.Values.image.tag }}
imagePullPolicy: {{ $.Values.imagePullPolicy }} imagePullPolicy: {{ $root.Values.image.pullPolicy }}
{{- if $svc.port }} {{- if $svc.port }}
ports: ports:
- containerPort: {{ $svc.port }} - containerPort: {{ $svc.port }}
{{- end }} {{- end }}
securityContext: securityContext:
allowPrivilegeEscalation: false {{- include "stonks.containerSecurityContext" $root | nindent 12 }}
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
envFrom: envFrom:
- configMapRef: - configMapRef:
name: stonks-config name: stonks-config
@@ -53,12 +46,7 @@ spec:
name: {{ . }} name: {{ . }}
{{- end }} {{- end }}
resources: resources:
requests: {{- toYaml $svc.resources | nindent 12 }}
cpu: {{ $svc.resources.requests.cpu }}
memory: {{ $svc.resources.requests.memory }}
limits:
cpu: {{ $svc.resources.limits.cpu | quote }}
memory: {{ $svc.resources.limits.memory }}
{{- if $svc.probes }} {{- if $svc.probes }}
{{- if $svc.probes.readiness }} {{- if $svc.probes.readiness }}
readinessProbe: readinessProbe:
infra/helm/stonks-oracle/templates/ghcr-secret.yaml
+4 -4
View File
@@ -1,12 +1,12 @@
{{- if and .Values.imagePullSecrets.enabled .Values.imagePullSecrets.createSecret }} {{- if .Values.ghcrAuth.enabled }}
apiVersion: v1 apiVersion: v1
kind: Secret kind: Secret
metadata: metadata:
name: {{ .Values.imagePullSecrets.name }} name: ghcr-credentials
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
labels: labels:
{{- include "stonks.labels" . | nindent 4 }} {{- include "stonks.labels" . | nindent 4 }}
type: kubernetes.io/dockerconfigjson type: kubernetes.io/dockerconfigjson
data: data:
.dockerconfigjson: {{ .Values.imagePullSecrets.dockerconfigjson }} .dockerconfigjson: {{ printf `{"auths":{"%s":{"username":"%s","password":"%s","auth":"%s"}}}` .Values.ghcrAuth.registry .Values.ghcrAuth.username .Values.ghcrAuth.password (printf "%s:%s" .Values.ghcrAuth.username .Values.ghcrAuth.password | b64enc) | b64enc }}
{{- end }} {{- end }}
infra/helm/stonks-oracle/templates/hive-metastore.yaml
+8 -18
View File
@@ -3,7 +3,7 @@ apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
name: hive-metastore name: hive-metastore
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
labels: labels:
app: hive-metastore app: hive-metastore
{{- include "stonks.labels" . | nindent 4 }} {{- include "stonks.labels" . | nindent 4 }}
@@ -20,17 +20,12 @@ spec:
stonks-oracle/tier: analytics stonks-oracle/tier: analytics
spec: spec:
automountServiceAccountToken: false automountServiceAccountToken: false
{{- if .Values.imagePullSecrets.enabled }} {{- with .Values.imagePullSecrets }}
imagePullSecrets: imagePullSecrets:
- name: {{ .Values.imagePullSecrets.name }} {{- toYaml . | nindent 8 }}
{{- end }} {{- end }}
securityContext: securityContext:
runAsNonRoot: true {{- include "stonks.podSecurityContext" . | nindent 8 }}
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
initContainers: initContainers:
- name: hive-config-init - name: hive-config-init
image: busybox:1.36 image: busybox:1.36
@@ -44,7 +39,7 @@ spec:
cat > /hive-config/core-site.xml <<EOF cat > /hive-config/core-site.xml <<EOF
<?xml version="1.0"?> <?xml version="1.0"?>
<configuration> <configuration>
<property><name>fs.s3a.endpoint</name><value>http://minio.minio-service.svc.cluster.local:80</value></property> <property><name>fs.s3a.endpoint</name><value>http://{{ index $.Values.config "MINIO_ENDPOINT" }}</value></property>
<property><name>fs.s3a.access.key</name><value>${MINIO_ACCESS_KEY}</value></property> <property><name>fs.s3a.access.key</name><value>${MINIO_ACCESS_KEY}</value></property>
<property><name>fs.s3a.secret.key</name><value>${MINIO_SECRET_KEY}</value></property> <property><name>fs.s3a.secret.key</name><value>${MINIO_SECRET_KEY}</value></property>
<property><name>fs.s3a.path.style.access</name><value>true</value></property> <property><name>fs.s3a.path.style.access</name><value>true</value></property>
@@ -101,12 +96,7 @@ spec:
mountPath: /opt/hive/conf/metastore-site.xml mountPath: /opt/hive/conf/metastore-site.xml
subPath: metastore-site.xml subPath: metastore-site.xml
resources: resources:
requests: {{- toYaml .Values.hiveMetastore.resources | nindent 12 }}
cpu: {{ .Values.hiveMetastore.resources.requests.cpu }}
memory: {{ .Values.hiveMetastore.resources.requests.memory }}
limits:
cpu: {{ .Values.hiveMetastore.resources.limits.cpu | quote }}
memory: {{ .Values.hiveMetastore.resources.limits.memory }}
volumes: volumes:
- name: hive-data - name: hive-data
persistentVolumeClaim: persistentVolumeClaim:
@@ -118,7 +108,7 @@ apiVersion: v1
kind: Service kind: Service
metadata: metadata:
name: hive-metastore name: hive-metastore
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
spec: spec:
selector: selector:
app: hive-metastore app: hive-metastore
@@ -130,7 +120,7 @@ apiVersion: v1
kind: PersistentVolumeClaim kind: PersistentVolumeClaim
metadata: metadata:
name: hive-metastore-data name: hive-metastore-data
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
spec: spec:
accessModes: accessModes:
- ReadWriteOnce - ReadWriteOnce
infra/helm/stonks-oracle/templates/ingress.yaml
+84 -11
View File
@@ -1,29 +1,102 @@
{{- if .Values.ingress.enabled }} {{- if .Values.ingress.enabled }}
{{- range $key, $ing := .Values.ingress.hosts }}
--- ---
apiVersion: networking.k8s.io/v1 apiVersion: networking.k8s.io/v1
kind: Ingress kind: Ingress
metadata: metadata:
name: stonks-{{ $key }} name: stonks-query-api-https
namespace: {{ $.Values.namespace }} namespace: {{ .Release.Namespace }}
annotations: annotations:
cert-manager.io/cluster-issuer: {{ $.Values.ingress.clusterIssuer }} cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }}
spec: spec:
ingressClassName: {{ $.Values.ingress.className }} ingressClassName: {{ .Values.ingress.className }}
tls: tls:
- hosts: - hosts:
- {{ $ing.host }} - {{ .Values.ingress.hosts.queryApi }}
secretName: stonks-{{ $key }}-tls secretName: stonks-api-tls
rules: rules:
- host: {{ $ing.host }} - host: {{ .Values.ingress.hosts.queryApi }}
http: http:
paths: paths:
- path: / - path: /
pathType: Prefix pathType: Prefix
backend: backend:
service: service:
name: {{ $ing.service }} name: query-api
port: port:
number: {{ $ing.port }} number: 8000
{{- end }} ---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: stonks-registry-https
namespace: {{ .Release.Namespace }}
annotations:
cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }}
spec:
ingressClassName: {{ .Values.ingress.className }}
tls:
- hosts:
- {{ .Values.ingress.hosts.symbolRegistry }}
secretName: stonks-registry-tls
rules:
- host: {{ .Values.ingress.hosts.symbolRegistry }}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: symbol-registry
port:
number: 8000
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: stonks-superset-https
namespace: {{ .Release.Namespace }}
annotations:
cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }}
spec:
ingressClassName: {{ .Values.ingress.className }}
tls:
- hosts:
- {{ .Values.ingress.hosts.superset }}
secretName: stonks-dash-tls
rules:
- host: {{ .Values.ingress.hosts.superset }}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: superset
port:
number: 8088
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: stonks-trino-https
namespace: {{ .Release.Namespace }}
annotations:
cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }}
spec:
ingressClassName: {{ .Values.ingress.className }}
tls:
- hosts:
- {{ .Values.ingress.hosts.trino }}
secretName: stonks-trino-tls
rules:
- host: {{ .Values.ingress.hosts.trino }}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: trino
port:
number: 8080
{{- end }} {{- end }}
infra/helm/stonks-oracle/templates/namespace.yaml
+2 -2
View File
@@ -1,7 +1,7 @@
apiVersion: v1 apiVersion: v1
kind: Namespace kind: Namespace
metadata: metadata:
name: {{ .Values.namespace }} name: {{ .Release.Namespace }}
labels: labels:
{{- include "stonks.labels" . | nindent 4 }} {{- include "stonks.labels" . | nindent 4 }}
kubernetes.io/metadata.name: {{ .Values.namespace }} kubernetes.io/metadata.name: {{ .Release.Namespace }}
infra/helm/stonks-oracle/templates/network-policies.yaml
+10 -18
View File
@@ -1,21 +1,19 @@
{{- if .Values.networkPolicies.enabled }} {{- if .Values.networkPolicies.enabled }}
# Default deny all ingress
apiVersion: networking.k8s.io/v1 apiVersion: networking.k8s.io/v1
kind: NetworkPolicy kind: NetworkPolicy
metadata: metadata:
name: default-deny-ingress name: default-deny-ingress
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
spec: spec:
podSelector: {} podSelector: {}
policyTypes: policyTypes:
- Ingress - Ingress
--- ---
# Query API: accept from Traefik ingress
apiVersion: networking.k8s.io/v1 apiVersion: networking.k8s.io/v1
kind: NetworkPolicy kind: NetworkPolicy
metadata: metadata:
name: allow-query-api-ingress name: allow-query-api-ingress
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
spec: spec:
podSelector: podSelector:
matchLabels: matchLabels:
@@ -31,16 +29,15 @@ spec:
- protocol: TCP - protocol: TCP
port: 8000 port: 8000
--- ---
# Symbol Registry API: accept from Traefik ingress
apiVersion: networking.k8s.io/v1 apiVersion: networking.k8s.io/v1
kind: NetworkPolicy kind: NetworkPolicy
metadata: metadata:
name: allow-symbol-registry-ingress name: allow-symbol-registry-ingress
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
spec: spec:
podSelector: podSelector:
matchLabels: matchLabels:
app: symbol-registry-api app: symbol-registry
policyTypes: policyTypes:
- Ingress - Ingress
ingress: ingress:
@@ -52,16 +49,15 @@ spec:
- protocol: TCP - protocol: TCP
port: 8000 port: 8000
--- ---
# Risk Engine: accept from broker-adapter and query-api
apiVersion: networking.k8s.io/v1 apiVersion: networking.k8s.io/v1
kind: NetworkPolicy kind: NetworkPolicy
metadata: metadata:
name: allow-risk-engine-ingress name: allow-risk-engine-ingress
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
spec: spec:
podSelector: podSelector:
matchLabels: matchLabels:
app: risk-engine app: risk
policyTypes: policyTypes:
- Ingress - Ingress
ingress: ingress:
@@ -76,12 +72,11 @@ spec:
- protocol: TCP - protocol: TCP
port: 8000 port: 8000
--- ---
# Superset: accept from Traefik ingress
apiVersion: networking.k8s.io/v1 apiVersion: networking.k8s.io/v1
kind: NetworkPolicy kind: NetworkPolicy
metadata: metadata:
name: allow-superset-ingress name: allow-superset-ingress
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
spec: spec:
podSelector: podSelector:
matchLabels: matchLabels:
@@ -97,12 +92,11 @@ spec:
- protocol: TCP - protocol: TCP
port: 8088 port: 8088
--- ---
# Trino: accept from Superset, query-api, and Traefik
apiVersion: networking.k8s.io/v1 apiVersion: networking.k8s.io/v1
kind: NetworkPolicy kind: NetworkPolicy
metadata: metadata:
name: allow-trino-ingress name: allow-trino-ingress
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
spec: spec:
podSelector: podSelector:
matchLabels: matchLabels:
@@ -124,12 +118,11 @@ spec:
- protocol: TCP - protocol: TCP
port: 8080 port: 8080
--- ---
# Hive Metastore: accept from Trino and lake-publisher
apiVersion: networking.k8s.io/v1 apiVersion: networking.k8s.io/v1
kind: NetworkPolicy kind: NetworkPolicy
metadata: metadata:
name: allow-hive-metastore-ingress name: allow-hive-metastore-ingress
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
spec: spec:
podSelector: podSelector:
matchLabels: matchLabels:
@@ -148,12 +141,11 @@ spec:
- protocol: TCP - protocol: TCP
port: 9083 port: 9083
--- ---
# Broker adapter: isolated — no inbound
apiVersion: networking.k8s.io/v1 apiVersion: networking.k8s.io/v1
kind: NetworkPolicy kind: NetworkPolicy
metadata: metadata:
name: deny-broker-adapter-ingress name: deny-broker-adapter-ingress
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
spec: spec:
podSelector: podSelector:
matchLabels: matchLabels:
infra/helm/stonks-oracle/templates/secrets.yaml
+4 -4
View File
@@ -2,7 +2,7 @@ apiVersion: v1
kind: Secret kind: Secret
metadata: metadata:
name: stonks-core-secrets name: stonks-core-secrets
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
labels: labels:
{{- include "stonks.labels" . | nindent 4 }} {{- include "stonks.labels" . | nindent 4 }}
type: Opaque type: Opaque
@@ -15,7 +15,7 @@ apiVersion: v1
kind: Secret kind: Secret
metadata: metadata:
name: stonks-broker-secrets name: stonks-broker-secrets
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
labels: labels:
{{- include "stonks.labels" . | nindent 4 }} {{- include "stonks.labels" . | nindent 4 }}
type: Opaque type: Opaque
@@ -28,7 +28,7 @@ apiVersion: v1
kind: Secret kind: Secret
metadata: metadata:
name: stonks-market-secrets name: stonks-market-secrets
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
labels: labels:
{{- include "stonks.labels" . | nindent 4 }} {{- include "stonks.labels" . | nindent 4 }}
type: Opaque type: Opaque
@@ -41,7 +41,7 @@ apiVersion: v1
kind: Secret kind: Secret
metadata: metadata:
name: stonks-dashboard-secrets name: stonks-dashboard-secrets
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
labels: labels:
{{- include "stonks.labels" . | nindent 4 }} {{- include "stonks.labels" . | nindent 4 }}
type: Opaque type: Opaque
infra/helm/stonks-oracle/templates/services.yaml
+4 -3
View File
@@ -1,14 +1,15 @@
{{- $root := . -}}
{{- range $name, $svc := .Values.services }} {{- range $name, $svc := .Values.services }}
{{- if $svc.port }} {{- if $svc.port }}
--- ---
apiVersion: v1 apiVersion: v1
kind: Service kind: Service
metadata: metadata:
name: {{ $name }} name: {{ $svc.image }}
namespace: {{ $.Values.namespace }} namespace: {{ $root.Release.Namespace }}
spec: spec:
selector: selector:
app: {{ $name }} app: {{ $svc.image }}
ports: ports:
- port: {{ $svc.port }} - port: {{ $svc.port }}
targetPort: {{ $svc.port }} targetPort: {{ $svc.port }}
infra/helm/stonks-oracle/templates/superset.yaml
+9 -19
View File
@@ -3,7 +3,7 @@ apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
name: superset name: superset
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
labels: labels:
app: superset app: superset
{{- include "stonks.labels" . | nindent 4 }} {{- include "stonks.labels" . | nindent 4 }}
@@ -20,17 +20,12 @@ spec:
stonks-oracle/tier: dashboard stonks-oracle/tier: dashboard
spec: spec:
automountServiceAccountToken: false automountServiceAccountToken: false
{{- if .Values.imagePullSecrets.enabled }} {{- with .Values.imagePullSecrets }}
imagePullSecrets: imagePullSecrets:
- name: {{ .Values.imagePullSecrets.name }} {{- toYaml . | nindent 8 }}
{{- end }} {{- end }}
securityContext: securityContext:
runAsNonRoot: true {{- include "stonks.podSecurityContext" . | nindent 8 }}
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers: containers:
- name: superset - name: superset
image: apache/superset:latest image: apache/superset:latest
@@ -62,12 +57,7 @@ spec:
mountPath: /app/pythonpath/superset_config.py mountPath: /app/pythonpath/superset_config.py
subPath: superset_config.py subPath: superset_config.py
resources: resources:
requests: {{- toYaml .Values.superset.resources | nindent 12 }}
cpu: {{ .Values.superset.resources.requests.cpu }}
memory: {{ .Values.superset.resources.requests.memory }}
limits:
cpu: {{ .Values.superset.resources.limits.cpu | quote }}
memory: {{ .Values.superset.resources.limits.memory }}
readinessProbe: readinessProbe:
httpGet: httpGet:
path: /health path: /health
@@ -86,7 +76,7 @@ apiVersion: v1
kind: Service kind: Service
metadata: metadata:
name: superset name: superset
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
spec: spec:
selector: selector:
app: superset app: superset
@@ -98,7 +88,7 @@ apiVersion: v1
kind: PersistentVolumeClaim kind: PersistentVolumeClaim
metadata: metadata:
name: superset-data name: superset-data
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
spec: spec:
accessModes: accessModes:
- ReadWriteOnce - ReadWriteOnce
@@ -110,12 +100,12 @@ apiVersion: v1
kind: ConfigMap kind: ConfigMap
metadata: metadata:
name: superset-config name: superset-config
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
data: data:
superset_config.py: | superset_config.py: |
import os import os
SECRET_KEY = os.getenv("SUPERSET_SECRET_KEY", "stonks-dev-secret-key-change-me") SECRET_KEY = os.getenv("SUPERSET_SECRET_KEY", "stonks-dev-secret-key-change-me")
SQLALCHEMY_DATABASE_URI = "trino://trino@trino.{{ .Values.namespace }}.svc.cluster.local:8080/lakehouse/stonks" SQLALCHEMY_DATABASE_URI = "trino://trino@trino.{{ .Release.Namespace }}.svc.cluster.local:8080/lakehouse/stonks"
FEATURE_FLAGS = {"ENABLE_TEMPLATE_PROCESSING": True} FEATURE_FLAGS = {"ENABLE_TEMPLATE_PROCESSING": True}
CACHE_CONFIG = { CACHE_CONFIG = {
"CACHE_TYPE": "RedisCache", "CACHE_TYPE": "RedisCache",
infra/helm/stonks-oracle/templates/trino.yaml
+11 -21
View File
@@ -3,7 +3,7 @@ apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
name: trino name: trino
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
labels: labels:
app: trino app: trino
{{- include "stonks.labels" . | nindent 4 }} {{- include "stonks.labels" . | nindent 4 }}
@@ -20,17 +20,12 @@ spec:
stonks-oracle/tier: analytics stonks-oracle/tier: analytics
spec: spec:
automountServiceAccountToken: false automountServiceAccountToken: false
{{- if .Values.imagePullSecrets.enabled }} {{- with .Values.imagePullSecrets }}
imagePullSecrets: imagePullSecrets:
- name: {{ .Values.imagePullSecrets.name }} {{- toYaml . | nindent 8 }}
{{- end }} {{- end }}
securityContext: securityContext:
runAsNonRoot: true {{- include "stonks.podSecurityContext" . | nindent 8 }}
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
initContainers: initContainers:
- name: catalog-init - name: catalog-init
image: busybox:1.36 image: busybox:1.36
@@ -44,21 +39,21 @@ spec:
cat > /catalog/iceberg.properties <<EOF cat > /catalog/iceberg.properties <<EOF
connector.name=iceberg connector.name=iceberg
iceberg.catalog.type=hive_metastore iceberg.catalog.type=hive_metastore
hive.metastore.uri=thrift://hive-metastore.{{ $.Values.namespace }}.svc.cluster.local:9083 hive.metastore.uri=thrift://hive-metastore.{{ $.Release.Namespace }}.svc.cluster.local:9083
hive.s3.endpoint=http://minio.minio-service.svc.cluster.local:80 hive.s3.endpoint=http://{{ index $.Values.config "MINIO_ENDPOINT" }}
hive.s3.path-style-access=true hive.s3.path-style-access=true
hive.s3.aws-access-key=${MINIO_ACCESS_KEY} hive.s3.aws-access-key=${MINIO_ACCESS_KEY}
hive.s3.aws-secret-key=${MINIO_SECRET_KEY} hive.s3.aws-secret-key=${MINIO_SECRET_KEY}
fs.native-s3.enabled=true fs.native-s3.enabled=true
s3.endpoint=http://minio.minio-service.svc.cluster.local:80 s3.endpoint=http://{{ index $.Values.config "MINIO_ENDPOINT" }}
s3.path-style-access=true s3.path-style-access=true
s3.aws-access-key=${MINIO_ACCESS_KEY} s3.aws-access-key=${MINIO_ACCESS_KEY}
s3.aws-secret-key=${MINIO_SECRET_KEY} s3.aws-secret-key=${MINIO_SECRET_KEY}
EOF EOF
cat > /catalog/lakehouse.properties <<EOF cat > /catalog/lakehouse.properties <<EOF
connector.name=hive connector.name=hive
hive.metastore.uri=thrift://hive-metastore.{{ $.Values.namespace }}.svc.cluster.local:9083 hive.metastore.uri=thrift://hive-metastore.{{ $.Release.Namespace }}.svc.cluster.local:9083
hive.s3.endpoint=http://minio.minio-service.svc.cluster.local:80 hive.s3.endpoint=http://{{ index $.Values.config "MINIO_ENDPOINT" }}
hive.s3.path-style-access=true hive.s3.path-style-access=true
hive.s3.aws-access-key=${MINIO_ACCESS_KEY} hive.s3.aws-access-key=${MINIO_ACCESS_KEY}
hive.s3.aws-secret-key=${MINIO_SECRET_KEY} hive.s3.aws-secret-key=${MINIO_SECRET_KEY}
@@ -92,12 +87,7 @@ spec:
- name: catalog-config - name: catalog-config
mountPath: /etc/trino/catalog mountPath: /etc/trino/catalog
resources: resources:
requests: {{- toYaml .Values.trino.resources | nindent 12 }}
cpu: {{ .Values.trino.resources.requests.cpu }}
memory: {{ .Values.trino.resources.requests.memory }}
limits:
cpu: {{ .Values.trino.resources.limits.cpu | quote }}
memory: {{ .Values.trino.resources.limits.memory }}
readinessProbe: readinessProbe:
httpGet: httpGet:
path: /v1/info path: /v1/info
@@ -112,7 +102,7 @@ apiVersion: v1
kind: Service kind: Service
metadata: metadata:
name: trino name: trino
namespace: {{ .Values.namespace }} namespace: {{ .Release.Namespace }}
spec: spec:
selector: selector:
app: trino app: trino
infra/helm/stonks-oracle/values.yaml
+63 -69
View File
@@ -1,39 +1,36 @@
## Stonks Oracle Helm Values ## Global image settings
image:
registry: ghcr.io/celesrenata/stonks-oracle
pullPolicy: Always
tag: latest
namespace: stonks-oracle
imageRegistry: ghcr.io/celesrenata/stonks-oracle
imageTag: latest
imagePullPolicy: Always
## GHCR image pull secret
imagePullSecrets: imagePullSecrets:
- name: ghcr-credentials
## GHCR authentication for private registry
ghcrAuth:
enabled: true enabled: true
name: ghcr-credentials registry: ghcr.io
# Set to true to have Helm manage the secret. If false, create it username: celesrenata
# manually with: kubectl create secret docker-registry ghcr-credentials ... # base64-encoded dockerconfigjson — override at install time
createSecret: false password: ""
# Base64-encoded .dockerconfigjson — only used when createSecret is true
dockerconfigjson: ""
## ── Services ──────────────────────────────────────────────────────────
## Each service maps to a Deployment. Services with `port` also get a
## ClusterIP Service resource. `secrets` lists which Secret names to
## mount via envFrom.
## Service deployments — replicas and resource overrides
services: services:
scheduler: scheduler:
image: scheduler
replicas: 1 replicas: 1
image: scheduler
command: "python -m services.scheduler.app"
tier: orchestration tier: orchestration
secrets: [stonks-core-secrets] secrets: [stonks-core-secrets]
resources: resources:
requests: { cpu: 50m, memory: 64Mi } requests: { cpu: 50m, memory: 64Mi }
limits: { cpu: 200m, memory: 128Mi } limits: { cpu: 200m, memory: 128Mi }
symbol-registry-api: symbolRegistry:
image: symbol-registry
replicas: 1 replicas: 1
image: symbol-registry
command: "uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000"
tier: api tier: api
port: 8000 port: 8000
secrets: [stonks-core-secrets] secrets: [stonks-core-secrets]
@@ -44,54 +41,60 @@ services:
readiness: { path: /docs, port: 8000, initialDelay: 5, period: 10 } readiness: { path: /docs, port: 8000, initialDelay: 5, period: 10 }
liveness: { path: /docs, port: 8000, initialDelay: 10, period: 30 } liveness: { path: /docs, port: 8000, initialDelay: 10, period: 30 }
ingestion-worker: ingestion:
image: ingestion
replicas: 2 replicas: 2
image: ingestion
command: "python -m services.ingestion.worker"
tier: ingestion tier: ingestion
secrets: [stonks-core-secrets, stonks-market-secrets] secrets: [stonks-core-secrets, stonks-market-secrets]
resources: resources:
requests: { cpu: 100m, memory: 128Mi } requests: { cpu: 100m, memory: 128Mi }
limits: { cpu: 500m, memory: 256Mi } limits: { cpu: 500m, memory: 256Mi }
parser-worker: parser:
image: parser
replicas: 2 replicas: 2
image: parser
command: "python -m services.parser.worker"
tier: processing tier: processing
secrets: [stonks-core-secrets] secrets: [stonks-core-secrets]
resources: resources:
requests: { cpu: 100m, memory: 128Mi } requests: { cpu: 100m, memory: 128Mi }
limits: { cpu: 500m, memory: 256Mi } limits: { cpu: 500m, memory: 256Mi }
extractor-worker: extractor:
image: extractor
replicas: 1 replicas: 1
image: extractor
command: "python -m services.extractor.worker"
tier: processing tier: processing
secrets: [stonks-core-secrets] secrets: [stonks-core-secrets]
resources: resources:
requests: { cpu: 200m, memory: 256Mi } requests: { cpu: 200m, memory: 256Mi }
limits: { cpu: "1", memory: 512Mi } limits: { cpu: "1", memory: 512Mi }
aggregation-worker: aggregation:
replicas: 1
image: aggregation image: aggregation
replicas: 1 command: "python -m services.aggregation.worker"
tier: processing tier: processing
secrets: [stonks-core-secrets] secrets: [stonks-core-secrets]
resources: resources:
requests: { cpu: 100m, memory: 128Mi } requests: { cpu: 100m, memory: 128Mi }
limits: { cpu: 500m, memory: 256Mi } limits: { cpu: 500m, memory: 256Mi }
recommendation-worker: recommendation:
replicas: 1
image: recommendation image: recommendation
replicas: 1 command: "python -m services.recommendation.worker"
tier: processing tier: processing
secrets: [stonks-core-secrets] secrets: [stonks-core-secrets]
resources: resources:
requests: { cpu: 100m, memory: 128Mi } requests: { cpu: 100m, memory: 128Mi }
limits: { cpu: 500m, memory: 256Mi } limits: { cpu: 500m, memory: 256Mi }
risk-engine: riskEngine:
image: risk
replicas: 1 replicas: 1
image: risk
command: "uvicorn services.risk.app:app --host 0.0.0.0 --port 8000"
tier: trading tier: trading
port: 8000 port: 8000
secrets: [stonks-core-secrets, stonks-broker-secrets] secrets: [stonks-core-secrets, stonks-broker-secrets]
@@ -99,27 +102,30 @@ services:
requests: { cpu: 100m, memory: 128Mi } requests: { cpu: 100m, memory: 128Mi }
limits: { cpu: 500m, memory: 256Mi } limits: { cpu: 500m, memory: 256Mi }
broker-adapter: brokerAdapter:
image: broker-adapter
replicas: 1 replicas: 1
image: broker-adapter
command: "python -m services.adapters.broker_adapter"
tier: trading tier: trading
secrets: [stonks-core-secrets, stonks-broker-secrets] secrets: [stonks-core-secrets, stonks-broker-secrets]
resources: resources:
requests: { cpu: 50m, memory: 64Mi } requests: { cpu: 50m, memory: 64Mi }
limits: { cpu: 200m, memory: 128Mi } limits: { cpu: 200m, memory: 128Mi }
lake-publisher: lakePublisher:
image: lake-publisher
replicas: 1 replicas: 1
image: lake-publisher
command: "python -m services.lake_publisher.worker"
tier: analytics tier: analytics
secrets: [stonks-core-secrets] secrets: [stonks-core-secrets]
resources: resources:
requests: { cpu: 100m, memory: 128Mi } requests: { cpu: 100m, memory: 128Mi }
limits: { cpu: 500m, memory: 256Mi } limits: { cpu: 500m, memory: 256Mi }
query-api: queryApi:
image: query-api
replicas: 1 replicas: 1
image: query-api
command: "uvicorn services.api.app:app --host 0.0.0.0 --port 8000"
tier: api tier: api
port: 8000 port: 8000
secrets: [stonks-core-secrets] secrets: [stonks-core-secrets]
@@ -129,7 +135,7 @@ services:
probes: probes:
readiness: { path: /docs, port: 8000, initialDelay: 5, period: 10 } readiness: { path: /docs, port: 8000, initialDelay: 5, period: 10 }
## ── ConfigMap data ──────────────────────────────────────────────────── ## ConfigMap data
config: config:
POSTGRES_HOST: "postgresql-rw.postgresql-service.svc.cluster.local" POSTGRES_HOST: "postgresql-rw.postgresql-service.svc.cluster.local"
POSTGRES_PORT: "5432" POSTGRES_PORT: "5432"
@@ -177,48 +183,35 @@ config:
ALERT_BROKER_ERROR_WINDOW_HOURS: "1" ALERT_BROKER_ERROR_WINDOW_HOURS: "1"
ALERT_CHECK_INTERVAL_SECONDS: "120" ALERT_CHECK_INTERVAL_SECONDS: "120"
## ── Secrets ─────────────────────────────────────────────────────────── ## Secrets
## Values here are placeholders. Override via --set or a secrets file.
secrets: secrets:
core: core:
POSTGRES_PASSWORD: "REPLACE_ME" POSTGRES_PASSWORD: ""
MINIO_ACCESS_KEY: "REPLACE_ME" MINIO_ACCESS_KEY: ""
MINIO_SECRET_KEY: "REPLACE_ME" MINIO_SECRET_KEY: ""
REDIS_PASSWORD: "" REDIS_PASSWORD: ""
broker: broker:
BROKER_API_KEY: "REPLACE_ME" BROKER_API_KEY: ""
BROKER_API_SECRET: "REPLACE_ME" BROKER_API_SECRET: ""
BROKER_BASE_URL: "https://paper-api.alpaca.markets" BROKER_BASE_URL: "https://paper-api.alpaca.markets"
market: market:
MARKET_DATA_API_KEY: "REPLACE_ME" MARKET_DATA_API_KEY: ""
dashboard: dashboard:
SUPERSET_SECRET_KEY: "REPLACE_ME" SUPERSET_SECRET_KEY: ""
SUPERSET_ADMIN_PASSWORD: "REPLACE_ME" SUPERSET_ADMIN_PASSWORD: ""
## ── Ingress ─────────────────────────────────────────────────────────── ## Ingress
ingress: ingress:
enabled: true enabled: true
className: traefik className: traefik
clusterIssuer: ca-issuer clusterIssuer: ca-issuer
hosts: hosts:
query-api: queryApi: stonks-api.celestium.life
host: stonks-api.celestium.life symbolRegistry: stonks-registry.celestium.life
service: query-api superset: stonks-dash.celestium.life
port: 8000 trino: stonks-trino.celestium.life
registry:
host: stonks-registry.celestium.life
service: symbol-registry-api
port: 8000
superset:
host: stonks-dash.celestium.life
service: superset
port: 8088
trino:
host: stonks-trino.celestium.life
service: trino
port: 8080
## ── Analytics stack ─────────────────────────────────────────────────── ## Analytics stack
trino: trino:
enabled: true enabled: true
resources: resources:
@@ -239,5 +232,6 @@ superset:
requests: { cpu: 200m, memory: 512Mi } requests: { cpu: 200m, memory: 512Mi }
limits: { cpu: "1", memory: 2Gi } limits: { cpu: "1", memory: 2Gi }
## Network policies
networkPolicies: networkPolicies:
enabled: true enabled: true
infra/migrations/003_trends_recommendations_orders.sql
+2 -2
View File
@@ -8,7 +8,7 @@ CREATE TABLE trend_windows (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
entity_type VARCHAR(50) NOT NULL DEFAULT 'company', entity_type VARCHAR(50) NOT NULL DEFAULT 'company',
entity_id VARCHAR(100) NOT NULL, entity_id VARCHAR(100) NOT NULL,
window VARCHAR(20) NOT NULL, "window" VARCHAR(20) NOT NULL,
trend_direction VARCHAR(20) NOT NULL DEFAULT 'neutral', trend_direction VARCHAR(20) NOT NULL DEFAULT 'neutral',
trend_strength FLOAT DEFAULT 0.5, trend_strength FLOAT DEFAULT 0.5,
confidence FLOAT DEFAULT 0.5, confidence FLOAT DEFAULT 0.5,
@@ -21,7 +21,7 @@ CREATE TABLE trend_windows (
generated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), generated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
); );
CREATE INDEX idx_trends_entity ON trend_windows(entity_type, entity_id, window); CREATE INDEX idx_trends_entity ON trend_windows(entity_type, entity_id, "window");
CREATE INDEX idx_trends_generated ON trend_windows(generated_at DESC); CREATE INDEX idx_trends_generated ON trend_windows(generated_at DESC);
-- ============================================================ -- ============================================================