diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c0d792..4cafbae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,7 +55,7 @@ jobs: - name: recommendation cmd: "python -m services.recommendation.worker" - name: risk - cmd: "uvicorn services.risk.engine:app --host 0.0.0.0 --port 8000" + cmd: "uvicorn services.risk.app:app --host 0.0.0.0 --port 8000" - name: broker-adapter cmd: "python -m services.adapters.broker_adapter" - name: lake-publisher diff --git a/.kiro/specs/stonks-oracle/tasks.md b/.kiro/specs/stonks-oracle/tasks.md index b87c909..7ee4ad9 100644 --- a/.kiro/specs/stonks-oracle/tasks.md +++ b/.kiro/specs/stonks-oracle/tasks.md @@ -146,7 +146,7 @@ - Monitor the GitHub Actions run to confirm lint-and-test and build-services jobs succeed - Fix any CI failures and re-push if needed - _Requirements: N1_ -- [x] 15.3 Create Helm chart for stonks-oracle deployment +- [-] 15.3 Create Helm chart for stonks-oracle deployment - Create `infra/helm/stonks-oracle/Chart.yaml` with chart metadata - Create `infra/helm/stonks-oracle/values.yaml` with configurable image tags, replica counts, resource limits, and environment references - Create Helm templates for all deployments, services, configmap, secrets, ingress, and network policies from existing K8s manifests diff --git a/infra/helm/stonks-oracle/Chart.yaml b/infra/helm/stonks-oracle/Chart.yaml index 14f08b8..bffdd27 100644 --- a/infra/helm/stonks-oracle/Chart.yaml +++ b/infra/helm/stonks-oracle/Chart.yaml @@ -4,6 +4,3 @@ description: AI market intelligence and paper-trading platform type: application version: 0.1.0 appVersion: "1.0.0" -maintainers: - - name: Celes Renata - email: celes@frameshift.net diff --git a/infra/helm/stonks-oracle/templates/_helpers.tpl b/infra/helm/stonks-oracle/templates/_helpers.tpl index e8f1bbf..e1e8b20 100644 --- a/infra/helm/stonks-oracle/templates/_helpers.tpl +++ b/infra/helm/stonks-oracle/templates/_helpers.tpl @@ -8,8 +8,23 @@ helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} {{- end }} {{/* -Full image path for a service +Pod security context — shared across all custom service pods */}} -{{- define "stonks.image" -}} -{{ $.Values.imageRegistry }}/{{ .image }}:{{ $.Values.imageTag }} +{{- define "stonks.podSecurityContext" -}} +runAsNonRoot: true +runAsUser: 1000 +runAsGroup: 1000 +fsGroup: 1000 +seccompProfile: + type: RuntimeDefault +{{- end }} + +{{/* +Container security context +*/}} +{{- define "stonks.containerSecurityContext" -}} +allowPrivilegeEscalation: false +readOnlyRootFilesystem: true +capabilities: + drop: ["ALL"] {{- end }} diff --git a/infra/helm/stonks-oracle/templates/configmap.yaml b/infra/helm/stonks-oracle/templates/configmap.yaml index c8cfa1f..462942b 100644 --- a/infra/helm/stonks-oracle/templates/configmap.yaml +++ b/infra/helm/stonks-oracle/templates/configmap.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: stonks-config - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} labels: {{- include "stonks.labels" . | nindent 4 }} data: diff --git a/infra/helm/stonks-oracle/templates/deployments.yaml b/infra/helm/stonks-oracle/templates/deployments.yaml index da165c6..14975c3 100644 --- a/infra/helm/stonks-oracle/templates/deployments.yaml +++ b/infra/helm/stonks-oracle/templates/deployments.yaml @@ -1,50 +1,43 @@ +{{- $root := . -}} {{- range $name, $svc := .Values.services }} --- apiVersion: apps/v1 kind: Deployment metadata: - name: {{ $name }} - namespace: {{ $.Values.namespace }} + name: {{ $svc.image }} + namespace: {{ $root.Release.Namespace }} labels: - app: {{ $name }} - {{- include "stonks.labels" $ | nindent 4 }} + app: {{ $svc.image }} + {{- include "stonks.labels" $root | nindent 4 }} stonks-oracle/tier: {{ $svc.tier }} spec: replicas: {{ $svc.replicas }} selector: matchLabels: - app: {{ $name }} + app: {{ $svc.image }} template: metadata: labels: - app: {{ $name }} + app: {{ $svc.image }} stonks-oracle/tier: {{ $svc.tier }} spec: automountServiceAccountToken: false - {{- if $.Values.imagePullSecrets.enabled }} + {{- with $root.Values.imagePullSecrets }} imagePullSecrets: - - name: {{ $.Values.imagePullSecrets.name }} + {{- toYaml . | nindent 8 }} {{- end }} securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault + {{- include "stonks.podSecurityContext" $root | nindent 8 }} containers: - - name: {{ $name }} - image: {{ $.Values.imageRegistry }}/{{ $svc.image }}:{{ $.Values.imageTag }} - imagePullPolicy: {{ $.Values.imagePullPolicy }} + - name: {{ $svc.image }} + image: {{ $root.Values.image.registry }}/{{ $svc.image }}:{{ $root.Values.image.tag }} + imagePullPolicy: {{ $root.Values.image.pullPolicy }} {{- if $svc.port }} ports: - containerPort: {{ $svc.port }} {{- end }} securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] + {{- include "stonks.containerSecurityContext" $root | nindent 12 }} envFrom: - configMapRef: name: stonks-config @@ -53,12 +46,7 @@ spec: name: {{ . }} {{- end }} resources: - requests: - cpu: {{ $svc.resources.requests.cpu }} - memory: {{ $svc.resources.requests.memory }} - limits: - cpu: {{ $svc.resources.limits.cpu | quote }} - memory: {{ $svc.resources.limits.memory }} + {{- toYaml $svc.resources | nindent 12 }} {{- if $svc.probes }} {{- if $svc.probes.readiness }} readinessProbe: diff --git a/infra/helm/stonks-oracle/templates/ghcr-secret.yaml b/infra/helm/stonks-oracle/templates/ghcr-secret.yaml index e1b8d0e..9111877 100644 --- a/infra/helm/stonks-oracle/templates/ghcr-secret.yaml +++ b/infra/helm/stonks-oracle/templates/ghcr-secret.yaml @@ -1,12 +1,12 @@ -{{- if and .Values.imagePullSecrets.enabled .Values.imagePullSecrets.createSecret }} +{{- if .Values.ghcrAuth.enabled }} apiVersion: v1 kind: Secret metadata: - name: {{ .Values.imagePullSecrets.name }} - namespace: {{ .Values.namespace }} + name: ghcr-credentials + namespace: {{ .Release.Namespace }} labels: {{- include "stonks.labels" . | nindent 4 }} type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: {{ .Values.imagePullSecrets.dockerconfigjson }} + .dockerconfigjson: {{ printf `{"auths":{"%s":{"username":"%s","password":"%s","auth":"%s"}}}` .Values.ghcrAuth.registry .Values.ghcrAuth.username .Values.ghcrAuth.password (printf "%s:%s" .Values.ghcrAuth.username .Values.ghcrAuth.password | b64enc) | b64enc }} {{- end }} diff --git a/infra/helm/stonks-oracle/templates/hive-metastore.yaml b/infra/helm/stonks-oracle/templates/hive-metastore.yaml index ce9eea4..90f3290 100644 --- a/infra/helm/stonks-oracle/templates/hive-metastore.yaml +++ b/infra/helm/stonks-oracle/templates/hive-metastore.yaml @@ -3,7 +3,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: hive-metastore - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} labels: app: hive-metastore {{- include "stonks.labels" . | nindent 4 }} @@ -20,17 +20,12 @@ spec: stonks-oracle/tier: analytics spec: automountServiceAccountToken: false - {{- if .Values.imagePullSecrets.enabled }} + {{- with .Values.imagePullSecrets }} imagePullSecrets: - - name: {{ .Values.imagePullSecrets.name }} + {{- toYaml . | nindent 8 }} {{- end }} securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault + {{- include "stonks.podSecurityContext" . | nindent 8 }} initContainers: - name: hive-config-init image: busybox:1.36 @@ -44,7 +39,7 @@ spec: cat > /hive-config/core-site.xml < - fs.s3a.endpointhttp://minio.minio-service.svc.cluster.local:80 + fs.s3a.endpointhttp://{{ index $.Values.config "MINIO_ENDPOINT" }} fs.s3a.access.key${MINIO_ACCESS_KEY} fs.s3a.secret.key${MINIO_SECRET_KEY} fs.s3a.path.style.accesstrue @@ -101,12 +96,7 @@ spec: mountPath: /opt/hive/conf/metastore-site.xml subPath: metastore-site.xml resources: - requests: - cpu: {{ .Values.hiveMetastore.resources.requests.cpu }} - memory: {{ .Values.hiveMetastore.resources.requests.memory }} - limits: - cpu: {{ .Values.hiveMetastore.resources.limits.cpu | quote }} - memory: {{ .Values.hiveMetastore.resources.limits.memory }} + {{- toYaml .Values.hiveMetastore.resources | nindent 12 }} volumes: - name: hive-data persistentVolumeClaim: @@ -118,7 +108,7 @@ apiVersion: v1 kind: Service metadata: name: hive-metastore - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} spec: selector: app: hive-metastore @@ -130,7 +120,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: hive-metastore-data - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} spec: accessModes: - ReadWriteOnce diff --git a/infra/helm/stonks-oracle/templates/ingress.yaml b/infra/helm/stonks-oracle/templates/ingress.yaml index 3e93569..2d5cfb7 100644 --- a/infra/helm/stonks-oracle/templates/ingress.yaml +++ b/infra/helm/stonks-oracle/templates/ingress.yaml @@ -1,29 +1,102 @@ {{- if .Values.ingress.enabled }} -{{- range $key, $ing := .Values.ingress.hosts }} --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: stonks-{{ $key }} - namespace: {{ $.Values.namespace }} + name: stonks-query-api-https + namespace: {{ .Release.Namespace }} annotations: - cert-manager.io/cluster-issuer: {{ $.Values.ingress.clusterIssuer }} + cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }} spec: - ingressClassName: {{ $.Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} tls: - hosts: - - {{ $ing.host }} - secretName: stonks-{{ $key }}-tls + - {{ .Values.ingress.hosts.queryApi }} + secretName: stonks-api-tls rules: - - host: {{ $ing.host }} + - host: {{ .Values.ingress.hosts.queryApi }} http: paths: - path: / pathType: Prefix backend: service: - name: {{ $ing.service }} + name: query-api port: - number: {{ $ing.port }} -{{- end }} + number: 8000 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: stonks-registry-https + namespace: {{ .Release.Namespace }} + annotations: + cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }} +spec: + ingressClassName: {{ .Values.ingress.className }} + tls: + - hosts: + - {{ .Values.ingress.hosts.symbolRegistry }} + secretName: stonks-registry-tls + rules: + - host: {{ .Values.ingress.hosts.symbolRegistry }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: symbol-registry + port: + number: 8000 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: stonks-superset-https + namespace: {{ .Release.Namespace }} + annotations: + cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }} +spec: + ingressClassName: {{ .Values.ingress.className }} + tls: + - hosts: + - {{ .Values.ingress.hosts.superset }} + secretName: stonks-dash-tls + rules: + - host: {{ .Values.ingress.hosts.superset }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: superset + port: + number: 8088 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: stonks-trino-https + namespace: {{ .Release.Namespace }} + annotations: + cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }} +spec: + ingressClassName: {{ .Values.ingress.className }} + tls: + - hosts: + - {{ .Values.ingress.hosts.trino }} + secretName: stonks-trino-tls + rules: + - host: {{ .Values.ingress.hosts.trino }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: trino + port: + number: 8080 {{- end }} diff --git a/infra/helm/stonks-oracle/templates/namespace.yaml b/infra/helm/stonks-oracle/templates/namespace.yaml index 4bbca16..f94a748 100644 --- a/infra/helm/stonks-oracle/templates/namespace.yaml +++ b/infra/helm/stonks-oracle/templates/namespace.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Namespace metadata: - name: {{ .Values.namespace }} + name: {{ .Release.Namespace }} labels: {{- include "stonks.labels" . | nindent 4 }} - kubernetes.io/metadata.name: {{ .Values.namespace }} + kubernetes.io/metadata.name: {{ .Release.Namespace }} diff --git a/infra/helm/stonks-oracle/templates/network-policies.yaml b/infra/helm/stonks-oracle/templates/network-policies.yaml index a7730f9..7a4d6f8 100644 --- a/infra/helm/stonks-oracle/templates/network-policies.yaml +++ b/infra/helm/stonks-oracle/templates/network-policies.yaml @@ -1,21 +1,19 @@ {{- if .Values.networkPolicies.enabled }} -# Default deny all ingress apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: default-deny-ingress - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} spec: podSelector: {} policyTypes: - Ingress --- -# Query API: accept from Traefik ingress apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-query-api-ingress - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} spec: podSelector: matchLabels: @@ -31,16 +29,15 @@ spec: - protocol: TCP port: 8000 --- -# Symbol Registry API: accept from Traefik ingress apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-symbol-registry-ingress - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} spec: podSelector: matchLabels: - app: symbol-registry-api + app: symbol-registry policyTypes: - Ingress ingress: @@ -52,16 +49,15 @@ spec: - protocol: TCP port: 8000 --- -# Risk Engine: accept from broker-adapter and query-api apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-risk-engine-ingress - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} spec: podSelector: matchLabels: - app: risk-engine + app: risk policyTypes: - Ingress ingress: @@ -76,12 +72,11 @@ spec: - protocol: TCP port: 8000 --- -# Superset: accept from Traefik ingress apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-superset-ingress - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} spec: podSelector: matchLabels: @@ -97,12 +92,11 @@ spec: - protocol: TCP port: 8088 --- -# Trino: accept from Superset, query-api, and Traefik apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-trino-ingress - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} spec: podSelector: matchLabels: @@ -124,12 +118,11 @@ spec: - protocol: TCP port: 8080 --- -# Hive Metastore: accept from Trino and lake-publisher apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-hive-metastore-ingress - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} spec: podSelector: matchLabels: @@ -148,12 +141,11 @@ spec: - protocol: TCP port: 9083 --- -# Broker adapter: isolated — no inbound apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: deny-broker-adapter-ingress - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} spec: podSelector: matchLabels: diff --git a/infra/helm/stonks-oracle/templates/secrets.yaml b/infra/helm/stonks-oracle/templates/secrets.yaml index 3ca8de4..646bb93 100644 --- a/infra/helm/stonks-oracle/templates/secrets.yaml +++ b/infra/helm/stonks-oracle/templates/secrets.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: name: stonks-core-secrets - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} labels: {{- include "stonks.labels" . | nindent 4 }} type: Opaque @@ -15,7 +15,7 @@ apiVersion: v1 kind: Secret metadata: name: stonks-broker-secrets - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} labels: {{- include "stonks.labels" . | nindent 4 }} type: Opaque @@ -28,7 +28,7 @@ apiVersion: v1 kind: Secret metadata: name: stonks-market-secrets - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} labels: {{- include "stonks.labels" . | nindent 4 }} type: Opaque @@ -41,7 +41,7 @@ apiVersion: v1 kind: Secret metadata: name: stonks-dashboard-secrets - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} labels: {{- include "stonks.labels" . | nindent 4 }} type: Opaque diff --git a/infra/helm/stonks-oracle/templates/services.yaml b/infra/helm/stonks-oracle/templates/services.yaml index a22d674..0ad5eac 100644 --- a/infra/helm/stonks-oracle/templates/services.yaml +++ b/infra/helm/stonks-oracle/templates/services.yaml @@ -1,14 +1,15 @@ +{{- $root := . -}} {{- range $name, $svc := .Values.services }} {{- if $svc.port }} --- apiVersion: v1 kind: Service metadata: - name: {{ $name }} - namespace: {{ $.Values.namespace }} + name: {{ $svc.image }} + namespace: {{ $root.Release.Namespace }} spec: selector: - app: {{ $name }} + app: {{ $svc.image }} ports: - port: {{ $svc.port }} targetPort: {{ $svc.port }} diff --git a/infra/helm/stonks-oracle/templates/superset.yaml b/infra/helm/stonks-oracle/templates/superset.yaml index 2c4ce0f..c84533b 100644 --- a/infra/helm/stonks-oracle/templates/superset.yaml +++ b/infra/helm/stonks-oracle/templates/superset.yaml @@ -3,7 +3,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: superset - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} labels: app: superset {{- include "stonks.labels" . | nindent 4 }} @@ -20,17 +20,12 @@ spec: stonks-oracle/tier: dashboard spec: automountServiceAccountToken: false - {{- if .Values.imagePullSecrets.enabled }} + {{- with .Values.imagePullSecrets }} imagePullSecrets: - - name: {{ .Values.imagePullSecrets.name }} + {{- toYaml . | nindent 8 }} {{- end }} securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault + {{- include "stonks.podSecurityContext" . | nindent 8 }} containers: - name: superset image: apache/superset:latest @@ -62,12 +57,7 @@ spec: mountPath: /app/pythonpath/superset_config.py subPath: superset_config.py resources: - requests: - cpu: {{ .Values.superset.resources.requests.cpu }} - memory: {{ .Values.superset.resources.requests.memory }} - limits: - cpu: {{ .Values.superset.resources.limits.cpu | quote }} - memory: {{ .Values.superset.resources.limits.memory }} + {{- toYaml .Values.superset.resources | nindent 12 }} readinessProbe: httpGet: path: /health @@ -86,7 +76,7 @@ apiVersion: v1 kind: Service metadata: name: superset - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} spec: selector: app: superset @@ -98,7 +88,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: superset-data - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} spec: accessModes: - ReadWriteOnce @@ -110,12 +100,12 @@ apiVersion: v1 kind: ConfigMap metadata: name: superset-config - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} data: superset_config.py: | import os SECRET_KEY = os.getenv("SUPERSET_SECRET_KEY", "stonks-dev-secret-key-change-me") - SQLALCHEMY_DATABASE_URI = "trino://trino@trino.{{ .Values.namespace }}.svc.cluster.local:8080/lakehouse/stonks" + SQLALCHEMY_DATABASE_URI = "trino://trino@trino.{{ .Release.Namespace }}.svc.cluster.local:8080/lakehouse/stonks" FEATURE_FLAGS = {"ENABLE_TEMPLATE_PROCESSING": True} CACHE_CONFIG = { "CACHE_TYPE": "RedisCache", diff --git a/infra/helm/stonks-oracle/templates/trino.yaml b/infra/helm/stonks-oracle/templates/trino.yaml index 5ebc3ea..9308e3c 100644 --- a/infra/helm/stonks-oracle/templates/trino.yaml +++ b/infra/helm/stonks-oracle/templates/trino.yaml @@ -3,7 +3,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: trino - namespace: {{ .Values.namespace }} + namespace: {{ .Release.Namespace }} labels: app: trino {{- include "stonks.labels" . | nindent 4 }} @@ -20,17 +20,12 @@ spec: stonks-oracle/tier: analytics spec: automountServiceAccountToken: false - {{- if .Values.imagePullSecrets.enabled }} + {{- with .Values.imagePullSecrets }} imagePullSecrets: - - name: {{ .Values.imagePullSecrets.name }} + {{- toYaml . | nindent 8 }} {{- end }} securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault + {{- include "stonks.podSecurityContext" . | nindent 8 }} initContainers: - name: catalog-init image: busybox:1.36 @@ -44,21 +39,21 @@ spec: cat > /catalog/iceberg.properties < /catalog/lakehouse.properties <