admin/stonks-oracle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

phase 14-15: docker build validation and helm deployment

Browse Source
This commit is contained in:
Celes Renata
2026-04-11 11:59:45 -07:00
parent 7394d241c9
commit ce10afa034
179 changed files with 32559 additions and 576 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/hive/core-site.xml
+27
View File
@@ -0,0 +1,27 @@
<?xml version="1.0"?>
<configuration>
<property>
<name>fs.s3a.endpoint</name>
<value>http://minio:9000</value>
</property>
<property>
<name>fs.s3a.access.key</name>
<value>minioadmin</value>
</property>
<property>
<name>fs.s3a.secret.key</name>
<value>minioadmin</value>
</property>
<property>
<name>fs.s3a.path.style.access</name>
<value>true</value>
</property>
<property>
<name>fs.s3a.impl</name>
<value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
</property>
<property>
<name>fs.s3a.connection.ssl.enabled</name>
<value>false</value>
</property>
</configuration>
infra/hive/metastore-site.xml
+27
View File
@@ -0,0 +1,27 @@
<?xml version="1.0"?>
<configuration>
<property>
<name>metastore.thrift.uris</name>
<value>thrift://0.0.0.0:9083</value>
</property>
<property>
<name>metastore.task.threads.always</name>
<value>org.apache.hadoop.hive.metastore.events.EventCleanerTask</value>
</property>
<property>
<name>metastore.expression.proxy</name>
<value>org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy</value>
</property>
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>org.apache.derby.jdbc.EmbeddedDriver</value>
</property>
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:derby:/opt/hive/data/metastore_db;create=true</value>
</property>
<property>
<name>metastore.warehouse.dir</name>
<value>s3a://stonks-lakehouse/warehouse</value>
</property>
</configuration>
infra/k8s/aggregation-worker.yaml
+23 -1
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: aggregation-worker
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: processing
spec:
replicas: 1
selector:
@@ -15,16 +16,30 @@ spec:
metadata:
labels:
app: aggregation-worker
stonks-oracle/tier: processing
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: aggregation-worker
image: ghcr.io/celesrenata/stonks-oracle/aggregation:latest
imagePullPolicy: Always
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
envFrom:
- configMapRef:
name: stonks-config
- secretRef:
name: stonks-secrets
name: stonks-core-secrets
resources:
requests:
cpu: 100m
@@ -32,3 +47,10 @@ spec:
limits:
cpu: 500m
memory: 256Mi
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir:
sizeLimit: 10Mi
infra/k8s/broker-adapter.yaml
+25 -1
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: broker-adapter
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: trading
spec:
replicas: 1
selector:
@@ -15,16 +16,32 @@ spec:
metadata:
labels:
app: broker-adapter
stonks-oracle/tier: trading
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: broker-adapter
image: ghcr.io/celesrenata/stonks-oracle/broker-adapter:latest
imagePullPolicy: Always
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
envFrom:
- configMapRef:
name: stonks-config
- secretRef:
name: stonks-secrets
name: stonks-core-secrets
- secretRef:
name: stonks-broker-secrets
resources:
requests:
cpu: 50m
@@ -32,3 +49,10 @@ spec:
limits:
cpu: 200m
memory: 128Mi
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir:
sizeLimit: 10Mi
infra/k8s/configmap.yaml
+33
View File
@@ -25,15 +25,48 @@ data:
OLLAMA_BASE_URL: "http://ollama.ollama-service.svc.cluster.local:11434"
OLLAMA_MODEL: "llama3.1:8b"
OLLAMA_TIMEOUT: "120"
OLLAMA_MAX_RETRIES: "2"
OLLAMA_RETRY_BASE_DELAY: "1.0"
OLLAMA_RETRY_MAX_DELAY: "10.0"
OLLAMA_RETRY_BACKOFF_MULTIPLIER: "2.0"
# Trino — deployed in stonks-oracle namespace
TRINO_HOST: "trino.stonks-oracle.svc.cluster.local"
TRINO_PORT: "8080"
TRINO_CATALOG: "lakehouse"
TRINO_SCHEMA: "stonks"
TRINO_ICEBERG_CATALOG: "iceberg"
# Broker
BROKER_MODE: "paper"
BROKER_PROVIDER: "alpaca"
# Market Data
MARKET_DATA_BASE_URL: "https://api.polygon.io"
MARKET_DATA_PROVIDER: "polygon"
# Retention (days per bucket class)
RETENTION_RAW_MARKET_DAYS: "90"
RETENTION_RAW_NEWS_DAYS: "180"
RETENTION_RAW_FILINGS_DAYS: "365"
RETENTION_NORMALIZED_DAYS: "180"
RETENTION_LLM_PROMPTS_DAYS: "365"
RETENTION_LLM_RESULTS_DAYS: "365"
RETENTION_LAKEHOUSE_DAYS: "730"
RETENTION_AUDIT_DAYS: "730"
RETENTION_CLEANUP_INTERVAL_HOURS: "24"
RETENTION_BATCH_SIZE: "1000"
# General
LOG_LEVEL: "INFO"
JSON_LOGS: "true"
# Alerting thresholds
ALERT_SOURCE_FAILURE_THRESHOLD: "3"
ALERT_SOURCE_FAILURE_WINDOW_HOURS: "6"
ALERT_SCHEMA_FAILURE_RATE_THRESHOLD: "0.3"
ALERT_SCHEMA_FAILURE_WINDOW_HOURS: "1"
ALERT_LAKE_LAG_THRESHOLD_MINUTES: "60"
ALERT_BROKER_ERROR_THRESHOLD: "3"
ALERT_BROKER_ERROR_WINDOW_HOURS: "1"
ALERT_CHECK_INTERVAL_SECONDS: "120"
infra/k8s/extractor-worker.yaml
+23 -1
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: extractor-worker
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: processing
spec:
replicas: 1
selector:
@@ -15,16 +16,30 @@ spec:
metadata:
labels:
app: extractor-worker
stonks-oracle/tier: processing
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: extractor-worker
image: ghcr.io/celesrenata/stonks-oracle/extractor:latest
imagePullPolicy: Always
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
envFrom:
- configMapRef:
name: stonks-config
- secretRef:
name: stonks-secrets
name: stonks-core-secrets
resources:
requests:
cpu: 200m
@@ -32,3 +47,10 @@ spec:
limits:
cpu: "1"
memory: 512Mi
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir:
sizeLimit: 10Mi
infra/k8s/hive-metastore.yaml
+104 -2
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: hive-metastore
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: analytics
spec:
replicas: 1
selector:
@@ -15,22 +16,121 @@ spec:
metadata:
labels:
app: hive-metastore
stonks-oracle/tier: analytics
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
initContainers:
- name: hive-config-init
image: busybox:1.36
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
command: ["sh", "-c"]
args:
- |
cat > /hive-config/core-site.xml <<EOF
<?xml version="1.0"?>
<configuration>
<property>
<name>fs.s3a.endpoint</name>
<value>http://minio.minio-service.svc.cluster.local:80</value>
</property>
<property>
<name>fs.s3a.access.key</name>
<value>${MINIO_ACCESS_KEY}</value>
</property>
<property>
<name>fs.s3a.secret.key</name>
<value>${MINIO_SECRET_KEY}</value>
</property>
<property>
<name>fs.s3a.path.style.access</name>
<value>true</value>
</property>
<property>
<name>fs.s3a.impl</name>
<value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
</property>
<property>
<name>fs.s3a.connection.ssl.enabled</name>
<value>false</value>
</property>
</configuration>
EOF
cat > /hive-config/metastore-site.xml <<EOF
<?xml version="1.0"?>
<configuration>
<property>
<name>metastore.thrift.uris</name>
<value>thrift://0.0.0.0:9083</value>
</property>
<property>
<name>metastore.task.threads.always</name>
<value>org.apache.hadoop.hive.metastore.events.EventCleanerTask</value>
</property>
<property>
<name>metastore.expression.proxy</name>
<value>org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy</value>
</property>
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>org.apache.derby.jdbc.EmbeddedDriver</value>
</property>
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:derby:/opt/hive/data/metastore_db;create=true</value>
</property>
<property>
<name>metastore.warehouse.dir</name>
<value>s3a://stonks-lakehouse/warehouse</value>
</property>
</configuration>
EOF
env:
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: stonks-core-secrets
key: MINIO_ACCESS_KEY
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: stonks-core-secrets
key: MINIO_SECRET_KEY
volumeMounts:
- name: hive-config
mountPath: /hive-config
containers:
- name: hive-metastore
image: apache/hive:4.0.0
ports:
- containerPort: 9083
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
env:
- name: SERVICE_NAME
value: metastore
- name: DB_DRIVER
value: derby
- name: SERVICE_OPTS
value: "-Djavax.jdo.option.ConnectionURL=jdbc:derby:/opt/hive/data/metastore_db;create=true"
volumeMounts:
- name: hive-data
mountPath: /opt/hive/data
- name: hive-config
mountPath: /opt/hive/conf/core-site.xml
subPath: core-site.xml
- name: hive-config
mountPath: /opt/hive/conf/metastore-site.xml
subPath: metastore-site.xml
resources:
requests:
cpu: 200m
@@ -42,6 +142,8 @@ spec:
- name: hive-data
persistentVolumeClaim:
claimName: hive-metastore-data
- name: hive-config
emptyDir: {}
---
apiVersion: v1
kind: Service
infra/k8s/ingestion-worker.yaml
+25 -1
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: ingestion-worker
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: ingestion
spec:
replicas: 2
selector:
@@ -15,16 +16,32 @@ spec:
metadata:
labels:
app: ingestion-worker
stonks-oracle/tier: ingestion
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: ingestion-worker
image: ghcr.io/celesrenata/stonks-oracle/ingestion:latest
imagePullPolicy: Always
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
envFrom:
- configMapRef:
name: stonks-config
- secretRef:
name: stonks-secrets
name: stonks-core-secrets
- secretRef:
name: stonks-market-secrets
resources:
requests:
cpu: 100m
@@ -32,3 +49,10 @@ spec:
limits:
cpu: 500m
memory: 256Mi
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir:
sizeLimit: 10Mi
infra/k8s/lake-publisher.yaml
+23 -1
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: lake-publisher
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: analytics
spec:
replicas: 1
selector:
@@ -15,16 +16,30 @@ spec:
metadata:
labels:
app: lake-publisher
stonks-oracle/tier: analytics
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: lake-publisher
image: ghcr.io/celesrenata/stonks-oracle/lake-publisher:latest
imagePullPolicy: Always
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
envFrom:
- configMapRef:
name: stonks-config
- secretRef:
name: stonks-secrets
name: stonks-core-secrets
resources:
requests:
cpu: 100m
@@ -32,3 +47,10 @@ spec:
limits:
cpu: 500m
memory: 256Mi
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir:
sizeLimit: 10Mi
infra/k8s/namespace.yaml
+1
View File
@@ -4,3 +4,4 @@ metadata:
name: stonks-oracle
labels:
app.kubernetes.io/part-of: stonks-oracle
kubernetes.io/metadata.name: stonks-oracle
infra/k8s/network-policies.yaml
+173
View File
@@ -0,0 +1,173 @@
##
## Stonks Oracle — Network Policies
##
## Default-deny ingress for the namespace, then allow only the
## traffic patterns each component actually needs.
##
## Requirements: 8.2 (trading isolation), 12.1 (observability)
##
# ── Default deny all ingress in the namespace ──────────────────────────
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: default-deny-ingress
namespace: stonks-oracle
spec:
podSelector: {}
policyTypes:
- Ingress
---
# ── Query API: accept from Traefik ingress only ───────────────────────
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-query-api-ingress
namespace: stonks-oracle
spec:
podSelector:
matchLabels:
app: query-api
policyTypes:
- Ingress
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
ports:
- protocol: TCP
port: 8000
---
# ── Symbol Registry API: accept from Traefik ingress only ─────────────
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-symbol-registry-ingress
namespace: stonks-oracle
spec:
podSelector:
matchLabels:
app: symbol-registry-api
policyTypes:
- Ingress
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
ports:
- protocol: TCP
port: 8000
---
# ── Risk Engine: accept from broker-adapter only ───────────────────────
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-risk-engine-ingress
namespace: stonks-oracle
spec:
podSelector:
matchLabels:
app: risk-engine
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
app: broker-adapter
- podSelector:
matchLabels:
app: query-api
ports:
- protocol: TCP
port: 8000
---
# ── Superset: accept from Traefik ingress only ────────────────────────
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-superset-ingress
namespace: stonks-oracle
spec:
podSelector:
matchLabels:
app: superset
policyTypes:
- Ingress
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
ports:
- protocol: TCP
port: 8088
---
# ── Trino: accept from Superset and query-api ─────────────────────────
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-trino-ingress
namespace: stonks-oracle
spec:
podSelector:
matchLabels:
app: trino
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
app: superset
- podSelector:
matchLabels:
app: query-api
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
ports:
- protocol: TCP
port: 8080
---
# ── Hive Metastore: accept from Trino and lake-publisher ──────────────
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-hive-metastore-ingress
namespace: stonks-oracle
spec:
podSelector:
matchLabels:
app: hive-metastore
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
app: trino
- podSelector:
matchLabels:
app: lake-publisher
ports:
- protocol: TCP
port: 9083
---
# ── Broker adapter: isolated — no inbound from other pods ──────────────
# The broker-adapter only makes outbound calls to the broker API
# and reads from Redis queues. No pod needs to call into it.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: deny-broker-adapter-ingress
namespace: stonks-oracle
spec:
podSelector:
matchLabels:
app: broker-adapter
policyTypes:
- Ingress
ingress: []
infra/k8s/parser-worker.yaml
+23 -1
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: parser-worker
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: processing
spec:
replicas: 2
selector:
@@ -15,16 +16,30 @@ spec:
metadata:
labels:
app: parser-worker
stonks-oracle/tier: processing
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: parser-worker
image: ghcr.io/celesrenata/stonks-oracle/parser:latest
imagePullPolicy: Always
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
envFrom:
- configMapRef:
name: stonks-config
- secretRef:
name: stonks-secrets
name: stonks-core-secrets
resources:
requests:
cpu: 100m
@@ -32,3 +47,10 @@ spec:
limits:
cpu: 500m
memory: 256Mi
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir:
sizeLimit: 10Mi
infra/k8s/query-api.yaml
+23 -1
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: query-api
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: api
spec:
replicas: 1
selector:
@@ -15,18 +16,32 @@ spec:
metadata:
labels:
app: query-api
stonks-oracle/tier: api
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: query-api
image: ghcr.io/celesrenata/stonks-oracle/query-api:latest
imagePullPolicy: Always
ports:
- containerPort: 8000
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
envFrom:
- configMapRef:
name: stonks-config
- secretRef:
name: stonks-secrets
name: stonks-core-secrets
resources:
requests:
cpu: 100m
@@ -40,6 +55,13 @@ spec:
port: 8000
initialDelaySeconds: 5
periodSeconds: 10
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir:
sizeLimit: 10Mi
---
apiVersion: v1
kind: Service
infra/k8s/recommendation-worker.yaml
+23 -1
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: recommendation-worker
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: processing
spec:
replicas: 1
selector:
@@ -15,16 +16,30 @@ spec:
metadata:
labels:
app: recommendation-worker
stonks-oracle/tier: processing
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: recommendation-worker
image: ghcr.io/celesrenata/stonks-oracle/recommendation:latest
imagePullPolicy: Always
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
envFrom:
- configMapRef:
name: stonks-config
- secretRef:
name: stonks-secrets
name: stonks-core-secrets
resources:
requests:
cpu: 100m
@@ -32,3 +47,10 @@ spec:
limits:
cpu: 500m
memory: 256Mi
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir:
sizeLimit: 10Mi
infra/k8s/risk-engine.yaml
+25 -1
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: risk-engine
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: trading
spec:
replicas: 1
selector:
@@ -15,18 +16,34 @@ spec:
metadata:
labels:
app: risk-engine
stonks-oracle/tier: trading
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: risk-engine
image: ghcr.io/celesrenata/stonks-oracle/risk:latest
imagePullPolicy: Always
ports:
- containerPort: 8000
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
envFrom:
- configMapRef:
name: stonks-config
- secretRef:
name: stonks-secrets
name: stonks-core-secrets
- secretRef:
name: stonks-broker-secrets
resources:
requests:
cpu: 100m
@@ -34,6 +51,13 @@ spec:
limits:
cpu: 500m
memory: 256Mi
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir:
sizeLimit: 10Mi
---
apiVersion: v1
kind: Service
infra/k8s/scheduler.yaml
+23 -1
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: scheduler
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: orchestration
spec:
replicas: 1
selector:
@@ -15,16 +16,30 @@ spec:
metadata:
labels:
app: scheduler
stonks-oracle/tier: orchestration
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: scheduler
image: ghcr.io/celesrenata/stonks-oracle/scheduler:latest
imagePullPolicy: Always
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
envFrom:
- configMapRef:
name: stonks-config
- secretRef:
name: stonks-secrets
name: stonks-core-secrets
resources:
requests:
cpu: 50m
@@ -32,3 +47,10 @@ spec:
limits:
cpu: 200m
memory: 128Mi
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir:
sizeLimit: 10Mi
infra/k8s/secrets.yaml
+54 -8
View File
@@ -1,17 +1,63 @@
##
## Stonks Oracle — Scoped Secrets
##
## Secrets are split by concern so that only the services that need
## broker or market-data credentials actually receive them.
## Replace placeholder values before deploying.
##
## Requirements: 8.2 (broker credential isolation)
##
# ── Core infrastructure secrets (DB, object store, cache) ──────────────
apiVersion: v1
kind: Secret
metadata:
name: stonks-secrets
name: stonks-core-secrets
namespace: stonks-oracle
labels:
app.kubernetes.io/part-of: stonks-oracle
type: Opaque
stringData:
POSTGRES_PASSWORD: "changeme"
MINIO_ACCESS_KEY: "changeme"
MINIO_SECRET_KEY: "changeme"
POSTGRES_PASSWORD: "REPLACE_ME"
MINIO_ACCESS_KEY: "REPLACE_ME"
MINIO_SECRET_KEY: "REPLACE_ME"
REDIS_PASSWORD: ""
BROKER_API_KEY: ""
BROKER_API_SECRET: ""
BROKER_BASE_URL: ""
SUPERSET_SECRET_KEY: "stonks-superset-secret-change-me"
---
# ── Broker secrets — only for broker-adapter and risk-engine ───────────
apiVersion: v1
kind: Secret
metadata:
name: stonks-broker-secrets
namespace: stonks-oracle
labels:
app.kubernetes.io/part-of: stonks-oracle
type: Opaque
stringData:
BROKER_API_KEY: "REPLACE_ME"
BROKER_API_SECRET: "REPLACE_ME"
BROKER_BASE_URL: "https://paper-api.alpaca.markets"
---
# ── Market data secrets — only for ingestion and adapters ──────────────
apiVersion: v1
kind: Secret
metadata:
name: stonks-market-secrets
namespace: stonks-oracle
labels:
app.kubernetes.io/part-of: stonks-oracle
type: Opaque
stringData:
MARKET_DATA_API_KEY: "REPLACE_ME"
---
# ── Dashboard secrets — only for Superset ──────────────────────────────
apiVersion: v1
kind: Secret
metadata:
name: stonks-dashboard-secrets
namespace: stonks-oracle
labels:
app.kubernetes.io/part-of: stonks-oracle
type: Opaque
stringData:
SUPERSET_SECRET_KEY: "REPLACE_ME"
SUPERSET_ADMIN_PASSWORD: "REPLACE_ME"
infra/k8s/superset.yaml
+47 -3
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: superset
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: dashboard
spec:
replicas: 1
selector:
@@ -15,22 +16,38 @@ spec:
metadata:
labels:
app: superset
stonks-oracle/tier: dashboard
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: superset
image: apache/superset:latest
ports:
- containerPort: 8088
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
env:
- name: SUPERSET_SECRET_KEY
valueFrom:
secretKeyRef:
name: stonks-secrets
name: stonks-dashboard-secrets
key: SUPERSET_SECRET_KEY
- name: ADMIN_USERNAME
value: admin
- name: ADMIN_PASSWORD
value: admin
valueFrom:
secretKeyRef:
name: stonks-dashboard-secrets
key: SUPERSET_ADMIN_PASSWORD
- name: ADMIN_EMAIL
value: admin@stonks.local
volumeMounts:
@@ -94,12 +111,39 @@ data:
import os
SECRET_KEY = os.getenv("SUPERSET_SECRET_KEY", "stonks-dev-secret-key-change-me")
SQLALCHEMY_DATABASE_URI = "trino://trino@trino.stonks-oracle.svc.cluster.local:8080/lakehouse/stonks"
# Additional database connections available in Superset UI:
# Hive catalog: trino://trino@trino.stonks-oracle.svc.cluster.local:8080/lakehouse/stonks
# Iceberg catalog: trino://trino@trino.stonks-oracle.svc.cluster.local:8080/iceberg/stonks
FEATURE_FLAGS = {"ENABLE_TEMPLATE_PROCESSING": True}
CACHE_CONFIG = {
"CACHE_TYPE": "RedisCache",
"CACHE_DEFAULT_TIMEOUT": 300,
"CACHE_KEY_PREFIX": "superset_",
"CACHE_REDIS_HOST": os.getenv("REDIS_HOST", "redis.redis-service.svc.cluster.local"),
"CACHE_REDIS_HOST": os.getenv("REDIS_HOST", "redis-master.redis-service.svc.cluster.local"),
"CACHE_REDIS_PORT": int(os.getenv("REDIS_PORT", "6379")),
"CACHE_REDIS_DB": 1,
}
# --- Security hardening ---
# Disable public user role (require login)
PUBLIC_ROLE_LIKE = None
# Session cookie security
SESSION_COOKIE_HTTPONLY = True
SESSION_COOKIE_SECURE = True
SESSION_COOKIE_SAMESITE = "Lax"
# Talisman CSP headers
TALISMAN_ENABLED = True
TALISMAN_CONFIG = {
"content_security_policy": {
"default-src": ["'self'"],
"img-src": ["'self'", "data:"],
"style-src": ["'self'", "'unsafe-inline'"],
"script-src": ["'self'", "'unsafe-inline'", "'unsafe-eval'"],
},
"force_https": False, # TLS terminated at ingress
}
# Prevent Superset from allowing arbitrary SQL database connections
PREVENT_UNSAFE_DB_CONNECTIONS = True
# Row limit for queries
ROW_LIMIT = 50000
SQL_MAX_ROW = 100000
infra/k8s/symbol-registry.yaml
+23 -1
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: symbol-registry-api
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: api
spec:
replicas: 1
selector:
@@ -15,18 +16,32 @@ spec:
metadata:
labels:
app: symbol-registry-api
stonks-oracle/tier: api
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: symbol-registry-api
image: ghcr.io/celesrenata/stonks-oracle/symbol-registry:latest
imagePullPolicy: Always
ports:
- containerPort: 8000
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
envFrom:
- configMapRef:
name: stonks-config
- secretRef:
name: stonks-secrets
name: stonks-core-secrets
resources:
requests:
cpu: 100m
@@ -46,6 +61,13 @@ spec:
port: 8000
initialDelaySeconds: 10
periodSeconds: 30
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir:
sizeLimit: 10Mi
---
apiVersion: v1
kind: Service
infra/k8s/trino.yaml
+63 -26
View File
@@ -6,6 +6,7 @@ metadata:
labels:
app: trino
app.kubernetes.io/part-of: stonks-oracle
stonks-oracle/tier: analytics
spec:
replicas: 1
selector:
@@ -15,12 +16,73 @@ spec:
metadata:
labels:
app: trino
stonks-oracle/tier: analytics
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
initContainers:
- name: catalog-init
image: busybox:1.36
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
command: ["sh", "-c"]
args:
- |
cat > /catalog/iceberg.properties <<EOF
connector.name=iceberg
iceberg.catalog.type=hive_metastore
hive.metastore.uri=thrift://hive-metastore.stonks-oracle.svc.cluster.local:9083
hive.s3.endpoint=http://minio.minio-service.svc.cluster.local:80
hive.s3.path-style-access=true
hive.s3.aws-access-key=${MINIO_ACCESS_KEY}
hive.s3.aws-secret-key=${MINIO_SECRET_KEY}
fs.native-s3.enabled=true
s3.endpoint=http://minio.minio-service.svc.cluster.local:80
s3.path-style-access=true
s3.aws-access-key=${MINIO_ACCESS_KEY}
s3.aws-secret-key=${MINIO_SECRET_KEY}
EOF
cat > /catalog/lakehouse.properties <<EOF
connector.name=hive
hive.metastore.uri=thrift://hive-metastore.stonks-oracle.svc.cluster.local:9083
hive.s3.endpoint=http://minio.minio-service.svc.cluster.local:80
hive.s3.path-style-access=true
hive.s3.aws-access-key=${MINIO_ACCESS_KEY}
hive.s3.aws-secret-key=${MINIO_SECRET_KEY}
hive.non-managed-table-writes-enabled=true
hive.s3select-pushdown.enabled=true
EOF
env:
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: stonks-core-secrets
key: MINIO_ACCESS_KEY
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: stonks-core-secrets
key: MINIO_SECRET_KEY
volumeMounts:
- name: catalog-config
mountPath: /catalog
containers:
- name: trino
image: trinodb/trino:latest
ports:
- containerPort: 8080
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
volumeMounts:
- name: catalog-config
mountPath: /etc/trino/catalog
@@ -39,8 +101,7 @@ spec:
periodSeconds: 10
volumes:
- name: catalog-config
configMap:
name: trino-catalog
emptyDir: {}
---
apiVersion: v1
kind: Service
@@ -53,27 +114,3 @@ spec:
ports:
- port: 8080
targetPort: 8080
---
apiVersion: v1
kind: ConfigMap
metadata:
name: trino-catalog
namespace: stonks-oracle
data:
iceberg.properties: |
connector.name=iceberg
iceberg.catalog.type=hive_metastore
hive.metastore.uri=thrift://hive-metastore.stonks-oracle.svc.cluster.local:9083
hive.s3.endpoint=http://minio.minio-service.svc.cluster.local:80
hive.s3.path-style-access=true
hive.s3.aws-access-key=changeme
hive.s3.aws-secret-key=changeme
lakehouse.properties: |
connector.name=hive
hive.metastore.uri=thrift://hive-metastore.stonks-oracle.svc.cluster.local:9083
hive.s3.endpoint=http://minio.minio-service.svc.cluster.local:80
hive.s3.path-style-access=true
hive.s3.aws-access-key=changeme
hive.s3.aws-secret-key=changeme
hive.non-managed-table-writes-enabled=true
hive.s3select-pushdown.enabled=true
infra/migrations/004_dedupe_indexes.sql
+13
View File
@@ -0,0 +1,13 @@
-- Stonks Oracle - Dedupe support indexes
-- Index on canonical_url for cross-source deduplication lookups.
-- The dedupe module queries documents by canonical_url to detect
-- the same article ingested from different source types.
CREATE INDEX idx_documents_canonical_url ON documents(canonical_url)
WHERE canonical_url IS NOT NULL;
-- Unique constraint on document_company_mentions to prevent duplicate
-- company links when cross-source dedupe links an existing document
-- to an additional company.
CREATE UNIQUE INDEX idx_doc_mentions_unique
ON document_company_mentions(document_id, company_id);
infra/migrations/005_parser_output_ref.sql
+5
View File
@@ -0,0 +1,5 @@
-- Stonks Oracle - Add parser_output_ref to documents table
-- Stores the MinIO reference to the structured parser output JSON
-- (metadata, quality signals, warnings, outbound links, tags, etc.)
ALTER TABLE documents ADD COLUMN IF NOT EXISTS parser_output_ref VARCHAR(1000);
infra/migrations/006_model_performance_metrics.sql
+40
View File
@@ -0,0 +1,40 @@
-- Stonks Oracle - Model Performance Metrics
-- Tracks extraction success/failure rates, latency, retries, confidence,
-- token usage estimates, and validation error distributions.
-- Requirements: 5.2, 5.4, 12.1, 12.2
CREATE TABLE model_performance_metrics (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
document_id UUID REFERENCES documents(id) ON DELETE SET NULL,
ticker VARCHAR(20),
model_name VARCHAR(200) NOT NULL,
prompt_version VARCHAR(100),
schema_version VARCHAR(50),
success BOOLEAN NOT NULL,
attempt_count INTEGER NOT NULL DEFAULT 1,
total_duration_ms INTEGER NOT NULL DEFAULT 0,
first_attempt_duration_ms INTEGER DEFAULT 0,
final_attempt_duration_ms INTEGER DEFAULT 0,
confidence FLOAT DEFAULT 0.0,
validation_status VARCHAR(50) NOT NULL DEFAULT 'unknown',
validation_error_count INTEGER DEFAULT 0,
validation_warning_count INTEGER DEFAULT 0,
validation_errors JSONB DEFAULT '[]',
retry_count INTEGER DEFAULT 0,
input_token_estimate INTEGER DEFAULT 0,
output_token_estimate INTEGER DEFAULT 0,
company_count INTEGER DEFAULT 0,
recorded_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
-- Query by time range (dashboard primary access pattern)
CREATE INDEX idx_model_perf_recorded ON model_performance_metrics(recorded_at DESC);
-- Filter by model for per-model dashboards
CREATE INDEX idx_model_perf_model ON model_performance_metrics(model_name, recorded_at DESC);
-- Filter by success for failure analysis
CREATE INDEX idx_model_perf_success ON model_performance_metrics(success, recorded_at DESC);
-- Filter by validation status for schema failure dashboards
CREATE INDEX idx_model_perf_validation ON model_performance_metrics(validation_status);
infra/migrations/007_disagreement_details.sql
+8
View File
@@ -0,0 +1,8 @@
-- Stonks Oracle - Add disagreement details to trend windows
-- Stores structured contradiction/disagreement representations
-- so downstream consumers can inspect *why* signals conflict
-- rather than relying on a single scalar contradiction_score.
-- Requirements: 6.4
ALTER TABLE trend_windows
ADD COLUMN IF NOT EXISTS disagreement_details JSONB DEFAULT '[]';
infra/migrations/008_trend_evidence.sql
+23
View File
@@ -0,0 +1,23 @@
-- Stonks Oracle - Trend evidence mappings
-- Links trend_windows to the documents that contributed as evidence,
-- storing the evidence type (supporting/opposing), rank score, and
-- weight breakdown for explainability and drill-down queries.
-- Requirements: 6.5, 10.4
CREATE TABLE trend_evidence (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
trend_window_id UUID NOT NULL REFERENCES trend_windows(id) ON DELETE CASCADE,
document_id UUID NOT NULL,
evidence_type VARCHAR(20) NOT NULL DEFAULT 'supporting', -- supporting | opposing
rank_score FLOAT DEFAULT 0.0,
weight_component FLOAT DEFAULT 0.0,
impact_component FLOAT DEFAULT 0.0,
recency_component FLOAT DEFAULT 0.0,
confidence_component FLOAT DEFAULT 0.0,
sentiment_value FLOAT DEFAULT 0.0,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX idx_trend_evidence_trend ON trend_evidence(trend_window_id);
CREATE INDEX idx_trend_evidence_doc ON trend_evidence(document_id);
CREATE INDEX idx_trend_evidence_type ON trend_evidence(trend_window_id, evidence_type);
infra/migrations/009_recommendation_persistence.sql
+15
View File
@@ -0,0 +1,15 @@
-- Stonks Oracle - Recommendation persistence enhancements
-- Adds full model metadata columns to recommendations table
-- and a risk_classification column for the computed risk label.
-- Requirements: 7.1, 7.2, 8.3
-- Store full model provenance on the recommendation itself
ALTER TABLE recommendations
ADD COLUMN IF NOT EXISTS model_provider VARCHAR(100) DEFAULT 'deterministic',
ADD COLUMN IF NOT EXISTS prompt_version VARCHAR(100) DEFAULT '',
ADD COLUMN IF NOT EXISTS schema_version VARCHAR(50) DEFAULT '1.0.0',
ADD COLUMN IF NOT EXISTS risk_classification VARCHAR(20) DEFAULT 'moderate';
-- Index for querying recommendations by risk classification
CREATE INDEX IF NOT EXISTS idx_recommendations_risk
ON recommendations(risk_classification);
infra/migrations/010_risk_configuration.sql
+55
View File
@@ -0,0 +1,55 @@
-- Stonks Oracle - Portfolio and account risk configuration
-- Persists risk configuration profiles and tracks risk state snapshots.
-- Requirements: 8.1, 8.2, 8.4
-- ============================================================
-- Risk Configuration Profiles
-- ============================================================
CREATE TABLE risk_configs (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
name VARCHAR(200) NOT NULL DEFAULT 'default',
trading_mode VARCHAR(20) NOT NULL DEFAULT 'paper',
config JSONB NOT NULL DEFAULT '{}',
active BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX idx_risk_configs_active_name
ON risk_configs(name) WHERE active = TRUE;
-- ============================================================
-- Symbol-level lockouts (news-shock, cooldown)
-- ============================================================
CREATE TABLE symbol_lockouts (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
ticker VARCHAR(20) NOT NULL,
lockout_type VARCHAR(50) NOT NULL,
reason TEXT DEFAULT '',
expires_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX idx_symbol_lockouts_ticker ON symbol_lockouts(ticker, expires_at);
CREATE INDEX idx_symbol_lockouts_expiry ON symbol_lockouts(expires_at);
-- ============================================================
-- Daily risk snapshots (for daily loss tracking)
-- ============================================================
CREATE TABLE daily_risk_snapshots (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
account_id VARCHAR(200) NOT NULL,
snapshot_date DATE NOT NULL DEFAULT CURRENT_DATE,
portfolio_value NUMERIC DEFAULT 0,
daily_pnl NUMERIC DEFAULT 0,
daily_trade_count INTEGER DEFAULT 0,
positions_by_sector JSONB DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE(account_id, snapshot_date)
);
CREATE INDEX idx_daily_risk_account ON daily_risk_snapshots(account_id, snapshot_date DESC);
infra/migrations/011_paper_trading_positions_unique.sql
+7
View File
@@ -0,0 +1,7 @@
-- Stonks Oracle - Add unique constraint for paper trading position upserts
-- Requirements: 8.1, 8.3
-- The paper trading adapter needs to upsert positions by (broker_account_id, ticker).
-- Add a unique constraint to support ON CONFLICT.
CREATE UNIQUE INDEX IF NOT EXISTS idx_positions_account_ticker
ON positions(broker_account_id, ticker);
infra/migrations/012_audit_trail_indexes.sql
+17
View File
@@ -0,0 +1,17 @@
-- Stonks Oracle - Execution audit trail indexes
-- Supports efficient querying of the full decision chain from
-- recommendation through risk evaluation to broker execution.
-- Requirements: 8.3, 11.3
-- GIN index on audit_events.data for JSONB key lookups
-- (e.g. data->>'recommendation_id', data->>'order_id')
CREATE INDEX IF NOT EXISTS idx_audit_events_data_gin
ON audit_events USING gin (data);
-- Index for chronological audit trail queries by entity
CREATE INDEX IF NOT EXISTS idx_audit_events_entity_created
ON audit_events (entity_id, created_at ASC);
-- Index for filtering by event_type + entity_type
CREATE INDEX IF NOT EXISTS idx_audit_events_type_entity
ON audit_events (event_type, entity_type);
infra/migrations/013_operator_approval_workflow.sql
+29
View File
@@ -0,0 +1,29 @@
-- Stonks Oracle - Operator approval workflow for live trading mode
-- Tracks pending, approved, rejected, and expired approval requests
-- for orders that require operator sign-off before broker submission.
-- Requirements: 8.2
CREATE TABLE operator_approvals (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
order_job JSONB NOT NULL DEFAULT '{}',
recommendation_id UUID REFERENCES recommendations(id),
ticker VARCHAR(20) NOT NULL,
side VARCHAR(10) NOT NULL DEFAULT 'buy',
quantity NUMERIC NOT NULL DEFAULT 0,
estimated_value NUMERIC NOT NULL DEFAULT 0,
status VARCHAR(20) NOT NULL DEFAULT 'pending',
risk_evaluation_id UUID,
requested_by VARCHAR(200) NOT NULL DEFAULT 'system',
reviewed_by VARCHAR(200),
review_note TEXT,
expires_at TIMESTAMPTZ NOT NULL,
requested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
reviewed_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX idx_operator_approvals_status ON operator_approvals(status);
CREATE INDEX idx_operator_approvals_ticker ON operator_approvals(ticker);
CREATE INDEX idx_operator_approvals_expires ON operator_approvals(expires_at)
WHERE status = 'pending';
infra/migrations/014_retention_policies.sql
+43
View File
@@ -0,0 +1,43 @@
-- Stonks Oracle - Data retention and lifecycle policies
-- Tracks per-bucket and per-artifact-class retention rules.
-- Requirements: N3
CREATE TABLE retention_policies (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
bucket_name VARCHAR(200) NOT NULL,
artifact_class VARCHAR(100) NOT NULL DEFAULT 'default',
retention_days INTEGER NOT NULL DEFAULT 365,
archive_before_delete BOOLEAN NOT NULL DEFAULT FALSE,
active BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE(bucket_name, artifact_class)
);
-- Seed default retention policies per bucket
INSERT INTO retention_policies (bucket_name, artifact_class, retention_days, archive_before_delete) VALUES
('stonks-raw-market', 'default', 90, FALSE),
('stonks-raw-news', 'default', 180, FALSE),
('stonks-raw-filings', 'default', 365, FALSE),
('stonks-normalized', 'default', 180, FALSE),
('stonks-llm-prompts', 'default', 365, FALSE),
('stonks-llm-results', 'default', 365, FALSE),
('stonks-lakehouse', 'default', 730, FALSE),
('stonks-audit', 'default', 730, FALSE);
-- Track retention cleanup runs for observability
CREATE TABLE retention_runs (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
bucket_name VARCHAR(200) NOT NULL,
objects_scanned INTEGER NOT NULL DEFAULT 0,
objects_deleted INTEGER NOT NULL DEFAULT 0,
bytes_freed BIGINT NOT NULL DEFAULT 0,
db_rows_deleted INTEGER NOT NULL DEFAULT 0,
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
completed_at TIMESTAMPTZ,
status VARCHAR(20) NOT NULL DEFAULT 'running',
error_message TEXT
);
CREATE INDEX idx_retention_runs_bucket ON retention_runs(bucket_name, started_at DESC);
CREATE INDEX idx_retention_runs_status ON retention_runs(status);
infra/minio/lifecycle.json
+81 -11
View File
@@ -1,14 +1,84 @@
{
"Rules": [
{
"ID": "raw-retention-365d",
"Status": "Enabled",
"Filter": {
"Prefix": ""
},
"Expiration": {
"Days": 365
}
"buckets": {
"stonks-raw-market": {
"Rules": [
{
"ID": "raw-market-retention-90d",
"Status": "Enabled",
"Filter": { "Prefix": "" },
"Expiration": { "Days": 90 }
}
]
},
"stonks-raw-news": {
"Rules": [
{
"ID": "raw-news-retention-180d",
"Status": "Enabled",
"Filter": { "Prefix": "" },
"Expiration": { "Days": 180 }
}
]
},
"stonks-raw-filings": {
"Rules": [
{
"ID": "raw-filings-retention-365d",
"Status": "Enabled",
"Filter": { "Prefix": "" },
"Expiration": { "Days": 365 }
}
]
},
"stonks-normalized": {
"Rules": [
{
"ID": "normalized-retention-180d",
"Status": "Enabled",
"Filter": { "Prefix": "" },
"Expiration": { "Days": 180 }
}
]
},
"stonks-llm-prompts": {
"Rules": [
{
"ID": "llm-prompts-retention-365d",
"Status": "Enabled",
"Filter": { "Prefix": "" },
"Expiration": { "Days": 365 }
}
]
},
"stonks-llm-results": {
"Rules": [
{
"ID": "llm-results-retention-365d",
"Status": "Enabled",
"Filter": { "Prefix": "" },
"Expiration": { "Days": 365 }
}
]
},
"stonks-lakehouse": {
"Rules": [
{
"ID": "lakehouse-retention-730d",
"Status": "Enabled",
"Filter": { "Prefix": "" },
"Expiration": { "Days": 730 }
}
]
},
"stonks-audit": {
"Rules": [
{
"ID": "audit-retention-730d",
"Status": "Enabled",
"Filter": { "Prefix": "" },
"Expiration": { "Days": 730 }
}
]
}
]
}
}
infra/superset/superset_config.py
+43 -3
View File
@@ -1,10 +1,18 @@
"""Apache Superset configuration for Stonks Oracle."""
"""Apache Superset configuration for Stonks Oracle.
Security hardening applied:
- Session cookies: HttpOnly, Secure, SameSite=Lax
- Talisman CSP headers enabled
- Public role disabled (login required)
- Unsafe DB connections blocked
- Row limits enforced
"""
import os
# Superset secret key
# Superset secret key — must be set via SUPERSET_SECRET_KEY env var
SECRET_KEY = os.getenv("SUPERSET_SECRET_KEY", "stonks-dev-secret-key-change-me")
# Trino datasource
# Default Trino datasource (Hive catalog for backward compatibility)
SQLALCHEMY_DATABASE_URI = "trino://trino@trino:8080/lakehouse/stonks"
# Feature flags
@@ -12,6 +20,10 @@ FEATURE_FLAGS = {
"ENABLE_TEMPLATE_PROCESSING": True,
}
# Additional database connections available in Superset UI:
# Hive catalog: trino://trino@trino:8080/lakehouse/stonks
# Iceberg catalog: trino://trino@trino:8080/iceberg/stonks
# Cache config (Redis-backed)
CACHE_CONFIG = {
"CACHE_TYPE": "RedisCache",
@@ -21,3 +33,31 @@ CACHE_CONFIG = {
"CACHE_REDIS_PORT": int(os.getenv("REDIS_PORT", "6379")),
"CACHE_REDIS_DB": 1,
}
# --- Security hardening ---
# Disable public user role (require login)
PUBLIC_ROLE_LIKE = None
# Session cookie security
SESSION_COOKIE_HTTPONLY = True
SESSION_COOKIE_SECURE = True
SESSION_COOKIE_SAMESITE = "Lax"
# Talisman CSP headers
TALISMAN_ENABLED = True
TALISMAN_CONFIG = {
"content_security_policy": {
"default-src": ["'self'"],
"img-src": ["'self'", "data:"],
"style-src": ["'self'", "'unsafe-inline'"],
"script-src": ["'self'", "'unsafe-inline'", "'unsafe-eval'"],
},
"force_https": False, # TLS terminated at ingress
}
# Prevent Superset from allowing arbitrary SQL database connections
PREVENT_UNSAFE_DB_CONNECTIONS = True
# Row limit for queries
ROW_LIMIT = 50000
SQL_MAX_ROW = 100000
infra/trino/catalog/iceberg.properties
+5
View File
@@ -5,3 +5,8 @@ hive.s3.endpoint=http://minio:9000
hive.s3.path-style-access=true
hive.s3.aws-access-key=minioadmin
hive.s3.aws-secret-key=minioadmin
fs.native-s3.enabled=true
s3.endpoint=http://minio:9000
s3.path-style-access=true
s3.aws-access-key=minioadmin
s3.aws-secret-key=minioadmin