Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
504ee20604 | ||
|
|
637d939d51 |
+32
-30
@@ -1,36 +1,38 @@
|
|||||||
# Woodpecker CI pipeline for DCOS
|
# Woodpecker CI pipeline for DCOS
|
||||||
# Integrates with stonks-ci (stonks-oracle's CI/CD at stonks-ci.celestium.life)
|
# Integrates with stonks-ci (stonks-oracle's CI/CD at stonks-ci.celestium.life)
|
||||||
|
|
||||||
steps:
|
pipeline:
|
||||||
- name: lint-and-test
|
name: Build and Deploy DCOS
|
||||||
image: python:3.12-slim
|
|
||||||
commands:
|
|
||||||
- pip install --break-system-packages ruff pytest asyncpg redis aiosqlite pyyaml
|
|
||||||
- ruff check src/dcos/
|
|
||||||
- pytest tests/test_memory/ tests/test_core/ -x --tb=short -q || true
|
|
||||||
|
|
||||||
- name: build-and-push-image
|
steps:
|
||||||
image: plugins/docker
|
- name: lint-and-test
|
||||||
settings:
|
image: python:3.12-slim
|
||||||
repo: ghcr.io/celesrenata/dcos
|
commands:
|
||||||
tags: ${CI_COMMIT_SHA}, latest
|
- pip install --break-system-packages ruff pytest asyncpg redis aiosqlite pyyaml
|
||||||
dockerfile: docker/Dockerfile.dcos
|
- ruff check src/dcos/
|
||||||
context: .
|
- pytest tests/test_memory/ tests/test_core/ -x --tb=short -q || true
|
||||||
registry: ghcr.io
|
|
||||||
username:
|
|
||||||
from_secret: github_token
|
|
||||||
password:
|
|
||||||
from_secret: github_token
|
|
||||||
|
|
||||||
- name: deploy-to-k8s
|
- name: build-and-push-image
|
||||||
image: bitnami/kubectl:latest
|
image: plugins/docker
|
||||||
commands:
|
settings:
|
||||||
# Deploy using runmefirst.sh (handles secrets, namespace, GHCR pull secret)
|
repo: ghcr.io/celesrenata/dcos
|
||||||
- bash ~/sources/kube/dcos/runmefirst.sh
|
tags: ${CI_COMMIT_SHA}, latest
|
||||||
|
dockerfile: docker/Dockerfile.dcos
|
||||||
|
context: .
|
||||||
|
registry: ghcr.io
|
||||||
|
username:
|
||||||
|
from_secret: github_token
|
||||||
|
password:
|
||||||
|
from_secret: github_token
|
||||||
|
|
||||||
environment:
|
- name: deploy-to-k8s
|
||||||
GITHUB_TOKEN:
|
image: bitnami/kubectl:latest
|
||||||
from_secret: github_token
|
commands:
|
||||||
GHCR_TOKEN:
|
# Deploy using runmefirst.sh (handles secrets, namespace, GHCR pull secret)
|
||||||
from_secret: ghcr_pull_token
|
- bash ~/sources/kube/dcos/runmefirst.sh
|
||||||
# Fix
|
|
||||||
|
environment:
|
||||||
|
GITHUB_TOKEN:
|
||||||
|
from_secret: github_token
|
||||||
|
GHCR_TOKEN:
|
||||||
|
from_secret: ghcr_pull_token
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
# DCOS — Single-stage Docker image (mirrors stonks-oracle pipeline pattern)
|
||||||
|
# Production backends: Redis (working, conversation, user_model, world_model, procedural)
|
||||||
|
# SQLite fallback: long_term, semantic, episodic
|
||||||
|
|
||||||
|
FROM python:3.12-slim
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1
|
||||||
|
ENV PYTHONUNBUFFERED=1
|
||||||
|
ENV PYTHONPATH=/app
|
||||||
|
|
||||||
|
# Install async database drivers (asyncpg needs libpq-dev)
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
gcc \
|
||||||
|
libpq-dev \
|
||||||
|
curl \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy DCOS source into container (dcos package at /app/dcos/)
|
||||||
|
COPY src/dcos/ /app/dcos/
|
||||||
|
COPY config/ /app/config/
|
||||||
|
COPY data/ /app/data/
|
||||||
|
|
||||||
|
# Install dependencies (PyYAML for config, async database drivers)
|
||||||
|
RUN pip install --no-cache-dir \
|
||||||
|
"pyyaml>=6.0" \
|
||||||
|
"asyncpg==0.29.0" \
|
||||||
|
"redis>=5.0" \
|
||||||
|
"aiosqlite>=0.20" \
|
||||||
|
--break-system-packages
|
||||||
|
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
# Set environment defaults (overridden by K8s manifests at runtime)
|
||||||
|
ENV DCOS_DATA_DIR=/app/data/memory_store
|
||||||
|
ENV DCOS_CONFIG_DIR=/app/config
|
||||||
|
ENV DCOS_LOG_LEVEL=INFO
|
||||||
|
ENV DCOS_PG_HOST=postgresql-rw.postgresql-service.svc.cluster.local
|
||||||
|
ENV DCOS_PG_PORT=5432
|
||||||
|
ENV DCOS_PG_USER=celes
|
||||||
|
ENV DCOS_PG_DB=dcos
|
||||||
|
ENV DCOS_REDIS_HOST=redis-master
|
||||||
|
ENV DCOS_REDIS_PORT=6379
|
||||||
|
|
||||||
|
# Create non-root user (matches stonks-oracle pipeline convention) — must come before chown
|
||||||
|
RUN useradd -m -u 1000 stonks && \
|
||||||
|
mkdir -p /app/data/memory_store && \
|
||||||
|
chown -R stonks:stonks /app
|
||||||
|
|
||||||
|
USER stonks
|
||||||
|
|
||||||
|
# SERVICE_CMD ARG — same pattern as stonks-oracle's docker/Dockerfile
|
||||||
|
ARG SERVICE_CMD="python -m dcos.core --init"
|
||||||
|
ENV SERVICE_CMD=${SERVICE_CMD}
|
||||||
|
|
||||||
|
CMD ["sh", "-c", "${SERVICE_CMD}"]
|
||||||
@@ -0,0 +1,217 @@
|
|||||||
|
# DCOS — Kubernetes Deployment
|
||||||
|
|
||||||
|
Kubernetes deployment for the Distributed Cognitive Operating System. Follows the standard homelab pattern (Traefik ingress, cert-manager TLS, local-path PVCs).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Quick Deploy
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# One-command deploy (follows your pattern)
|
||||||
|
bash runmefirst.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What Gets Deployed
|
||||||
|
|
||||||
|
| Resource | Name | Purpose |
|
||||||
|
|----------|------|---------|
|
||||||
|
| Namespace | `dcos-service` | Isolation |
|
||||||
|
| Deployment | `dcos` | Main DCOS pods |
|
||||||
|
| Service | `dcos` | ClusterIP:8080 |
|
||||||
|
| Ingress | `dcos` | `dcos.celestium.life` |
|
||||||
|
| PVC | `dcos-memory-pvc` | 20Gi memory store |
|
||||||
|
| PVC | `dcos-config-pvc` | 2Gi config |
|
||||||
|
| ServiceAccount | `dcos` | Pod identity |
|
||||||
|
| Middleware | `dcos-stripprefix` | Traefik path stripping |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Files
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
|------|---------|
|
||||||
|
| [`deployment.yaml`](deployment.yaml) | Deployment: 1 replica, 4 CPU/4Gi RAM, PVC mounts |
|
||||||
|
| [`service.yaml`](service.yaml) | ClusterIP service on port 8080 |
|
||||||
|
| [`ingress.yaml`](ingress.yaml) | Traefik ingress → `dcos.celestium.life` with TLS |
|
||||||
|
| [`pvc.yaml`](pvc.yaml) | 20Gi memory + 2Gi config (local-path) |
|
||||||
|
| [`serviceaccount.yaml`](serviceaccount.yaml) | Service account |
|
||||||
|
| [`middleware.yaml`](middleware.yaml) | Traefik Middleware for path stripping |
|
||||||
|
| [`values.yaml`](values.yaml) | Helm values (image, resources, autoscaling) |
|
||||||
|
| [`runmefirst.sh`](runmefirst.sh) | Namespace + kubectl apply |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Helm Values
|
||||||
|
|
||||||
|
Key values in [`values.yaml`](values.yaml):
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
image:
|
||||||
|
repository: ghcr.io/celesrenata/dcos
|
||||||
|
tag: "1.0.0"
|
||||||
|
pullPolicy: IfNotPresent
|
||||||
|
|
||||||
|
service:
|
||||||
|
type: ClusterIP
|
||||||
|
port: 8080
|
||||||
|
|
||||||
|
ingress:
|
||||||
|
enabled: true
|
||||||
|
hosts:
|
||||||
|
- name: dcos.celestium.life
|
||||||
|
tls:
|
||||||
|
- secretName: dcos-cert
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: "4"
|
||||||
|
memory: "4Gi"
|
||||||
|
|
||||||
|
autoscaling:
|
||||||
|
enabled: false
|
||||||
|
minReplicas: 1
|
||||||
|
maxReplicas: 10
|
||||||
|
|
||||||
|
dcos:
|
||||||
|
maxAgents: 100
|
||||||
|
selfOrgEnabled: true
|
||||||
|
decentralized: true
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Commands
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Deploy
|
||||||
|
bash runmefirst.sh
|
||||||
|
|
||||||
|
# Helm install
|
||||||
|
helm upgrade -i dcos . --namespace dcos-service --create-namespace -f values.yaml
|
||||||
|
|
||||||
|
# Helm upgrade with new image
|
||||||
|
helm upgrade -i dcos . --namespace dcos-service --set image.tag="1.0.1"
|
||||||
|
|
||||||
|
# Scale
|
||||||
|
kubectl scale deployment dcos --replicas=3 -n dcos-service
|
||||||
|
|
||||||
|
# Port-forward for local testing
|
||||||
|
kubectl port-forward service/dcos 8080:8080 -n dcos-service
|
||||||
|
|
||||||
|
# Check status
|
||||||
|
kubectl get all -n dcos-service
|
||||||
|
|
||||||
|
# View logs
|
||||||
|
kubectl logs -f deployment/dcos -n dcos-service
|
||||||
|
|
||||||
|
# Exec into pod
|
||||||
|
kubectl exec -it deployment/dcos -n dcos-service -- python -m dcos --init
|
||||||
|
|
||||||
|
# Uninstall
|
||||||
|
helm uninstall dcos --namespace dcos-service
|
||||||
|
kubectl delete -f pvc.yaml -n dcos-service
|
||||||
|
kubectl delete namespace dcos-service
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Storage
|
||||||
|
|
||||||
|
### PVC Details
|
||||||
|
|
||||||
|
| PVC | Size | StorageClass | Mount Path | Contents |
|
||||||
|
|-----|------|-------------|------------|----------|
|
||||||
|
| `dcos-memory-pvc` | 20Gi | local-path | `/app/data/memory_store` | SQLite databases |
|
||||||
|
| `dcos-config-pvc` | 2Gi | local-path | `/app/config` | YAML configs |
|
||||||
|
|
||||||
|
### Persistent Data
|
||||||
|
|
||||||
|
Memory databases are stored in `data/memory_store/`:
|
||||||
|
|
||||||
|
```
|
||||||
|
memory_store/
|
||||||
|
├── episodic.db # Episodic memory (temporal events)
|
||||||
|
├── knowledge_graph.db # Semantic memory (knowledge graph)
|
||||||
|
├── learning.db # Learning data
|
||||||
|
├── long_term.db # Long-term memory
|
||||||
|
├── procedural.db # Procedural memory (patterns)
|
||||||
|
├── user_model.db # User model
|
||||||
|
└── world_model.db # World model
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Networking
|
||||||
|
|
||||||
|
### Service
|
||||||
|
|
||||||
|
```
|
||||||
|
dcos.celestium.life
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
Ingress (traefik) → TLS (cert-manager)
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
Service (ClusterIP:8080) → Sticky cookies (8h)
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
Pod:8080
|
||||||
|
```
|
||||||
|
|
||||||
|
### Ingress Annotations
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
traefik.ingress.kubernetes.io/affinity: "true"
|
||||||
|
traefik.ingress.kubernetes.io/service.sticky.cookie: "true"
|
||||||
|
traefik.ingress.kubernetes.io/service.sticky.cookie.maxage: "28800"
|
||||||
|
cert-manager.io/cluster-issuer: ca-issuer
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Monitoring
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Pod health
|
||||||
|
kubectl get pods -n dcos-service -w
|
||||||
|
|
||||||
|
# PVC usage
|
||||||
|
kubectl get pvc -n dcos-service
|
||||||
|
|
||||||
|
# Ingress
|
||||||
|
kubectl get ingress -n dcos-service
|
||||||
|
|
||||||
|
# Service endpoints
|
||||||
|
kubectl get endpoints -n dcos-service
|
||||||
|
|
||||||
|
# Resource usage
|
||||||
|
kubectl top pods -n dcos-service
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
| Problem | Check | Solution |
|
||||||
|
|---------|-------|----------|
|
||||||
|
| Pod not starting | `kubectl describe pod dcos-* -n dcos-service` | Check events for image pull errors |
|
||||||
|
| PVC pending | `kubectl get pvc -n dcos-service` | Verify `local-path` storage class exists |
|
||||||
|
| Ingress not routing | `kubectl get ingress -n dcos-service` | Check Traefik config |
|
||||||
|
| TLS errors | `kubectl get certificates -n dcos-service` | Verify cert-manager is running |
|
||||||
|
| Memory full | `kubectl exec dcoss-*/df -h /app/data` | Expand PVC or clean old data |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Image
|
||||||
|
|
||||||
|
```
|
||||||
|
Repository: ghcr.io/celesrenata/dcos
|
||||||
|
Tags: 1.0.0 (production), latest (development), <commit> (CI)
|
||||||
|
Pull Policy: IfNotPresent (production), Always (development)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
*Deployed to `dcos-service` namespace. Ingress at `dcos.celestium.life`.*
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: dcos-db-secret
|
||||||
|
namespace: dcos-service
|
||||||
|
stringData:
|
||||||
|
# PostgreSQL — celes user, dcos database (created via postInitSQL)
|
||||||
|
POSTGRES_PASSWORD: "PSCh4ng3me!"
|
||||||
|
POSTGRES_DSN: "postgresql://celes:PSCh4ng3me!@postgresql-rw:5432/dcos?sslmode=disable"
|
||||||
|
POSTGRES_RO_DSN: "postgresql://celes:PSCh4ng3me!@postgresql-ro:5432/dcos?sslmode=disable"
|
||||||
|
|
||||||
|
# Redis — master endpoint, password
|
||||||
|
REDIS_PASSWORD: "PSCh4ng3me!"
|
||||||
|
REDIS_ADDR: "redis-master:6379"
|
||||||
|
|
||||||
|
# MinIO S3 — for episodic snapshots and backup storage
|
||||||
|
MINIO_ACCESS_KEY: "AKIA6V7J3N9B5P0D2YQH"
|
||||||
|
MINIO_SECRET_KEY: "8fG3!v2rJ7$wN@9mLpQ6zXbC4tKdPqW1"
|
||||||
|
MINIO_ENDPOINT: "minio-crawler-console:9090"
|
||||||
|
type: Opaque
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: dcos
|
||||||
|
labels:
|
||||||
|
helm.sh/chart: dcos-0.1.0
|
||||||
|
app.kubernetes.io/name: dcos
|
||||||
|
app.kubernetes.io/instance: dcos
|
||||||
|
app.kubernetes.io/version: "1.0.0"
|
||||||
|
app.kubernetes.io/managed-by: Helm
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: dcos
|
||||||
|
app.kubernetes.io/instance: dcos
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
helm.sh/chart: dcos-0.1.0
|
||||||
|
app.kubernetes.io/name: dcos
|
||||||
|
app.kubernetes.io/instance: dcos
|
||||||
|
app.kubernetes.io/version: "1.0.0"
|
||||||
|
app.kubernetes.io/managed-by: Helm
|
||||||
|
spec:
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: ghcr-credentials
|
||||||
|
serviceAccountName: dcos
|
||||||
|
securityContext:
|
||||||
|
runAsUser: 1000
|
||||||
|
runAsGroup: 1000
|
||||||
|
fsGroup: 1000
|
||||||
|
containers:
|
||||||
|
- name: dcos
|
||||||
|
image: "ghcr.io/celesrenata/dcos:dev"
|
||||||
|
imagePullPolicy: Always
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 8080
|
||||||
|
protocol: TCP
|
||||||
|
env:
|
||||||
|
- name: DCOS_DATA_DIR
|
||||||
|
value: /app/data/memory_store
|
||||||
|
- name: DCOS_CONFIG_DIR
|
||||||
|
value: /app/config
|
||||||
|
- name: DCOS_LOG_LEVEL
|
||||||
|
value: "INFO"
|
||||||
|
# ──────────────────────────────────────────
|
||||||
|
# Database connection credentials
|
||||||
|
# Managed via dcos-db-secret
|
||||||
|
# ──────────────────────────────────────────
|
||||||
|
# Cross-namespace DNS requires FQDN (postgresql-rw is in postgresql-service namespace)
|
||||||
|
- name: DCOS_PG_HOST
|
||||||
|
value: "postgresql-rw.postgresql-service.svc.cluster.local"
|
||||||
|
- name: DCOS_PG_PORT
|
||||||
|
value: "5432"
|
||||||
|
- name: DCOS_PG_USER
|
||||||
|
value: "celes"
|
||||||
|
- name: DCOS_PG_DB
|
||||||
|
value: "dcos"
|
||||||
|
- name: DCOS_PG_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: dcos-db-secret
|
||||||
|
key: POSTGRES_PASSWORD
|
||||||
|
# Cross-namespace DNS requires FQDN (redis-master is in redis-service namespace)
|
||||||
|
- name: DCOS_REDIS_HOST
|
||||||
|
value: "redis-master.redis-service.svc.cluster.local"
|
||||||
|
- name: DCOS_REDIS_PORT
|
||||||
|
value: "6379"
|
||||||
|
- name: DCOS_REDIS_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: dcos-db-secret
|
||||||
|
key: REDIS_PASSWORD
|
||||||
|
- name: DCOS_MINIO_ENDPOINT
|
||||||
|
value: "minio-crawler-console:9090"
|
||||||
|
- name: DCOS_MINIO_ACCESS_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: dcos-db-secret
|
||||||
|
key: MINIO_ACCESS_KEY
|
||||||
|
- name: DCOS_MINIO_SECRET_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: dcos-db-secret
|
||||||
|
key: MINIO_SECRET_KEY
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: "4"
|
||||||
|
memory: "4Gi"
|
||||||
|
requests:
|
||||||
|
cpu: "1"
|
||||||
|
memory: "1Gi"
|
||||||
|
volumeMounts:
|
||||||
|
- name: memory-store
|
||||||
|
mountPath: /app/data/memory_store
|
||||||
|
- name: config
|
||||||
|
mountPath: /app/config
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: memory-store
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: dcos-memory-pvc
|
||||||
|
- name: config
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: dcos-config-pvc
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: dcos
|
||||||
|
labels:
|
||||||
|
helm.sh/chart: dcos-0.1.0
|
||||||
|
app.kubernetes.io/name: dcos
|
||||||
|
app.kubernetes.io/instance: dcos
|
||||||
|
app.kubernetes.io/version: "1.1.0"
|
||||||
|
app.kubernetes.io/managed-by: Helm
|
||||||
|
annotations:
|
||||||
|
traefik.ingress.kubernetes.io/router.middlewares: default-dcos-stripprefix@kubernetescrd
|
||||||
|
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
||||||
|
cert-manager.io/cluster-issuer: ca-issuer
|
||||||
|
spec:
|
||||||
|
ingressClassName: traefik
|
||||||
|
rules:
|
||||||
|
- host: dcos.celestium.life
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: dcos
|
||||||
|
port:
|
||||||
|
number: 8080
|
||||||
|
tls:
|
||||||
|
- hosts:
|
||||||
|
- dcos.celestium.life
|
||||||
|
secretName: dcos-cert
|
||||||
|
---
|
||||||
|
# Source: dcos/templates/ingress.yaml
|
||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: Middleware
|
||||||
|
metadata:
|
||||||
|
name: dcos-stripprefix
|
||||||
|
spec:
|
||||||
|
stripPrefix:
|
||||||
|
prefixes:
|
||||||
|
- /assets
|
||||||
|
- /dcos
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: Middleware
|
||||||
|
metadata:
|
||||||
|
name: dcos-stripprefix
|
||||||
|
spec:
|
||||||
|
stripPrefix:
|
||||||
|
prefixes:
|
||||||
|
- /assets
|
||||||
|
- /dcos
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
AKIA6V7J3N9B5P0D2YQH
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
8fG3!v2rJ7$wN@9mLpQ6zXbC4tKdPqW1
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
PSCh4ng3me!
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: dcos-memory-pvc
|
||||||
|
namespace: dcos-service
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
storageClassName: local-path
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 20Gi
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: dcos-config-pvc
|
||||||
|
namespace: dcos-service
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
storageClassName: local-path
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 2Gi
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
PSCh4ng3me!
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Deploy DCOS to Kubernetes (standalone pipeline, mirrors stonks-oracle pattern)
|
||||||
|
# Usage: runmefirst.sh
|
||||||
|
# Detects existing deploy and adds/updates; idempotent.
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
NAMESPACE="dcos-service"
|
||||||
|
REPO_DIR="$HOME/sources/sama/sama"
|
||||||
|
KUBE_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
GHCR="ghcr.io/celesrenata/dcos"
|
||||||
|
|
||||||
|
# --- Secrets ---
|
||||||
|
# All secrets are read from ~/sources/kube/dcos/ on the deploy host.
|
||||||
|
# This directory is NOT a git repo — secrets stay local to the deploy host.
|
||||||
|
#
|
||||||
|
# Required files:
|
||||||
|
# /run/secrets/github_token (for GHCR push/pull)
|
||||||
|
# ~/sources/kube/dcos/postgres.password (PG password for dcos user)
|
||||||
|
# ~/sources/kube/dcos/redis.password (Redis password)
|
||||||
|
|
||||||
|
_read_secret() {
|
||||||
|
local file="$1"
|
||||||
|
local default="${2:-}"
|
||||||
|
if [ -f "$file" ]; then
|
||||||
|
cat "$file" | tr -d '[:space:]'
|
||||||
|
elif [ -n "$default" ]; then
|
||||||
|
echo "$default"
|
||||||
|
else
|
||||||
|
echo "ERROR: Secret file not found: $file" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
GHCR_TOKEN=$(_read_secret /run/secrets/github_token)
|
||||||
|
PG_PASSWORD=$(_read_secret "$KUBE_DIR/postgres.password")
|
||||||
|
REDIS_PASSWORD=$(_read_secret "$KUBE_DIR/redis.password")
|
||||||
|
|
||||||
|
# Compute git SHA for image tagging (same pattern as stonks-oracle Makefile)
|
||||||
|
SHA=$(git -C "$REPO_DIR" rev-parse --short HEAD 2>/dev/null || echo "dev")
|
||||||
|
|
||||||
|
echo "=== DCOS Deployment ==="
|
||||||
|
echo "Namespace: $NAMESPACE"
|
||||||
|
echo "Image: $GHCR:$SHA"
|
||||||
|
echo "Repo: $REPO_DIR"
|
||||||
|
echo "Secrets: $KUBE_DIR"
|
||||||
|
|
||||||
|
# --- 0. Pull latest code ---
|
||||||
|
echo "[0/6] Pulling latest code..."
|
||||||
|
git -C "$REPO_DIR" pull --ff-only || echo "WARNING: git pull failed — using existing code"
|
||||||
|
|
||||||
|
# Force-pull the image on all gremlin nodes to avoid stale cache (IfNotPresent uses local)
|
||||||
|
echo "[0.5/6] Forcing fresh image pull on cluster nodes..."
|
||||||
|
for node in $(kubectl get nodes -o name 2>/dev/null | sed 's/node\.//'); do
|
||||||
|
echo " Pulling on $node..."
|
||||||
|
kubectl debug node/$node --image=busybox:1.36 -- chroot /host sh -c "docker rmi ghcr.io/celesrenata/dcos:$SHA 2>/dev/null; docker pull ghcr.io/celesrenata/dcos:$SHA" || true
|
||||||
|
done
|
||||||
|
|
||||||
|
# --- 1. Ensure namespace exists with correct labels ---
|
||||||
|
echo "[1/6] Ensuring namespace $NAMESPACE exists..."
|
||||||
|
if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
|
||||||
|
kubectl create namespace "$NAMESPACE"
|
||||||
|
fi
|
||||||
|
kubectl label namespace "$NAMESPACE" app.kubernetes.io/managed-by=Helm --overwrite
|
||||||
|
kubectl annotate namespace "$NAMESPACE" meta.helm.sh/release-name=dcos meta.helm.sh/release-namespace=$NAMESPACE --overwrite
|
||||||
|
|
||||||
|
# --- 2. Build and push image ---
|
||||||
|
echo "[2/6] Building DCOS image..."
|
||||||
|
docker build \
|
||||||
|
--build-arg "SERVICE_CMD=python -m dcos.core --init" \
|
||||||
|
-t $GHCR:$SHA \
|
||||||
|
-t $GHCR:latest \
|
||||||
|
-f "$REPO_DIR/docker/Dockerfile.dcos" "$REPO_DIR" || {
|
||||||
|
echo "ERROR: Docker build failed." >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "Pushing image..."
|
||||||
|
if ! echo "$GHCR_TOKEN" | docker login ghcr.io -u celesrenata --password-stdin >/dev/null 2>&1; then
|
||||||
|
echo "WARNING: GHCR login failed — token may be expired or missing scopes." >&2
|
||||||
|
echo " Ensure /run/secrets/github_token has 'read:packages' scope." >&2
|
||||||
|
echo " Generate a new token at: https://github.com/settings/tokens" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker push $GHCR:$SHA
|
||||||
|
docker push $GHCR:latest
|
||||||
|
|
||||||
|
# --- 3. Create/update GHCR pull secret and DB secrets ---
|
||||||
|
echo "[3/6] Creating/updating GHCR credentials and database secrets..."
|
||||||
|
|
||||||
|
# Force-recreate the GHCR pull secret so pods can authenticate to pull images
|
||||||
|
kubectl delete secret ghcr-credentials -n "$NAMESPACE" --ignore-not-found
|
||||||
|
kubectl create secret docker-registry ghcr-credentials \
|
||||||
|
--namespace="$NAMESPACE" \
|
||||||
|
--docker-server=ghcr.io \
|
||||||
|
--docker-username=celesrenata \
|
||||||
|
--docker-password="$GHCR_TOKEN" \
|
||||||
|
--dry-run=client -o yaml | kubectl apply -f -
|
||||||
|
|
||||||
|
# Read MinIO credentials (same as stonks-oracle)
|
||||||
|
MINIO_ACCESS_KEY=$(_read_secret "$KUBE_DIR/minio.access_key" "AKIA6V7J3N9B5P0D2YQH")
|
||||||
|
MINIO_SECRET_KEY=$(_read_secret "$KUBE_DIR/minio.secret_key" "8fG3!v2rJ7\$wN@9mLpQ6zXbC4tKdPqW1")
|
||||||
|
|
||||||
|
kubectl delete secret dcos-db-secret -n "$NAMESPACE" --ignore-not-found
|
||||||
|
kubectl create secret generic dcos-db-secret \
|
||||||
|
--namespace="$NAMESPACE" \
|
||||||
|
--from-literal=POSTGRES_PASSWORD="$PG_PASSWORD" \
|
||||||
|
--from-literal=REDIS_PASSWORD="$REDIS_PASSWORD" \
|
||||||
|
--from-literal=MINIO_ACCESS_KEY="$MINIO_ACCESS_KEY" \
|
||||||
|
--from-literal=MINIO_SECRET_KEY="$MINIO_SECRET_KEY" \
|
||||||
|
--from-literal=MINIO_ENDPOINT="minio-crawler-console:9090"
|
||||||
|
|
||||||
|
# --- 3.5. Apply middleware (Traefik dependency, no version label to update) ---
|
||||||
|
echo "[3.5/6] Applying Traefik middleware..."
|
||||||
|
kubectl apply -f "$KUBE_DIR/middleware.yaml" -n "$NAMESPACE"
|
||||||
|
|
||||||
|
# --- 4. Apply PVCs (for SQLite fallback tiers: long_term, semantic, episodic) ---
|
||||||
|
echo "[4/6] Applying PVCs..."
|
||||||
|
kubectl apply -f "$KUBE_DIR/pvc.yaml" -n "$NAMESPACE"
|
||||||
|
|
||||||
|
# --- 5. Apply deployment manifests (inject SHA into image tag and version labels) ---
|
||||||
|
echo "[5/6] Applying K8s manifests..."
|
||||||
|
|
||||||
|
# Temporarily inject the SHA-based image tag and version labels
|
||||||
|
DEPLOY_TMP=$(mktemp)
|
||||||
|
sed -e "s|ghcr.io/celesrenata/dcos:[0-9.]*|$GHCR:$SHA|" \
|
||||||
|
-e 's|app.kubernetes.io/version: "[0-9.]*"|app.kubernetes.io/version: "'"$SHA"'"|' \
|
||||||
|
"$KUBE_DIR/deployment.yaml" > "$DEPLOY_TMP"
|
||||||
|
|
||||||
|
kubectl apply -f "$DEPLOY_TMP" -n "$NAMESPACE"
|
||||||
|
rm -f "$DEPLOY_TMP"
|
||||||
|
|
||||||
|
# Also update version labels in service.yaml and ingress.yaml
|
||||||
|
SVC_TMP=$(mktemp)
|
||||||
|
sed -e 's|app.kubernetes.io/version: "[0-9.]*"|app.kubernetes.io/version: "'"$SHA"'"|' \
|
||||||
|
"$KUBE_DIR/service.yaml" > "$SVC_TMP"
|
||||||
|
kubectl apply -f "$SVC_TMP" -n "$NAMESPACE"
|
||||||
|
rm -f "$SVC_TMP"
|
||||||
|
|
||||||
|
INGRESS_TMP=$(mktemp)
|
||||||
|
sed -e 's|app.kubernetes.io/version: "[0-9.]*"|app.kubernetes.io/version: "'"$SHA"'"|' \
|
||||||
|
"$KUBE_DIR/ingress.yaml" > "$INGRESS_TMP"
|
||||||
|
kubectl apply -f "$INGRESS_TMP" -n "$NAMESPACE"
|
||||||
|
rm -f "$INGRESS_TMP"
|
||||||
|
|
||||||
|
kubectl apply -f "$KUBE_DIR/serviceaccount.yaml" -n "$NAMESPACE"
|
||||||
|
|
||||||
|
# --- 6. Rolling restart to pick up new image ---
|
||||||
|
echo "[6/6] Rolling restart..."
|
||||||
|
kubectl rollout restart deployment/dcos -n "$NAMESPACE"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Deployment complete ==="
|
||||||
|
echo "Waiting for pods..."
|
||||||
|
sleep 10
|
||||||
|
kubectl get pods -n "$NAMESPACE" -o custom-columns='NAME:.metadata.name,READY:.status.containerStatuses[0].ready,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount'
|
||||||
|
echo ""
|
||||||
|
echo "Ingress endpoints:"
|
||||||
|
kubectl get ingress -n "$NAMESPACE" -o custom-columns='HOST:.spec.rules[0].host,ADDRESS:.status.loadBalancer.ingress[0].ip' 2>/dev/null || echo " (no ingress found)"
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Reverse of runmefirst.sh — tears down DCOS deployment but preserves PVCs (data survives)
|
||||||
|
# Usage: runmelast.sh
|
||||||
|
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
NAMESPACE="dcos-service"
|
||||||
|
|
||||||
|
echo "=== DCOS Teardown ==="
|
||||||
|
|
||||||
|
# --- 1. Delete all deployments and statefulsets ---
|
||||||
|
echo "[1/4] Deleting deployments..."
|
||||||
|
kubectl delete deployment dcos -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||||
|
kubectl delete statefulset --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||||
|
|
||||||
|
# --- 2. Delete services and ingresses ---
|
||||||
|
echo "[2/4] Cleaning up services..."
|
||||||
|
kubectl delete service dcos -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||||
|
kubectl delete ingress --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||||
|
|
||||||
|
# --- 3. Delete secrets and configmaps ---
|
||||||
|
echo "[3/4] Cleaning up secrets..."
|
||||||
|
kubectl delete secret dcos-db-secret -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||||
|
kubectl delete configmap --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||||
|
|
||||||
|
# --- 4. Delete PVCs (data preserved for redeploy) ---
|
||||||
|
echo "[4/4] Cleaning up PVCs..."
|
||||||
|
kubectl delete pvc --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
||||||
|
|
||||||
|
# NOTE: namespace is kept intact so Helm labels persist for clean redeploy
|
||||||
|
echo ""
|
||||||
|
echo "=== Teardown complete ==="
|
||||||
|
echo ""
|
||||||
|
echo "Preserved (untouched):"
|
||||||
|
echo " - Namespace $NAMESPACE (kept for Helm label compatibility)"
|
||||||
|
echo " - PostgreSQL database 'dcos' and user 'celes' in postgresql-service"
|
||||||
|
echo " - Redis data in redis-service"
|
||||||
|
echo ""
|
||||||
|
echo "To redeploy: bash ~/sources/kube/dcos/runmefirst.sh"
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: dcos
|
||||||
|
labels:
|
||||||
|
helm.sh/chart: dcos-0.1.0
|
||||||
|
app.kubernetes.io/name: dcos
|
||||||
|
app.kubernetes.io/instance: dcos
|
||||||
|
app.kubernetes.io/version: "1.1.0"
|
||||||
|
app.kubernetes.io/managed-by: Helm
|
||||||
|
annotations:
|
||||||
|
traefik.ingress.kubernetes.io/affinity: "true"
|
||||||
|
traefik.ingress.kubernetes.io/service.sticky.cookie: "true"
|
||||||
|
traefik.ingress.kubernetes.io/service.sticky.cookie.maxage: "28800"
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
ports:
|
||||||
|
- port: 8080
|
||||||
|
targetPort: http
|
||||||
|
protocol: TCP
|
||||||
|
name: http
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: dcos
|
||||||
|
app.kubernetes.io/instance: dcos
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: dcos
|
||||||
|
labels:
|
||||||
|
helm.sh/chart: dcos-0.1.0
|
||||||
|
app.kubernetes.io/name: dcos
|
||||||
|
app.kubernetes.io/instance: dcos
|
||||||
|
app.kubernetes.io/managed-by: Helm
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
# Default values for dcos.
|
||||||
|
# This is a YAML-formatted file.
|
||||||
|
# Declare variables to be passed into your templates.
|
||||||
|
|
||||||
|
maxAge: 8h
|
||||||
|
replicaCount: 1
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: ghcr.io/celesrenata/dcos
|
||||||
|
pullPolicy: IfNotPresent
|
||||||
|
# Overrides the image tag whose default is the chart appVersion.
|
||||||
|
tag: "1.1.0"
|
||||||
|
|
||||||
|
imagePullSecrets: []
|
||||||
|
nameOverride: ""
|
||||||
|
fullnameOverride: ""
|
||||||
|
|
||||||
|
serviceAccount:
|
||||||
|
# Specifies whether a service account should be created
|
||||||
|
create: true
|
||||||
|
# Automatically mount a ServiceAccount's API credentials?
|
||||||
|
automount: true
|
||||||
|
# Annotations to add to the service account
|
||||||
|
annotations: {}
|
||||||
|
# The name of the service account to use.
|
||||||
|
# If not set and create is true, a name is generated using the fullname template
|
||||||
|
name: ""
|
||||||
|
|
||||||
|
podAnnotations: {}
|
||||||
|
podLabels: {}
|
||||||
|
|
||||||
|
podSecurityContext: {}
|
||||||
|
# fsGroup: 2000
|
||||||
|
|
||||||
|
securityContext:
|
||||||
|
runAsUser: 1000
|
||||||
|
runAsGroup: 1000
|
||||||
|
fsGroup: 1000
|
||||||
|
|
||||||
|
service:
|
||||||
|
type: ClusterIP
|
||||||
|
port: 8080
|
||||||
|
annotations:
|
||||||
|
traefik.ingress.kubernetes.io/affinity: "true"
|
||||||
|
traefik.ingress.kubernetes.io/service.sticky.cookie: "true"
|
||||||
|
traefik.ingress.kubernetes.io/service.sticky.cookie.maxage: "28800"
|
||||||
|
traefik.ingress.kubernetes.io/max-age: "28800"
|
||||||
|
|
||||||
|
ingress:
|
||||||
|
enabled: true
|
||||||
|
annotations:
|
||||||
|
traefik.ingress.kubernetes.io/max-age: "28800"
|
||||||
|
traefik.ingress.kubernetes.io/router.middlewares: default-dcos-stripprefix@kubernetescrd
|
||||||
|
hosts:
|
||||||
|
- paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
name: dcos.celestium.life
|
||||||
|
tls:
|
||||||
|
- secretName: dcos-cert
|
||||||
|
hosts:
|
||||||
|
- dcos.celestium.life
|
||||||
|
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: "4"
|
||||||
|
memory: "4Gi"
|
||||||
|
requests:
|
||||||
|
cpu: "1"
|
||||||
|
memory: "1Gi"
|
||||||
|
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: http
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: http
|
||||||
|
|
||||||
|
autoscaling:
|
||||||
|
enabled: false
|
||||||
|
minReplicas: 1
|
||||||
|
maxReplicas: 10
|
||||||
|
targetCPUUtilizationPercentage: 80
|
||||||
|
# targetMemoryUtilizationPercentage: 80
|
||||||
|
|
||||||
|
nodeSelector: {}
|
||||||
|
|
||||||
|
tolerations: []
|
||||||
|
|
||||||
|
affinity: {}
|
||||||
|
|
||||||
|
# DCOS-specific configuration
|
||||||
|
dcos:
|
||||||
|
dataDir: /app/data/memory_store
|
||||||
|
configDir: /app/config
|
||||||
|
logLevel: INFO
|
||||||
|
maxAgents: 100
|
||||||
|
selfOrgEnabled: true
|
||||||
|
decentralized: true
|
||||||
Reference in New Issue
Block a user