Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d437b680e5 | ||
|
|
285a7ca305 |
+1
-3
@@ -1,9 +1,6 @@
|
|||||||
# Woodpecker CI pipeline for DCOS
|
# Woodpecker CI pipeline for DCOS
|
||||||
# Integrates with stonks-ci (stonks-oracle's CI/CD at stonks-ci.celestium.life)
|
# Integrates with stonks-ci (stonks-oracle's CI/CD at stonks-ci.celestium.life)
|
||||||
|
|
||||||
pipeline:
|
|
||||||
name: Build and Deploy DCOS
|
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: lint-and-test
|
- name: lint-and-test
|
||||||
image: python:3.12-slim
|
image: python:3.12-slim
|
||||||
@@ -36,3 +33,4 @@ pipeline:
|
|||||||
from_secret: github_token
|
from_secret: github_token
|
||||||
GHCR_TOKEN:
|
GHCR_TOKEN:
|
||||||
from_secret: ghcr_pull_token
|
from_secret: ghcr_pull_token
|
||||||
|
# Fix
|
||||||
|
|||||||
@@ -1,57 +0,0 @@
|
|||||||
# DCOS — Single-stage Docker image (mirrors stonks-oracle pipeline pattern)
|
|
||||||
# Production backends: Redis (working, conversation, user_model, world_model, procedural)
|
|
||||||
# SQLite fallback: long_term, semantic, episodic
|
|
||||||
|
|
||||||
FROM python:3.12-slim
|
|
||||||
|
|
||||||
ENV PYTHONDONTWRITEBYTECODE=1
|
|
||||||
ENV PYTHONUNBUFFERED=1
|
|
||||||
ENV PYTHONPATH=/app
|
|
||||||
|
|
||||||
# Install async database drivers (asyncpg needs libpq-dev)
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
||||||
gcc \
|
|
||||||
libpq-dev \
|
|
||||||
curl \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# Copy DCOS source into container (dcos package at /app/dcos/)
|
|
||||||
COPY src/dcos/ /app/dcos/
|
|
||||||
COPY config/ /app/config/
|
|
||||||
COPY data/ /app/data/
|
|
||||||
|
|
||||||
# Install dependencies (PyYAML for config, async database drivers)
|
|
||||||
RUN pip install --no-cache-dir \
|
|
||||||
"pyyaml>=6.0" \
|
|
||||||
"asyncpg==0.29.0" \
|
|
||||||
"redis>=5.0" \
|
|
||||||
"aiosqlite>=0.20" \
|
|
||||||
--break-system-packages
|
|
||||||
|
|
||||||
EXPOSE 8080
|
|
||||||
|
|
||||||
# Set environment defaults (overridden by K8s manifests at runtime)
|
|
||||||
ENV DCOS_DATA_DIR=/app/data/memory_store
|
|
||||||
ENV DCOS_CONFIG_DIR=/app/config
|
|
||||||
ENV DCOS_LOG_LEVEL=INFO
|
|
||||||
ENV DCOS_PG_HOST=postgresql-rw.postgresql-service.svc.cluster.local
|
|
||||||
ENV DCOS_PG_PORT=5432
|
|
||||||
ENV DCOS_PG_USER=celes
|
|
||||||
ENV DCOS_PG_DB=dcos
|
|
||||||
ENV DCOS_REDIS_HOST=redis-master
|
|
||||||
ENV DCOS_REDIS_PORT=6379
|
|
||||||
|
|
||||||
# Create non-root user (matches stonks-oracle pipeline convention) — must come before chown
|
|
||||||
RUN useradd -m -u 1000 stonks && \
|
|
||||||
mkdir -p /app/data/memory_store && \
|
|
||||||
chown -R stonks:stonks /app
|
|
||||||
|
|
||||||
USER stonks
|
|
||||||
|
|
||||||
# SERVICE_CMD ARG — same pattern as stonks-oracle's docker/Dockerfile
|
|
||||||
ARG SERVICE_CMD="python -m dcos.core --init"
|
|
||||||
ENV SERVICE_CMD=${SERVICE_CMD}
|
|
||||||
|
|
||||||
CMD ["sh", "-c", "${SERVICE_CMD}"]
|
|
||||||
@@ -1,217 +0,0 @@
|
|||||||
# DCOS — Kubernetes Deployment
|
|
||||||
|
|
||||||
Kubernetes deployment for the Distributed Cognitive Operating System. Follows the standard homelab pattern (Traefik ingress, cert-manager TLS, local-path PVCs).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Quick Deploy
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# One-command deploy (follows your pattern)
|
|
||||||
bash runmefirst.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## What Gets Deployed
|
|
||||||
|
|
||||||
| Resource | Name | Purpose |
|
|
||||||
|----------|------|---------|
|
|
||||||
| Namespace | `dcos-service` | Isolation |
|
|
||||||
| Deployment | `dcos` | Main DCOS pods |
|
|
||||||
| Service | `dcos` | ClusterIP:8080 |
|
|
||||||
| Ingress | `dcos` | `dcos.celestium.life` |
|
|
||||||
| PVC | `dcos-memory-pvc` | 20Gi memory store |
|
|
||||||
| PVC | `dcos-config-pvc` | 2Gi config |
|
|
||||||
| ServiceAccount | `dcos` | Pod identity |
|
|
||||||
| Middleware | `dcos-stripprefix` | Traefik path stripping |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Files
|
|
||||||
|
|
||||||
| File | Purpose |
|
|
||||||
|------|---------|
|
|
||||||
| [`deployment.yaml`](deployment.yaml) | Deployment: 1 replica, 4 CPU/4Gi RAM, PVC mounts |
|
|
||||||
| [`service.yaml`](service.yaml) | ClusterIP service on port 8080 |
|
|
||||||
| [`ingress.yaml`](ingress.yaml) | Traefik ingress → `dcos.celestium.life` with TLS |
|
|
||||||
| [`pvc.yaml`](pvc.yaml) | 20Gi memory + 2Gi config (local-path) |
|
|
||||||
| [`serviceaccount.yaml`](serviceaccount.yaml) | Service account |
|
|
||||||
| [`middleware.yaml`](middleware.yaml) | Traefik Middleware for path stripping |
|
|
||||||
| [`values.yaml`](values.yaml) | Helm values (image, resources, autoscaling) |
|
|
||||||
| [`runmefirst.sh`](runmefirst.sh) | Namespace + kubectl apply |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Helm Values
|
|
||||||
|
|
||||||
Key values in [`values.yaml`](values.yaml):
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
image:
|
|
||||||
repository: ghcr.io/celesrenata/dcos
|
|
||||||
tag: "1.0.0"
|
|
||||||
pullPolicy: IfNotPresent
|
|
||||||
|
|
||||||
service:
|
|
||||||
type: ClusterIP
|
|
||||||
port: 8080
|
|
||||||
|
|
||||||
ingress:
|
|
||||||
enabled: true
|
|
||||||
hosts:
|
|
||||||
- name: dcos.celestium.life
|
|
||||||
tls:
|
|
||||||
- secretName: dcos-cert
|
|
||||||
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
cpu: "4"
|
|
||||||
memory: "4Gi"
|
|
||||||
|
|
||||||
autoscaling:
|
|
||||||
enabled: false
|
|
||||||
minReplicas: 1
|
|
||||||
maxReplicas: 10
|
|
||||||
|
|
||||||
dcos:
|
|
||||||
maxAgents: 100
|
|
||||||
selfOrgEnabled: true
|
|
||||||
decentralized: true
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Deploy
|
|
||||||
bash runmefirst.sh
|
|
||||||
|
|
||||||
# Helm install
|
|
||||||
helm upgrade -i dcos . --namespace dcos-service --create-namespace -f values.yaml
|
|
||||||
|
|
||||||
# Helm upgrade with new image
|
|
||||||
helm upgrade -i dcos . --namespace dcos-service --set image.tag="1.0.1"
|
|
||||||
|
|
||||||
# Scale
|
|
||||||
kubectl scale deployment dcos --replicas=3 -n dcos-service
|
|
||||||
|
|
||||||
# Port-forward for local testing
|
|
||||||
kubectl port-forward service/dcos 8080:8080 -n dcos-service
|
|
||||||
|
|
||||||
# Check status
|
|
||||||
kubectl get all -n dcos-service
|
|
||||||
|
|
||||||
# View logs
|
|
||||||
kubectl logs -f deployment/dcos -n dcos-service
|
|
||||||
|
|
||||||
# Exec into pod
|
|
||||||
kubectl exec -it deployment/dcos -n dcos-service -- python -m dcos --init
|
|
||||||
|
|
||||||
# Uninstall
|
|
||||||
helm uninstall dcos --namespace dcos-service
|
|
||||||
kubectl delete -f pvc.yaml -n dcos-service
|
|
||||||
kubectl delete namespace dcos-service
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Storage
|
|
||||||
|
|
||||||
### PVC Details
|
|
||||||
|
|
||||||
| PVC | Size | StorageClass | Mount Path | Contents |
|
|
||||||
|-----|------|-------------|------------|----------|
|
|
||||||
| `dcos-memory-pvc` | 20Gi | local-path | `/app/data/memory_store` | SQLite databases |
|
|
||||||
| `dcos-config-pvc` | 2Gi | local-path | `/app/config` | YAML configs |
|
|
||||||
|
|
||||||
### Persistent Data
|
|
||||||
|
|
||||||
Memory databases are stored in `data/memory_store/`:
|
|
||||||
|
|
||||||
```
|
|
||||||
memory_store/
|
|
||||||
├── episodic.db # Episodic memory (temporal events)
|
|
||||||
├── knowledge_graph.db # Semantic memory (knowledge graph)
|
|
||||||
├── learning.db # Learning data
|
|
||||||
├── long_term.db # Long-term memory
|
|
||||||
├── procedural.db # Procedural memory (patterns)
|
|
||||||
├── user_model.db # User model
|
|
||||||
└── world_model.db # World model
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Networking
|
|
||||||
|
|
||||||
### Service
|
|
||||||
|
|
||||||
```
|
|
||||||
dcos.celestium.life
|
|
||||||
│
|
|
||||||
▼
|
|
||||||
Ingress (traefik) → TLS (cert-manager)
|
|
||||||
│
|
|
||||||
▼
|
|
||||||
Service (ClusterIP:8080) → Sticky cookies (8h)
|
|
||||||
│
|
|
||||||
▼
|
|
||||||
Pod:8080
|
|
||||||
```
|
|
||||||
|
|
||||||
### Ingress Annotations
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
traefik.ingress.kubernetes.io/affinity: "true"
|
|
||||||
traefik.ingress.kubernetes.io/service.sticky.cookie: "true"
|
|
||||||
traefik.ingress.kubernetes.io/service.sticky.cookie.maxage: "28800"
|
|
||||||
cert-manager.io/cluster-issuer: ca-issuer
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Monitoring
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Pod health
|
|
||||||
kubectl get pods -n dcos-service -w
|
|
||||||
|
|
||||||
# PVC usage
|
|
||||||
kubectl get pvc -n dcos-service
|
|
||||||
|
|
||||||
# Ingress
|
|
||||||
kubectl get ingress -n dcos-service
|
|
||||||
|
|
||||||
# Service endpoints
|
|
||||||
kubectl get endpoints -n dcos-service
|
|
||||||
|
|
||||||
# Resource usage
|
|
||||||
kubectl top pods -n dcos-service
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
| Problem | Check | Solution |
|
|
||||||
|---------|-------|----------|
|
|
||||||
| Pod not starting | `kubectl describe pod dcos-* -n dcos-service` | Check events for image pull errors |
|
|
||||||
| PVC pending | `kubectl get pvc -n dcos-service` | Verify `local-path` storage class exists |
|
|
||||||
| Ingress not routing | `kubectl get ingress -n dcos-service` | Check Traefik config |
|
|
||||||
| TLS errors | `kubectl get certificates -n dcos-service` | Verify cert-manager is running |
|
|
||||||
| Memory full | `kubectl exec dcoss-*/df -h /app/data` | Expand PVC or clean old data |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Image
|
|
||||||
|
|
||||||
```
|
|
||||||
Repository: ghcr.io/celesrenata/dcos
|
|
||||||
Tags: 1.0.0 (production), latest (development), <commit> (CI)
|
|
||||||
Pull Policy: IfNotPresent (production), Always (development)
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
*Deployed to `dcos-service` namespace. Ingress at `dcos.celestium.life`.*
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: Secret
|
|
||||||
metadata:
|
|
||||||
name: dcos-db-secret
|
|
||||||
namespace: dcos-service
|
|
||||||
stringData:
|
|
||||||
# PostgreSQL — celes user, dcos database (created via postInitSQL)
|
|
||||||
POSTGRES_PASSWORD: "PSCh4ng3me!"
|
|
||||||
POSTGRES_DSN: "postgresql://celes:PSCh4ng3me!@postgresql-rw:5432/dcos?sslmode=disable"
|
|
||||||
POSTGRES_RO_DSN: "postgresql://celes:PSCh4ng3me!@postgresql-ro:5432/dcos?sslmode=disable"
|
|
||||||
|
|
||||||
# Redis — master endpoint, password
|
|
||||||
REDIS_PASSWORD: "PSCh4ng3me!"
|
|
||||||
REDIS_ADDR: "redis-master:6379"
|
|
||||||
|
|
||||||
# MinIO S3 — for episodic snapshots and backup storage
|
|
||||||
MINIO_ACCESS_KEY: "AKIA6V7J3N9B5P0D2YQH"
|
|
||||||
MINIO_SECRET_KEY: "8fG3!v2rJ7$wN@9mLpQ6zXbC4tKdPqW1"
|
|
||||||
MINIO_ENDPOINT: "minio-crawler-console:9090"
|
|
||||||
type: Opaque
|
|
||||||
@@ -1,107 +0,0 @@
|
|||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: dcos
|
|
||||||
labels:
|
|
||||||
helm.sh/chart: dcos-0.1.0
|
|
||||||
app.kubernetes.io/name: dcos
|
|
||||||
app.kubernetes.io/instance: dcos
|
|
||||||
app.kubernetes.io/version: "1.0.0"
|
|
||||||
app.kubernetes.io/managed-by: Helm
|
|
||||||
spec:
|
|
||||||
replicas: 1
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app.kubernetes.io/name: dcos
|
|
||||||
app.kubernetes.io/instance: dcos
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
helm.sh/chart: dcos-0.1.0
|
|
||||||
app.kubernetes.io/name: dcos
|
|
||||||
app.kubernetes.io/instance: dcos
|
|
||||||
app.kubernetes.io/version: "1.0.0"
|
|
||||||
app.kubernetes.io/managed-by: Helm
|
|
||||||
spec:
|
|
||||||
imagePullSecrets:
|
|
||||||
- name: ghcr-credentials
|
|
||||||
serviceAccountName: dcos
|
|
||||||
securityContext:
|
|
||||||
runAsUser: 1000
|
|
||||||
runAsGroup: 1000
|
|
||||||
fsGroup: 1000
|
|
||||||
containers:
|
|
||||||
- name: dcos
|
|
||||||
image: "ghcr.io/celesrenata/dcos:dev"
|
|
||||||
imagePullPolicy: Always
|
|
||||||
ports:
|
|
||||||
- name: http
|
|
||||||
containerPort: 8080
|
|
||||||
protocol: TCP
|
|
||||||
env:
|
|
||||||
- name: DCOS_DATA_DIR
|
|
||||||
value: /app/data/memory_store
|
|
||||||
- name: DCOS_CONFIG_DIR
|
|
||||||
value: /app/config
|
|
||||||
- name: DCOS_LOG_LEVEL
|
|
||||||
value: "INFO"
|
|
||||||
# ──────────────────────────────────────────
|
|
||||||
# Database connection credentials
|
|
||||||
# Managed via dcos-db-secret
|
|
||||||
# ──────────────────────────────────────────
|
|
||||||
# Cross-namespace DNS requires FQDN (postgresql-rw is in postgresql-service namespace)
|
|
||||||
- name: DCOS_PG_HOST
|
|
||||||
value: "postgresql-rw.postgresql-service.svc.cluster.local"
|
|
||||||
- name: DCOS_PG_PORT
|
|
||||||
value: "5432"
|
|
||||||
- name: DCOS_PG_USER
|
|
||||||
value: "celes"
|
|
||||||
- name: DCOS_PG_DB
|
|
||||||
value: "dcos"
|
|
||||||
- name: DCOS_PG_PASSWORD
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: dcos-db-secret
|
|
||||||
key: POSTGRES_PASSWORD
|
|
||||||
# Cross-namespace DNS requires FQDN (redis-master is in redis-service namespace)
|
|
||||||
- name: DCOS_REDIS_HOST
|
|
||||||
value: "redis-master.redis-service.svc.cluster.local"
|
|
||||||
- name: DCOS_REDIS_PORT
|
|
||||||
value: "6379"
|
|
||||||
- name: DCOS_REDIS_PASSWORD
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: dcos-db-secret
|
|
||||||
key: REDIS_PASSWORD
|
|
||||||
- name: DCOS_MINIO_ENDPOINT
|
|
||||||
value: "minio-crawler-console:9090"
|
|
||||||
- name: DCOS_MINIO_ACCESS_KEY
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: dcos-db-secret
|
|
||||||
key: MINIO_ACCESS_KEY
|
|
||||||
- name: DCOS_MINIO_SECRET_KEY
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: dcos-db-secret
|
|
||||||
key: MINIO_SECRET_KEY
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
cpu: "4"
|
|
||||||
memory: "4Gi"
|
|
||||||
requests:
|
|
||||||
cpu: "1"
|
|
||||||
memory: "1Gi"
|
|
||||||
volumeMounts:
|
|
||||||
- name: memory-store
|
|
||||||
mountPath: /app/data/memory_store
|
|
||||||
- name: config
|
|
||||||
mountPath: /app/config
|
|
||||||
readOnly: true
|
|
||||||
volumes:
|
|
||||||
- name: memory-store
|
|
||||||
persistentVolumeClaim:
|
|
||||||
claimName: dcos-memory-pvc
|
|
||||||
- name: config
|
|
||||||
persistentVolumeClaim:
|
|
||||||
claimName: dcos-config-pvc
|
|
||||||
@@ -1,42 +0,0 @@
|
|||||||
apiVersion: networking.k8s.io/v1
|
|
||||||
kind: Ingress
|
|
||||||
metadata:
|
|
||||||
name: dcos
|
|
||||||
labels:
|
|
||||||
helm.sh/chart: dcos-0.1.0
|
|
||||||
app.kubernetes.io/name: dcos
|
|
||||||
app.kubernetes.io/instance: dcos
|
|
||||||
app.kubernetes.io/version: "1.1.0"
|
|
||||||
app.kubernetes.io/managed-by: Helm
|
|
||||||
annotations:
|
|
||||||
traefik.ingress.kubernetes.io/router.middlewares: default-dcos-stripprefix@kubernetescrd
|
|
||||||
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
|
||||||
cert-manager.io/cluster-issuer: ca-issuer
|
|
||||||
spec:
|
|
||||||
ingressClassName: traefik
|
|
||||||
rules:
|
|
||||||
- host: dcos.celestium.life
|
|
||||||
http:
|
|
||||||
paths:
|
|
||||||
- path: /
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: dcos
|
|
||||||
port:
|
|
||||||
number: 8080
|
|
||||||
tls:
|
|
||||||
- hosts:
|
|
||||||
- dcos.celestium.life
|
|
||||||
secretName: dcos-cert
|
|
||||||
---
|
|
||||||
# Source: dcos/templates/ingress.yaml
|
|
||||||
apiVersion: traefik.io/v1alpha1
|
|
||||||
kind: Middleware
|
|
||||||
metadata:
|
|
||||||
name: dcos-stripprefix
|
|
||||||
spec:
|
|
||||||
stripPrefix:
|
|
||||||
prefixes:
|
|
||||||
- /assets
|
|
||||||
- /dcos
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
apiVersion: traefik.io/v1alpha1
|
|
||||||
kind: Middleware
|
|
||||||
metadata:
|
|
||||||
name: dcos-stripprefix
|
|
||||||
spec:
|
|
||||||
stripPrefix:
|
|
||||||
prefixes:
|
|
||||||
- /assets
|
|
||||||
- /dcos
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
AKIA6V7J3N9B5P0D2YQH
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
8fG3!v2rJ7$wN@9mLpQ6zXbC4tKdPqW1
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
PSCh4ng3me!
|
|
||||||
@@ -1,25 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: PersistentVolumeClaim
|
|
||||||
metadata:
|
|
||||||
name: dcos-memory-pvc
|
|
||||||
namespace: dcos-service
|
|
||||||
spec:
|
|
||||||
accessModes:
|
|
||||||
- ReadWriteOnce
|
|
||||||
storageClassName: local-path
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
storage: 20Gi
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: PersistentVolumeClaim
|
|
||||||
metadata:
|
|
||||||
name: dcos-config-pvc
|
|
||||||
namespace: dcos-service
|
|
||||||
spec:
|
|
||||||
accessModes:
|
|
||||||
- ReadWriteOnce
|
|
||||||
storageClassName: local-path
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
storage: 2Gi
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
PSCh4ng3me!
|
|
||||||
@@ -1,160 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# Deploy DCOS to Kubernetes (standalone pipeline, mirrors stonks-oracle pattern)
|
|
||||||
# Usage: runmefirst.sh
|
|
||||||
# Detects existing deploy and adds/updates; idempotent.
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
NAMESPACE="dcos-service"
|
|
||||||
REPO_DIR="$HOME/sources/sama/sama"
|
|
||||||
KUBE_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
||||||
GHCR="ghcr.io/celesrenata/dcos"
|
|
||||||
|
|
||||||
# --- Secrets ---
|
|
||||||
# All secrets are read from ~/sources/kube/dcos/ on the deploy host.
|
|
||||||
# This directory is NOT a git repo — secrets stay local to the deploy host.
|
|
||||||
#
|
|
||||||
# Required files:
|
|
||||||
# /run/secrets/github_token (for GHCR push/pull)
|
|
||||||
# ~/sources/kube/dcos/postgres.password (PG password for dcos user)
|
|
||||||
# ~/sources/kube/dcos/redis.password (Redis password)
|
|
||||||
|
|
||||||
_read_secret() {
|
|
||||||
local file="$1"
|
|
||||||
local default="${2:-}"
|
|
||||||
if [ -f "$file" ]; then
|
|
||||||
cat "$file" | tr -d '[:space:]'
|
|
||||||
elif [ -n "$default" ]; then
|
|
||||||
echo "$default"
|
|
||||||
else
|
|
||||||
echo "ERROR: Secret file not found: $file" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
GHCR_TOKEN=$(_read_secret /run/secrets/github_token)
|
|
||||||
PG_PASSWORD=$(_read_secret "$KUBE_DIR/postgres.password")
|
|
||||||
REDIS_PASSWORD=$(_read_secret "$KUBE_DIR/redis.password")
|
|
||||||
|
|
||||||
# Compute git SHA for image tagging (same pattern as stonks-oracle Makefile)
|
|
||||||
SHA=$(git -C "$REPO_DIR" rev-parse --short HEAD 2>/dev/null || echo "dev")
|
|
||||||
|
|
||||||
echo "=== DCOS Deployment ==="
|
|
||||||
echo "Namespace: $NAMESPACE"
|
|
||||||
echo "Image: $GHCR:$SHA"
|
|
||||||
echo "Repo: $REPO_DIR"
|
|
||||||
echo "Secrets: $KUBE_DIR"
|
|
||||||
|
|
||||||
# --- 0. Pull latest code ---
|
|
||||||
echo "[0/6] Pulling latest code..."
|
|
||||||
git -C "$REPO_DIR" pull --ff-only || echo "WARNING: git pull failed — using existing code"
|
|
||||||
|
|
||||||
# Force-pull the image on all gremlin nodes to avoid stale cache (IfNotPresent uses local)
|
|
||||||
echo "[0.5/6] Forcing fresh image pull on cluster nodes..."
|
|
||||||
for node in $(kubectl get nodes -o name 2>/dev/null | sed 's/node\.//'); do
|
|
||||||
echo " Pulling on $node..."
|
|
||||||
kubectl debug node/$node --image=busybox:1.36 -- chroot /host sh -c "docker rmi ghcr.io/celesrenata/dcos:$SHA 2>/dev/null; docker pull ghcr.io/celesrenata/dcos:$SHA" || true
|
|
||||||
done
|
|
||||||
|
|
||||||
# --- 1. Ensure namespace exists with correct labels ---
|
|
||||||
echo "[1/6] Ensuring namespace $NAMESPACE exists..."
|
|
||||||
if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
|
|
||||||
kubectl create namespace "$NAMESPACE"
|
|
||||||
fi
|
|
||||||
kubectl label namespace "$NAMESPACE" app.kubernetes.io/managed-by=Helm --overwrite
|
|
||||||
kubectl annotate namespace "$NAMESPACE" meta.helm.sh/release-name=dcos meta.helm.sh/release-namespace=$NAMESPACE --overwrite
|
|
||||||
|
|
||||||
# --- 2. Build and push image ---
|
|
||||||
echo "[2/6] Building DCOS image..."
|
|
||||||
docker build \
|
|
||||||
--build-arg "SERVICE_CMD=python -m dcos.core --init" \
|
|
||||||
-t $GHCR:$SHA \
|
|
||||||
-t $GHCR:latest \
|
|
||||||
-f "$REPO_DIR/docker/Dockerfile.dcos" "$REPO_DIR" || {
|
|
||||||
echo "ERROR: Docker build failed." >&2
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
echo "Pushing image..."
|
|
||||||
if ! echo "$GHCR_TOKEN" | docker login ghcr.io -u celesrenata --password-stdin >/dev/null 2>&1; then
|
|
||||||
echo "WARNING: GHCR login failed — token may be expired or missing scopes." >&2
|
|
||||||
echo " Ensure /run/secrets/github_token has 'read:packages' scope." >&2
|
|
||||||
echo " Generate a new token at: https://github.com/settings/tokens" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
docker push $GHCR:$SHA
|
|
||||||
docker push $GHCR:latest
|
|
||||||
|
|
||||||
# --- 3. Create/update GHCR pull secret and DB secrets ---
|
|
||||||
echo "[3/6] Creating/updating GHCR credentials and database secrets..."
|
|
||||||
|
|
||||||
# Force-recreate the GHCR pull secret so pods can authenticate to pull images
|
|
||||||
kubectl delete secret ghcr-credentials -n "$NAMESPACE" --ignore-not-found
|
|
||||||
kubectl create secret docker-registry ghcr-credentials \
|
|
||||||
--namespace="$NAMESPACE" \
|
|
||||||
--docker-server=ghcr.io \
|
|
||||||
--docker-username=celesrenata \
|
|
||||||
--docker-password="$GHCR_TOKEN" \
|
|
||||||
--dry-run=client -o yaml | kubectl apply -f -
|
|
||||||
|
|
||||||
# Read MinIO credentials (same as stonks-oracle)
|
|
||||||
MINIO_ACCESS_KEY=$(_read_secret "$KUBE_DIR/minio.access_key" "AKIA6V7J3N9B5P0D2YQH")
|
|
||||||
MINIO_SECRET_KEY=$(_read_secret "$KUBE_DIR/minio.secret_key" "8fG3!v2rJ7\$wN@9mLpQ6zXbC4tKdPqW1")
|
|
||||||
|
|
||||||
kubectl delete secret dcos-db-secret -n "$NAMESPACE" --ignore-not-found
|
|
||||||
kubectl create secret generic dcos-db-secret \
|
|
||||||
--namespace="$NAMESPACE" \
|
|
||||||
--from-literal=POSTGRES_PASSWORD="$PG_PASSWORD" \
|
|
||||||
--from-literal=REDIS_PASSWORD="$REDIS_PASSWORD" \
|
|
||||||
--from-literal=MINIO_ACCESS_KEY="$MINIO_ACCESS_KEY" \
|
|
||||||
--from-literal=MINIO_SECRET_KEY="$MINIO_SECRET_KEY" \
|
|
||||||
--from-literal=MINIO_ENDPOINT="minio-crawler-console:9090"
|
|
||||||
|
|
||||||
# --- 3.5. Apply middleware (Traefik dependency, no version label to update) ---
|
|
||||||
echo "[3.5/6] Applying Traefik middleware..."
|
|
||||||
kubectl apply -f "$KUBE_DIR/middleware.yaml" -n "$NAMESPACE"
|
|
||||||
|
|
||||||
# --- 4. Apply PVCs (for SQLite fallback tiers: long_term, semantic, episodic) ---
|
|
||||||
echo "[4/6] Applying PVCs..."
|
|
||||||
kubectl apply -f "$KUBE_DIR/pvc.yaml" -n "$NAMESPACE"
|
|
||||||
|
|
||||||
# --- 5. Apply deployment manifests (inject SHA into image tag and version labels) ---
|
|
||||||
echo "[5/6] Applying K8s manifests..."
|
|
||||||
|
|
||||||
# Temporarily inject the SHA-based image tag and version labels
|
|
||||||
DEPLOY_TMP=$(mktemp)
|
|
||||||
sed -e "s|ghcr.io/celesrenata/dcos:[0-9.]*|$GHCR:$SHA|" \
|
|
||||||
-e 's|app.kubernetes.io/version: "[0-9.]*"|app.kubernetes.io/version: "'"$SHA"'"|' \
|
|
||||||
"$KUBE_DIR/deployment.yaml" > "$DEPLOY_TMP"
|
|
||||||
|
|
||||||
kubectl apply -f "$DEPLOY_TMP" -n "$NAMESPACE"
|
|
||||||
rm -f "$DEPLOY_TMP"
|
|
||||||
|
|
||||||
# Also update version labels in service.yaml and ingress.yaml
|
|
||||||
SVC_TMP=$(mktemp)
|
|
||||||
sed -e 's|app.kubernetes.io/version: "[0-9.]*"|app.kubernetes.io/version: "'"$SHA"'"|' \
|
|
||||||
"$KUBE_DIR/service.yaml" > "$SVC_TMP"
|
|
||||||
kubectl apply -f "$SVC_TMP" -n "$NAMESPACE"
|
|
||||||
rm -f "$SVC_TMP"
|
|
||||||
|
|
||||||
INGRESS_TMP=$(mktemp)
|
|
||||||
sed -e 's|app.kubernetes.io/version: "[0-9.]*"|app.kubernetes.io/version: "'"$SHA"'"|' \
|
|
||||||
"$KUBE_DIR/ingress.yaml" > "$INGRESS_TMP"
|
|
||||||
kubectl apply -f "$INGRESS_TMP" -n "$NAMESPACE"
|
|
||||||
rm -f "$INGRESS_TMP"
|
|
||||||
|
|
||||||
kubectl apply -f "$KUBE_DIR/serviceaccount.yaml" -n "$NAMESPACE"
|
|
||||||
|
|
||||||
# --- 6. Rolling restart to pick up new image ---
|
|
||||||
echo "[6/6] Rolling restart..."
|
|
||||||
kubectl rollout restart deployment/dcos -n "$NAMESPACE"
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "=== Deployment complete ==="
|
|
||||||
echo "Waiting for pods..."
|
|
||||||
sleep 10
|
|
||||||
kubectl get pods -n "$NAMESPACE" -o custom-columns='NAME:.metadata.name,READY:.status.containerStatuses[0].ready,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount'
|
|
||||||
echo ""
|
|
||||||
echo "Ingress endpoints:"
|
|
||||||
kubectl get ingress -n "$NAMESPACE" -o custom-columns='HOST:.spec.rules[0].host,ADDRESS:.status.loadBalancer.ingress[0].ip' 2>/dev/null || echo " (no ingress found)"
|
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# Reverse of runmefirst.sh — tears down DCOS deployment but preserves PVCs (data survives)
|
|
||||||
# Usage: runmelast.sh
|
|
||||||
|
|
||||||
set -uo pipefail
|
|
||||||
|
|
||||||
NAMESPACE="dcos-service"
|
|
||||||
|
|
||||||
echo "=== DCOS Teardown ==="
|
|
||||||
|
|
||||||
# --- 1. Delete all deployments and statefulsets ---
|
|
||||||
echo "[1/4] Deleting deployments..."
|
|
||||||
kubectl delete deployment dcos -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
|
||||||
kubectl delete statefulset --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
|
||||||
|
|
||||||
# --- 2. Delete services and ingresses ---
|
|
||||||
echo "[2/4] Cleaning up services..."
|
|
||||||
kubectl delete service dcos -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
|
||||||
kubectl delete ingress --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
|
||||||
|
|
||||||
# --- 3. Delete secrets and configmaps ---
|
|
||||||
echo "[3/4] Cleaning up secrets..."
|
|
||||||
kubectl delete secret dcos-db-secret -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
|
||||||
kubectl delete configmap --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
|
||||||
|
|
||||||
# --- 4. Delete PVCs (data preserved for redeploy) ---
|
|
||||||
echo "[4/4] Cleaning up PVCs..."
|
|
||||||
kubectl delete pvc --all -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
|
|
||||||
|
|
||||||
# NOTE: namespace is kept intact so Helm labels persist for clean redeploy
|
|
||||||
echo ""
|
|
||||||
echo "=== Teardown complete ==="
|
|
||||||
echo ""
|
|
||||||
echo "Preserved (untouched):"
|
|
||||||
echo " - Namespace $NAMESPACE (kept for Helm label compatibility)"
|
|
||||||
echo " - PostgreSQL database 'dcos' and user 'celes' in postgresql-service"
|
|
||||||
echo " - Redis data in redis-service"
|
|
||||||
echo ""
|
|
||||||
echo "To redeploy: bash ~/sources/kube/dcos/runmefirst.sh"
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: dcos
|
|
||||||
labels:
|
|
||||||
helm.sh/chart: dcos-0.1.0
|
|
||||||
app.kubernetes.io/name: dcos
|
|
||||||
app.kubernetes.io/instance: dcos
|
|
||||||
app.kubernetes.io/version: "1.1.0"
|
|
||||||
app.kubernetes.io/managed-by: Helm
|
|
||||||
annotations:
|
|
||||||
traefik.ingress.kubernetes.io/affinity: "true"
|
|
||||||
traefik.ingress.kubernetes.io/service.sticky.cookie: "true"
|
|
||||||
traefik.ingress.kubernetes.io/service.sticky.cookie.maxage: "28800"
|
|
||||||
spec:
|
|
||||||
type: ClusterIP
|
|
||||||
ports:
|
|
||||||
- port: 8080
|
|
||||||
targetPort: http
|
|
||||||
protocol: TCP
|
|
||||||
name: http
|
|
||||||
selector:
|
|
||||||
app.kubernetes.io/name: dcos
|
|
||||||
app.kubernetes.io/instance: dcos
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: ServiceAccount
|
|
||||||
metadata:
|
|
||||||
name: dcos
|
|
||||||
labels:
|
|
||||||
helm.sh/chart: dcos-0.1.0
|
|
||||||
app.kubernetes.io/name: dcos
|
|
||||||
app.kubernetes.io/instance: dcos
|
|
||||||
app.kubernetes.io/managed-by: Helm
|
|
||||||
@@ -1,101 +0,0 @@
|
|||||||
# Default values for dcos.
|
|
||||||
# This is a YAML-formatted file.
|
|
||||||
# Declare variables to be passed into your templates.
|
|
||||||
|
|
||||||
maxAge: 8h
|
|
||||||
replicaCount: 1
|
|
||||||
|
|
||||||
image:
|
|
||||||
repository: ghcr.io/celesrenata/dcos
|
|
||||||
pullPolicy: IfNotPresent
|
|
||||||
# Overrides the image tag whose default is the chart appVersion.
|
|
||||||
tag: "1.1.0"
|
|
||||||
|
|
||||||
imagePullSecrets: []
|
|
||||||
nameOverride: ""
|
|
||||||
fullnameOverride: ""
|
|
||||||
|
|
||||||
serviceAccount:
|
|
||||||
# Specifies whether a service account should be created
|
|
||||||
create: true
|
|
||||||
# Automatically mount a ServiceAccount's API credentials?
|
|
||||||
automount: true
|
|
||||||
# Annotations to add to the service account
|
|
||||||
annotations: {}
|
|
||||||
# The name of the service account to use.
|
|
||||||
# If not set and create is true, a name is generated using the fullname template
|
|
||||||
name: ""
|
|
||||||
|
|
||||||
podAnnotations: {}
|
|
||||||
podLabels: {}
|
|
||||||
|
|
||||||
podSecurityContext: {}
|
|
||||||
# fsGroup: 2000
|
|
||||||
|
|
||||||
securityContext:
|
|
||||||
runAsUser: 1000
|
|
||||||
runAsGroup: 1000
|
|
||||||
fsGroup: 1000
|
|
||||||
|
|
||||||
service:
|
|
||||||
type: ClusterIP
|
|
||||||
port: 8080
|
|
||||||
annotations:
|
|
||||||
traefik.ingress.kubernetes.io/affinity: "true"
|
|
||||||
traefik.ingress.kubernetes.io/service.sticky.cookie: "true"
|
|
||||||
traefik.ingress.kubernetes.io/service.sticky.cookie.maxage: "28800"
|
|
||||||
traefik.ingress.kubernetes.io/max-age: "28800"
|
|
||||||
|
|
||||||
ingress:
|
|
||||||
enabled: true
|
|
||||||
annotations:
|
|
||||||
traefik.ingress.kubernetes.io/max-age: "28800"
|
|
||||||
traefik.ingress.kubernetes.io/router.middlewares: default-dcos-stripprefix@kubernetescrd
|
|
||||||
hosts:
|
|
||||||
- paths:
|
|
||||||
- path: /
|
|
||||||
pathType: Prefix
|
|
||||||
name: dcos.celestium.life
|
|
||||||
tls:
|
|
||||||
- secretName: dcos-cert
|
|
||||||
hosts:
|
|
||||||
- dcos.celestium.life
|
|
||||||
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
cpu: "4"
|
|
||||||
memory: "4Gi"
|
|
||||||
requests:
|
|
||||||
cpu: "1"
|
|
||||||
memory: "1Gi"
|
|
||||||
|
|
||||||
livenessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /health
|
|
||||||
port: http
|
|
||||||
readinessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /health
|
|
||||||
port: http
|
|
||||||
|
|
||||||
autoscaling:
|
|
||||||
enabled: false
|
|
||||||
minReplicas: 1
|
|
||||||
maxReplicas: 10
|
|
||||||
targetCPUUtilizationPercentage: 80
|
|
||||||
# targetMemoryUtilizationPercentage: 80
|
|
||||||
|
|
||||||
nodeSelector: {}
|
|
||||||
|
|
||||||
tolerations: []
|
|
||||||
|
|
||||||
affinity: {}
|
|
||||||
|
|
||||||
# DCOS-specific configuration
|
|
||||||
dcos:
|
|
||||||
dataDir: /app/data/memory_store
|
|
||||||
configDir: /app/config
|
|
||||||
logLevel: INFO
|
|
||||||
maxAgents: 100
|
|
||||||
selfOrgEnabled: true
|
|
||||||
decentralized: true
|
|
||||||
Reference in New Issue
Block a user