diff --git a/infra/helm/stonks-oracle/values-beta.yaml b/infra/helm/stonks-oracle/values-beta.yaml index 1a1678e..abf2cf6 100644 --- a/infra/helm/stonks-oracle/values-beta.yaml +++ b/infra/helm/stonks-oracle/values-beta.yaml @@ -41,7 +41,7 @@ config: BROKER_PROVIDER: "alpaca" OLLAMA_BASE_URL: "http://10.1.1.12:2701" OLLAMA_MODEL: "qwen3.5:4b-fast" - VLLM_BASE_URL: "http://192.168.42.254:2701" + VLLM_BASE_URL: "http://vllm-external.vllm-service.svc.cluster.local:2701" VLLM_MODEL: "numind/NuExtract3" VLLM_TIMEOUT: "120" VLLM_MAX_RETRIES: "2" diff --git a/infra/helm/stonks-oracle/values-paper.yaml b/infra/helm/stonks-oracle/values-paper.yaml index 0040f76..43c7d6b 100644 --- a/infra/helm/stonks-oracle/values-paper.yaml +++ b/infra/helm/stonks-oracle/values-paper.yaml @@ -18,7 +18,7 @@ config: POSTGRES_USER: "stonks_paper" OLLAMA_BASE_URL: "http://10.1.1.12:2701" OLLAMA_MODEL: "qwen3.5:4b-fast" - VLLM_BASE_URL: "http://192.168.42.254:2701" + VLLM_BASE_URL: "http://vllm-external.vllm-service.svc.cluster.local:2701" VLLM_MODEL: "numind/NuExtract3" MARKET_DATA_BASE_URL: "https://api.polygon.io" diff --git a/infra/helm/stonks-oracle/values.yaml b/infra/helm/stonks-oracle/values.yaml index ee1c429..66d959d 100644 --- a/infra/helm/stonks-oracle/values.yaml +++ b/infra/helm/stonks-oracle/values.yaml @@ -192,7 +192,7 @@ config: OLLAMA_RETRY_BASE_DELAY: "1.0" OLLAMA_RETRY_MAX_DELAY: "10.0" OLLAMA_RETRY_BACKOFF_MULTIPLIER: "2.0" - VLLM_BASE_URL: "http://192.168.42.254:2701" + VLLM_BASE_URL: "http://vllm-external.vllm-service.svc.cluster.local:2701" VLLM_MODEL: "numind/NuExtract3" VLLM_TIMEOUT: "120" VLLM_MAX_RETRIES: "2" diff --git a/infra/kube-vllm/deployment.yaml b/infra/kube-vllm/deployment.yaml new file mode 100644 index 0000000..53060c3 --- /dev/null +++ b/infra/kube-vllm/deployment.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vllm + namespace: vllm-service + labels: + app: vllm +spec: + replicas: 1 + selector: + matchLabels: + app: vllm + template: + metadata: + labels: + app: vllm + spec: + runtimeClassName: nvidia + nodeSelector: + kubernetes.io/hostname: gremlin-1 + containers: + - name: vllm + image: registry.celestium.life/stonks-oracle/vllm-openai:latest + imagePullPolicy: Always + args: + - "--model" + - "numind/NuExtract3" + - "--served-model-name" + - "numind/NuExtract3" + - "--host" + - "0.0.0.0" + - "--port" + - "8000" + - "--gpu-memory-utilization" + - "0.45" + - "--max-model-len" + - "8192" + - "--max-num-seqs" + - "8" + env: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: vllm-secrets + key: HF_TOKEN + - name: VLLM_ATTENTION_BACKEND + value: "FLASHINFER" + ports: + - containerPort: 8000 + name: http + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "2" + memory: "8Gi" + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 120 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 300 + periodSeconds: 30 + volumeMounts: + - name: hf-cache + mountPath: /root/.cache/huggingface + volumes: + - name: hf-cache + persistentVolumeClaim: + claimName: vllm-hf-cache-pvc diff --git a/infra/kube-vllm/monitor.sh b/infra/kube-vllm/monitor.sh new file mode 100644 index 0000000..2ee5fa6 --- /dev/null +++ b/infra/kube-vllm/monitor.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +VLLM="http://10.1.1.12:31508" +GPU_HOST="root@10.1.1.12" +INTERVAL=3 + +while true; do + clear + echo "═══════════════════════════════════════════════════════════════" + echo " vLLM MONITOR (K8s) $(date '+%Y-%m-%d %H:%M:%S')" + echo "═══════════════════════════════════════════════════════════════" + + # GPU + gpu=$(ssh -o ConnectTimeout=2 -o BatchMode=yes "$GPU_HOST" \ + 'nvidia-smi --query-gpu=name,temperature.gpu,power.draw,power.limit,memory.used,memory.total,utilization.gpu --format=csv,noheader,nounits' 2>/dev/null) + if [ -n "$gpu" ]; then + IFS=',' read -r name temp power power_cap mem_used mem_total gpu_util <<< "$gpu" + mem_free=$(awk "BEGIN{printf \"%.0f\", $mem_total-$mem_used}") + echo "" + echo " GPU: ${name}" + echo " ├─ Temp: ${temp}°C Power: ${power}W / ${power_cap}W" + echo " ├─ VRAM: ${mem_used} / ${mem_total} MiB (${mem_free} MiB free)" + echo " └─ Util: ${gpu_util}%" + fi + + # vLLM model info + models_json=$(curl -sf --max-time 2 "$VLLM/v1/models" 2>/dev/null) + if [ -n "$models_json" ]; then + echo "" + python3 -c " +import json,sys +data = json.loads(sys.argv[1]) +for m in data.get('data',[]): + print(f' MODEL: {m[\"id\"]}') +" "$models_json" 2>/dev/null + fi + + # Prometheus metrics from vLLM /metrics endpoint + prom=$(curl -sf --max-time 2 "$VLLM/metrics" 2>/dev/null) + if [ -n "$prom" ]; then + python3 -c " +import sys +lines = sys.argv[1].split('\n') +def gauge(prefix): + for l in lines: + if l.startswith(prefix) and not l.startswith('#'): + return float(l.split()[-1]) + return 0 +def counter(prefix): + return sum(float(l.split()[-1]) for l in lines if l.startswith(prefix) and not l.startswith('#')) +def histo_avg(prefix): + s = counter(prefix + '_sum') + c = counter(prefix + '_count') + return s/c if c > 0 else 0 + +running = gauge('vllm:num_requests_running') +waiting = gauge('vllm:num_requests_waiting') +kv_pct = gauge('vllm:gpu_cache_usage_perc') * 100 +prompt_tok = counter('vllm:prompt_tokens_total') +gen_tok = counter('vllm:generation_tokens_total') +req_ok = counter('vllm:request_success_total') +preempts = counter('vllm:num_preemptions_total') + +ttft = histo_avg('vllm:time_to_first_token_seconds') +itl = histo_avg('vllm:inter_token_latency_seconds') +e2e = histo_avg('vllm:e2e_request_latency_seconds') +tok_s = 1/itl if itl > 0 else 0 + +print() +print(' REQUESTS:') +print(f' ├─ Running: {int(running)} Waiting: {int(waiting)}') +print(f' ├─ Completed: {int(req_ok)} Preemptions: {int(preempts)}') +print(f' └─ KV Cache: {kv_pct:.1f}%') +print() +print(' TOKENS:') +print(f' ├─ Prompt: {int(prompt_tok):,}') +print(f' └─ Generated: {int(gen_tok):,}') +print() +print(' LATENCY:') +print(f' ├─ TTFT: {ttft*1000:.0f}ms') +print(f' ├─ ITL: {itl*1000:.1f}ms') +print(f' ├─ Tok/s: {tok_s:.1f}') +print(f' └─ E2E avg: {e2e:.2f}s') +" "$prom" 2>/dev/null + else + echo "" + echo " METRICS: unreachable" + fi + + echo "" + echo "═══════════════════════════════════════════════════════════════" + echo " Ctrl+C to exit" + sleep "$INTERVAL" +done diff --git a/infra/kube-vllm/service.yaml b/infra/kube-vllm/service.yaml new file mode 100644 index 0000000..446c319 --- /dev/null +++ b/infra/kube-vllm/service.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Service +metadata: + name: vllm + namespace: vllm-service +spec: + ports: + - name: http + port: 8000 + targetPort: 8000 + selector: + app: vllm + type: ClusterIP +--- +# External access via NodePort (like ollama's port 2701 pattern) +apiVersion: v1 +kind: Service +metadata: + name: vllm-external + namespace: vllm-service +spec: + ports: + - nodePort: 31508 + name: vllm-api + port: 2701 + targetPort: 8000 + selector: + app: vllm-metrics + type: LoadBalancer diff --git a/infra/kube-vllm/storage.yaml b/infra/kube-vllm/storage.yaml new file mode 100644 index 0000000..4393f11 --- /dev/null +++ b/infra/kube-vllm/storage.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: vllm-hf-cache-pv +spec: + capacity: + storage: 50Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: local-path + hostPath: + path: /var/lib/vllm/hf-cache + type: DirectoryOrCreate + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - gremlin-1 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: vllm-hf-cache-pvc + namespace: vllm-service +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + storageClassName: local-path diff --git a/infra/kube-vllm/values.yaml b/infra/kube-vllm/values.yaml new file mode 100644 index 0000000..c14590a --- /dev/null +++ b/infra/kube-vllm/values.yaml @@ -0,0 +1,45 @@ +replicaCount: 1 + +image: + repository: vllm/vllm-openai + tag: latest + pullPolicy: Always + +resources: + limits: + nvidia.com/gpu: 1 + requests: + cpu: "2" + memory: "8Gi" + +runtimeClassName: nvidia + +env: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: vllm-secrets + key: HF_TOKEN + +args: + - "serve" + - "numind/NuExtract3" + - "--served-model-name" + - "numind/NuExtract3" + - "--host" + - "0.0.0.0" + - "--port" + - "8000" + - "--gpu-memory-utilization" + - "0.45" + - "--max-model-len" + - "8192" + - "--max-num-seqs" + - "8" + +service: + type: ClusterIP + port: 8000 + +nodeSelector: + kubernetes.io/hostname: gremlin-1 diff --git a/infra/kube-vllm/vllm-metrics.yaml b/infra/kube-vllm/vllm-metrics.yaml new file mode 100644 index 0000000..f894d9f --- /dev/null +++ b/infra/kube-vllm/vllm-metrics.yaml @@ -0,0 +1,74 @@ +# vLLM metrics proxy — similar to ollama-metrics +# Proxies requests and exposes Prometheus metrics +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vllm-metrics + namespace: vllm-service +spec: + replicas: 1 + selector: + matchLabels: + app: vllm-metrics + template: + metadata: + labels: + app: vllm-metrics + spec: + containers: + - name: proxy + image: nginx:alpine + ports: + - containerPort: 8080 + volumeMounts: + - name: nginx-conf + mountPath: /etc/nginx/conf.d/default.conf + subPath: default.conf + volumes: + - name: nginx-conf + configMap: + name: vllm-proxy-config +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vllm-proxy-config + namespace: vllm-service +data: + default.conf: | + upstream vllm_backend { + server vllm.vllm-service.svc.cluster.local:8000; + } + + server { + listen 8080; + + # API proxy + location / { + proxy_pass http://vllm_backend; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_connect_timeout 300s; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + } + + # vLLM exposes /metrics natively + location /metrics { + proxy_pass http://vllm_backend/metrics; + } + } +--- +apiVersion: v1 +kind: Service +metadata: + name: vllm-metrics + namespace: vllm-service +spec: + ports: + - name: proxy + port: 8080 + targetPort: 8080 + selector: + app: vllm-metrics + type: ClusterIP