feat: vLLM in K8s via Harbor mirror, pipelines point to internal svc

- vLLM image mirrored to registry.celestium.life/stonks-oracle/vllm-openai - Deployment uses Harbor image (Docker Hub IPv6 unreachable from cluster) - All 3 pipelines use vllm-external.vllm-service.svc.cluster.local:2701 - K8s manifests at infra/kube-vllm/ and synced to ~/sources/kube/vllm
2026-07-03 17:02:31 +00:00
parent 3a9894cd03
commit ecade0dd52
9 changed files with 354 additions and 3 deletions
@@ -41,7 +41,7 @@ config:
  BROKER_PROVIDER: "alpaca"
  OLLAMA_BASE_URL: "http://10.1.1.12:2701"
  OLLAMA_MODEL: "qwen3.5:4b-fast"
-  VLLM_BASE_URL: "http://192.168.42.254:2701"
+  VLLM_BASE_URL: "http://vllm-external.vllm-service.svc.cluster.local:2701"
  VLLM_MODEL: "numind/NuExtract3"
  VLLM_TIMEOUT: "120"
  VLLM_MAX_RETRIES: "2"
@@ -18,7 +18,7 @@ config:
  POSTGRES_USER: "stonks_paper"
  OLLAMA_BASE_URL: "http://10.1.1.12:2701"
  OLLAMA_MODEL: "qwen3.5:4b-fast"
-  VLLM_BASE_URL: "http://192.168.42.254:2701"
+  VLLM_BASE_URL: "http://vllm-external.vllm-service.svc.cluster.local:2701"
  VLLM_MODEL: "numind/NuExtract3"
  MARKET_DATA_BASE_URL: "https://api.polygon.io"

@@ -192,7 +192,7 @@ config:
  OLLAMA_RETRY_BASE_DELAY: "1.0"
  OLLAMA_RETRY_MAX_DELAY: "10.0"
  OLLAMA_RETRY_BACKOFF_MULTIPLIER: "2.0"
-  VLLM_BASE_URL: "http://192.168.42.254:2701"
+  VLLM_BASE_URL: "http://vllm-external.vllm-service.svc.cluster.local:2701"
  VLLM_MODEL: "numind/NuExtract3"
  VLLM_TIMEOUT: "120"
  VLLM_MAX_RETRIES: "2"
@@ -0,0 +1,75 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: vllm
+  namespace: vllm-service
+  labels:
+    app: vllm
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: vllm
+  template:
+    metadata:
+      labels:
+        app: vllm
+    spec:
+      runtimeClassName: nvidia
+      nodeSelector:
+        kubernetes.io/hostname: gremlin-1
+      containers:
+        - name: vllm
+          image: registry.celestium.life/stonks-oracle/vllm-openai:latest
+          imagePullPolicy: Always
+          args:
+            - "--model"
+            - "numind/NuExtract3"
+            - "--served-model-name"
+            - "numind/NuExtract3"
+            - "--host"
+            - "0.0.0.0"
+            - "--port"
+            - "8000"
+            - "--gpu-memory-utilization"
+            - "0.45"
+            - "--max-model-len"
+            - "8192"
+            - "--max-num-seqs"
+            - "8"
+          env:
+            - name: HF_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: vllm-secrets
+                  key: HF_TOKEN
+            - name: VLLM_ATTENTION_BACKEND
+              value: "FLASHINFER"
+          ports:
+            - containerPort: 8000
+              name: http
+          resources:
+            limits:
+              nvidia.com/gpu: "1"
+            requests:
+              cpu: "2"
+              memory: "8Gi"
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8000
+            initialDelaySeconds: 120
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8000
+            initialDelaySeconds: 300
+            periodSeconds: 30
+          volumeMounts:
+            - name: hf-cache
+              mountPath: /root/.cache/huggingface
+      volumes:
+        - name: hf-cache
+          persistentVolumeClaim:
+            claimName: vllm-hf-cache-pvc
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+VLLM="http://10.1.1.12:31508"
+GPU_HOST="root@10.1.1.12"
+INTERVAL=3
+
+while true; do
+  clear
+  echo "═══════════════════════════════════════════════════════════════"
+  echo "  vLLM MONITOR (K8s)  $(date '+%Y-%m-%d %H:%M:%S')"
+  echo "═══════════════════════════════════════════════════════════════"
+
+  # GPU
+  gpu=$(ssh -o ConnectTimeout=2 -o BatchMode=yes "$GPU_HOST" \
+    'nvidia-smi --query-gpu=name,temperature.gpu,power.draw,power.limit,memory.used,memory.total,utilization.gpu --format=csv,noheader,nounits' 2>/dev/null)
+  if [ -n "$gpu" ]; then
+    IFS=',' read -r name temp power power_cap mem_used mem_total gpu_util <<< "$gpu"
+    mem_free=$(awk "BEGIN{printf \"%.0f\", $mem_total-$mem_used}")
+    echo ""
+    echo "  GPU: ${name}"
+    echo "  ├─ Temp:  ${temp}°C    Power: ${power}W / ${power_cap}W"
+    echo "  ├─ VRAM:  ${mem_used} / ${mem_total} MiB  (${mem_free} MiB free)"
+    echo "  └─ Util:  ${gpu_util}%"
+  fi
+
+  # vLLM model info
+  models_json=$(curl -sf --max-time 2 "$VLLM/v1/models" 2>/dev/null)
+  if [ -n "$models_json" ]; then
+    echo ""
+    python3 -c "
+import json,sys
+data = json.loads(sys.argv[1])
+for m in data.get('data',[]):
+    print(f'  MODEL: {m[\"id\"]}')
+" "$models_json" 2>/dev/null
+  fi
+
+  # Prometheus metrics from vLLM /metrics endpoint
+  prom=$(curl -sf --max-time 2 "$VLLM/metrics" 2>/dev/null)
+  if [ -n "$prom" ]; then
+    python3 -c "
+import sys
+lines = sys.argv[1].split('\n')
+def gauge(prefix):
+    for l in lines:
+        if l.startswith(prefix) and not l.startswith('#'):
+            return float(l.split()[-1])
+    return 0
+def counter(prefix):
+    return sum(float(l.split()[-1]) for l in lines if l.startswith(prefix) and not l.startswith('#'))
+def histo_avg(prefix):
+    s = counter(prefix + '_sum')
+    c = counter(prefix + '_count')
+    return s/c if c > 0 else 0
+
+running = gauge('vllm:num_requests_running')
+waiting = gauge('vllm:num_requests_waiting')
+kv_pct  = gauge('vllm:gpu_cache_usage_perc') * 100
+prompt_tok = counter('vllm:prompt_tokens_total')
+gen_tok    = counter('vllm:generation_tokens_total')
+req_ok     = counter('vllm:request_success_total')
+preempts   = counter('vllm:num_preemptions_total')
+
+ttft   = histo_avg('vllm:time_to_first_token_seconds')
+itl    = histo_avg('vllm:inter_token_latency_seconds')
+e2e    = histo_avg('vllm:e2e_request_latency_seconds')
+tok_s  = 1/itl if itl > 0 else 0
+
+print()
+print('  REQUESTS:')
+print(f'  ├─ Running:    {int(running)}    Waiting: {int(waiting)}')
+print(f'  ├─ Completed:  {int(req_ok)}     Preemptions: {int(preempts)}')
+print(f'  └─ KV Cache:   {kv_pct:.1f}%')
+print()
+print('  TOKENS:')
+print(f'  ├─ Prompt:     {int(prompt_tok):,}')
+print(f'  └─ Generated:  {int(gen_tok):,}')
+print()
+print('  LATENCY:')
+print(f'  ├─ TTFT:       {ttft*1000:.0f}ms')
+print(f'  ├─ ITL:        {itl*1000:.1f}ms')
+print(f'  ├─ Tok/s:      {tok_s:.1f}')
+print(f'  └─ E2E avg:    {e2e:.2f}s')
+" "$prom" 2>/dev/null
+  else
+    echo ""
+    echo "  METRICS: unreachable"
+  fi
+
+  echo ""
+  echo "═══════════════════════════════════════════════════════════════"
+  echo "  Ctrl+C to exit"
+  sleep "$INTERVAL"
+done
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: vllm
+  namespace: vllm-service
+spec:
+  ports:
+    - name: http
+      port: 8000
+      targetPort: 8000
+  selector:
+    app: vllm
+  type: ClusterIP
+---
+# External access via NodePort (like ollama's port 2701 pattern)
+apiVersion: v1
+kind: Service
+metadata:
+  name: vllm-external
+  namespace: vllm-service
+spec:
+  ports:
+    - nodePort: 31508
+      name: vllm-api
+      port: 2701
+      targetPort: 8000
+  selector:
+    app: vllm-metrics
+  type: LoadBalancer
@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: vllm-hf-cache-pv
+spec:
+  capacity:
+    storage: 50Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: local-path
+  hostPath:
+    path: /var/lib/vllm/hf-cache
+    type: DirectoryOrCreate
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - gremlin-1
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: vllm-hf-cache-pvc
+  namespace: vllm-service
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 50Gi
+  storageClassName: local-path
@@ -0,0 +1,45 @@
+replicaCount: 1
+
+image:
+  repository: vllm/vllm-openai
+  tag: latest
+  pullPolicy: Always
+
+resources:
+  limits:
+    nvidia.com/gpu: 1
+  requests:
+    cpu: "2"
+    memory: "8Gi"
+
+runtimeClassName: nvidia
+
+env:
+  - name: HF_TOKEN
+    valueFrom:
+      secretKeyRef:
+        name: vllm-secrets
+        key: HF_TOKEN
+
+args:
+  - "serve"
+  - "numind/NuExtract3"
+  - "--served-model-name"
+  - "numind/NuExtract3"
+  - "--host"
+  - "0.0.0.0"
+  - "--port"
+  - "8000"
+  - "--gpu-memory-utilization"
+  - "0.45"
+  - "--max-model-len"
+  - "8192"
+  - "--max-num-seqs"
+  - "8"
+
+service:
+  type: ClusterIP
+  port: 8000
+
+nodeSelector:
+  kubernetes.io/hostname: gremlin-1
@@ -0,0 +1,74 @@
+# vLLM metrics proxy — similar to ollama-metrics
+# Proxies requests and exposes Prometheus metrics
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: vllm-metrics
+  namespace: vllm-service
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: vllm-metrics
+  template:
+    metadata:
+      labels:
+        app: vllm-metrics
+    spec:
+      containers:
+        - name: proxy
+          image: nginx:alpine
+          ports:
+            - containerPort: 8080
+          volumeMounts:
+            - name: nginx-conf
+              mountPath: /etc/nginx/conf.d/default.conf
+              subPath: default.conf
+      volumes:
+        - name: nginx-conf
+          configMap:
+            name: vllm-proxy-config
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vllm-proxy-config
+  namespace: vllm-service
+data:
+  default.conf: |
+    upstream vllm_backend {
+        server vllm.vllm-service.svc.cluster.local:8000;
+    }
+
+    server {
+        listen 8080;
+
+        # API proxy
+        location / {
+            proxy_pass http://vllm_backend;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_connect_timeout 300s;
+            proxy_read_timeout 300s;
+            proxy_send_timeout 300s;
+        }
+
+        # vLLM exposes /metrics natively
+        location /metrics {
+            proxy_pass http://vllm_backend/metrics;
+        }
+    }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: vllm-metrics
+  namespace: vllm-service
+spec:
+  ports:
+    - name: proxy
+      port: 8080
+      targetPort: 8080
+  selector:
+    app: vllm-metrics
+  type: ClusterIP