feat: vLLM in K8s via Harbor mirror, pipelines point to internal svc
- vLLM image mirrored to registry.celestium.life/stonks-oracle/vllm-openai - Deployment uses Harbor image (Docker Hub IPv6 unreachable from cluster) - All 3 pipelines use vllm-external.vllm-service.svc.cluster.local:2701 - K8s manifests at infra/kube-vllm/ and synced to ~/sources/kube/vllm
This commit is contained in:
Celes Renata
@@ -0,0 +1,75 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: vllm
|
||||
namespace: vllm-service
|
||||
labels:
|
||||
app: vllm
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: vllm
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: vllm
|
||||
spec:
|
||||
runtimeClassName: nvidia
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: gremlin-1
|
||||
containers:
|
||||
- name: vllm
|
||||
image: registry.celestium.life/stonks-oracle/vllm-openai:latest
|
||||
imagePullPolicy: Always
|
||||
args:
|
||||
- "--model"
|
||||
- "numind/NuExtract3"
|
||||
- "--served-model-name"
|
||||
- "numind/NuExtract3"
|
||||
- "--host"
|
||||
- "0.0.0.0"
|
||||
- "--port"
|
||||
- "8000"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.45"
|
||||
- "--max-model-len"
|
||||
- "8192"
|
||||
- "--max-num-seqs"
|
||||
- "8"
|
||||
env:
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: vllm-secrets
|
||||
key: HF_TOKEN
|
||||
- name: VLLM_ATTENTION_BACKEND
|
||||
value: "FLASHINFER"
|
||||
ports:
|
||||
- containerPort: 8000
|
||||
name: http
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: "1"
|
||||
requests:
|
||||
cpu: "2"
|
||||
memory: "8Gi"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 120
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 300
|
||||
periodSeconds: 30
|
||||
volumeMounts:
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
volumes:
|
||||
- name: hf-cache
|
||||
persistentVolumeClaim:
|
||||
claimName: vllm-hf-cache-pvc
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env bash
|
||||
VLLM="http://10.1.1.12:31508"
|
||||
GPU_HOST="root@10.1.1.12"
|
||||
INTERVAL=3
|
||||
|
||||
while true; do
|
||||
clear
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
echo " vLLM MONITOR (K8s) $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
|
||||
# GPU
|
||||
gpu=$(ssh -o ConnectTimeout=2 -o BatchMode=yes "$GPU_HOST" \
|
||||
'nvidia-smi --query-gpu=name,temperature.gpu,power.draw,power.limit,memory.used,memory.total,utilization.gpu --format=csv,noheader,nounits' 2>/dev/null)
|
||||
if [ -n "$gpu" ]; then
|
||||
IFS=',' read -r name temp power power_cap mem_used mem_total gpu_util <<< "$gpu"
|
||||
mem_free=$(awk "BEGIN{printf \"%.0f\", $mem_total-$mem_used}")
|
||||
echo ""
|
||||
echo " GPU: ${name}"
|
||||
echo " ├─ Temp: ${temp}°C Power: ${power}W / ${power_cap}W"
|
||||
echo " ├─ VRAM: ${mem_used} / ${mem_total} MiB (${mem_free} MiB free)"
|
||||
echo " └─ Util: ${gpu_util}%"
|
||||
fi
|
||||
|
||||
# vLLM model info
|
||||
models_json=$(curl -sf --max-time 2 "$VLLM/v1/models" 2>/dev/null)
|
||||
if [ -n "$models_json" ]; then
|
||||
echo ""
|
||||
python3 -c "
|
||||
import json,sys
|
||||
data = json.loads(sys.argv[1])
|
||||
for m in data.get('data',[]):
|
||||
print(f' MODEL: {m[\"id\"]}')
|
||||
" "$models_json" 2>/dev/null
|
||||
fi
|
||||
|
||||
# Prometheus metrics from vLLM /metrics endpoint
|
||||
prom=$(curl -sf --max-time 2 "$VLLM/metrics" 2>/dev/null)
|
||||
if [ -n "$prom" ]; then
|
||||
python3 -c "
|
||||
import sys
|
||||
lines = sys.argv[1].split('\n')
|
||||
def gauge(prefix):
|
||||
for l in lines:
|
||||
if l.startswith(prefix) and not l.startswith('#'):
|
||||
return float(l.split()[-1])
|
||||
return 0
|
||||
def counter(prefix):
|
||||
return sum(float(l.split()[-1]) for l in lines if l.startswith(prefix) and not l.startswith('#'))
|
||||
def histo_avg(prefix):
|
||||
s = counter(prefix + '_sum')
|
||||
c = counter(prefix + '_count')
|
||||
return s/c if c > 0 else 0
|
||||
|
||||
running = gauge('vllm:num_requests_running')
|
||||
waiting = gauge('vllm:num_requests_waiting')
|
||||
kv_pct = gauge('vllm:gpu_cache_usage_perc') * 100
|
||||
prompt_tok = counter('vllm:prompt_tokens_total')
|
||||
gen_tok = counter('vllm:generation_tokens_total')
|
||||
req_ok = counter('vllm:request_success_total')
|
||||
preempts = counter('vllm:num_preemptions_total')
|
||||
|
||||
ttft = histo_avg('vllm:time_to_first_token_seconds')
|
||||
itl = histo_avg('vllm:inter_token_latency_seconds')
|
||||
e2e = histo_avg('vllm:e2e_request_latency_seconds')
|
||||
tok_s = 1/itl if itl > 0 else 0
|
||||
|
||||
print()
|
||||
print(' REQUESTS:')
|
||||
print(f' ├─ Running: {int(running)} Waiting: {int(waiting)}')
|
||||
print(f' ├─ Completed: {int(req_ok)} Preemptions: {int(preempts)}')
|
||||
print(f' └─ KV Cache: {kv_pct:.1f}%')
|
||||
print()
|
||||
print(' TOKENS:')
|
||||
print(f' ├─ Prompt: {int(prompt_tok):,}')
|
||||
print(f' └─ Generated: {int(gen_tok):,}')
|
||||
print()
|
||||
print(' LATENCY:')
|
||||
print(f' ├─ TTFT: {ttft*1000:.0f}ms')
|
||||
print(f' ├─ ITL: {itl*1000:.1f}ms')
|
||||
print(f' ├─ Tok/s: {tok_s:.1f}')
|
||||
print(f' └─ E2E avg: {e2e:.2f}s')
|
||||
" "$prom" 2>/dev/null
|
||||
else
|
||||
echo ""
|
||||
echo " METRICS: unreachable"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
echo " Ctrl+C to exit"
|
||||
sleep "$INTERVAL"
|
||||
done
|
||||
@@ -0,0 +1,29 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: vllm
|
||||
namespace: vllm-service
|
||||
spec:
|
||||
ports:
|
||||
- name: http
|
||||
port: 8000
|
||||
targetPort: 8000
|
||||
selector:
|
||||
app: vllm
|
||||
type: ClusterIP
|
||||
---
|
||||
# External access via NodePort (like ollama's port 2701 pattern)
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: vllm-external
|
||||
namespace: vllm-service
|
||||
spec:
|
||||
ports:
|
||||
- nodePort: 31508
|
||||
name: vllm-api
|
||||
port: 2701
|
||||
targetPort: 8000
|
||||
selector:
|
||||
app: vllm-metrics
|
||||
type: LoadBalancer
|
||||
@@ -0,0 +1,35 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: vllm-hf-cache-pv
|
||||
spec:
|
||||
capacity:
|
||||
storage: 50Gi
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: local-path
|
||||
hostPath:
|
||||
path: /var/lib/vllm/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
nodeAffinity:
|
||||
required:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: In
|
||||
values:
|
||||
- gremlin-1
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: vllm-hf-cache-pvc
|
||||
namespace: vllm-service
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
storageClassName: local-path
|
||||
@@ -0,0 +1,45 @@
|
||||
replicaCount: 1
|
||||
|
||||
image:
|
||||
repository: vllm/vllm-openai
|
||||
tag: latest
|
||||
pullPolicy: Always
|
||||
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
requests:
|
||||
cpu: "2"
|
||||
memory: "8Gi"
|
||||
|
||||
runtimeClassName: nvidia
|
||||
|
||||
env:
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: vllm-secrets
|
||||
key: HF_TOKEN
|
||||
|
||||
args:
|
||||
- "serve"
|
||||
- "numind/NuExtract3"
|
||||
- "--served-model-name"
|
||||
- "numind/NuExtract3"
|
||||
- "--host"
|
||||
- "0.0.0.0"
|
||||
- "--port"
|
||||
- "8000"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.45"
|
||||
- "--max-model-len"
|
||||
- "8192"
|
||||
- "--max-num-seqs"
|
||||
- "8"
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 8000
|
||||
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: gremlin-1
|
||||
@@ -0,0 +1,74 @@
|
||||
# vLLM metrics proxy — similar to ollama-metrics
|
||||
# Proxies requests and exposes Prometheus metrics
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: vllm-metrics
|
||||
namespace: vllm-service
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: vllm-metrics
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: vllm-metrics
|
||||
spec:
|
||||
containers:
|
||||
- name: proxy
|
||||
image: nginx:alpine
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
volumeMounts:
|
||||
- name: nginx-conf
|
||||
mountPath: /etc/nginx/conf.d/default.conf
|
||||
subPath: default.conf
|
||||
volumes:
|
||||
- name: nginx-conf
|
||||
configMap:
|
||||
name: vllm-proxy-config
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: vllm-proxy-config
|
||||
namespace: vllm-service
|
||||
data:
|
||||
default.conf: |
|
||||
upstream vllm_backend {
|
||||
server vllm.vllm-service.svc.cluster.local:8000;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
|
||||
# API proxy
|
||||
location / {
|
||||
proxy_pass http://vllm_backend;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_connect_timeout 300s;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_send_timeout 300s;
|
||||
}
|
||||
|
||||
# vLLM exposes /metrics natively
|
||||
location /metrics {
|
||||
proxy_pass http://vllm_backend/metrics;
|
||||
}
|
||||
}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: vllm-metrics
|
||||
namespace: vllm-service
|
||||
spec:
|
||||
ports:
|
||||
- name: proxy
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
selector:
|
||||
app: vllm-metrics
|
||||
type: ClusterIP
|
||||
Reference in New Issue
Block a user