Celes Renata
ecade0dd52
- vLLM image mirrored to registry.celestium.life/stonks-oracle/vllm-openai - Deployment uses Harbor image (Docker Hub IPv6 unreachable from cluster) - All 3 pipelines use vllm-external.vllm-service.svc.cluster.local:2701 - K8s manifests at infra/kube-vllm/ and synced to ~/sources/kube/vllm
94 lines
3.5 KiB
Bash
94 lines
3.5 KiB
Bash
#!/usr/bin/env bash
|
|
VLLM="http://10.1.1.12:31508"
|
|
GPU_HOST="root@10.1.1.12"
|
|
INTERVAL=3
|
|
|
|
while true; do
|
|
clear
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
echo " vLLM MONITOR (K8s) $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
|
|
# GPU
|
|
gpu=$(ssh -o ConnectTimeout=2 -o BatchMode=yes "$GPU_HOST" \
|
|
'nvidia-smi --query-gpu=name,temperature.gpu,power.draw,power.limit,memory.used,memory.total,utilization.gpu --format=csv,noheader,nounits' 2>/dev/null)
|
|
if [ -n "$gpu" ]; then
|
|
IFS=',' read -r name temp power power_cap mem_used mem_total gpu_util <<< "$gpu"
|
|
mem_free=$(awk "BEGIN{printf \"%.0f\", $mem_total-$mem_used}")
|
|
echo ""
|
|
echo " GPU: ${name}"
|
|
echo " ├─ Temp: ${temp}°C Power: ${power}W / ${power_cap}W"
|
|
echo " ├─ VRAM: ${mem_used} / ${mem_total} MiB (${mem_free} MiB free)"
|
|
echo " └─ Util: ${gpu_util}%"
|
|
fi
|
|
|
|
# vLLM model info
|
|
models_json=$(curl -sf --max-time 2 "$VLLM/v1/models" 2>/dev/null)
|
|
if [ -n "$models_json" ]; then
|
|
echo ""
|
|
python3 -c "
|
|
import json,sys
|
|
data = json.loads(sys.argv[1])
|
|
for m in data.get('data',[]):
|
|
print(f' MODEL: {m[\"id\"]}')
|
|
" "$models_json" 2>/dev/null
|
|
fi
|
|
|
|
# Prometheus metrics from vLLM /metrics endpoint
|
|
prom=$(curl -sf --max-time 2 "$VLLM/metrics" 2>/dev/null)
|
|
if [ -n "$prom" ]; then
|
|
python3 -c "
|
|
import sys
|
|
lines = sys.argv[1].split('\n')
|
|
def gauge(prefix):
|
|
for l in lines:
|
|
if l.startswith(prefix) and not l.startswith('#'):
|
|
return float(l.split()[-1])
|
|
return 0
|
|
def counter(prefix):
|
|
return sum(float(l.split()[-1]) for l in lines if l.startswith(prefix) and not l.startswith('#'))
|
|
def histo_avg(prefix):
|
|
s = counter(prefix + '_sum')
|
|
c = counter(prefix + '_count')
|
|
return s/c if c > 0 else 0
|
|
|
|
running = gauge('vllm:num_requests_running')
|
|
waiting = gauge('vllm:num_requests_waiting')
|
|
kv_pct = gauge('vllm:gpu_cache_usage_perc') * 100
|
|
prompt_tok = counter('vllm:prompt_tokens_total')
|
|
gen_tok = counter('vllm:generation_tokens_total')
|
|
req_ok = counter('vllm:request_success_total')
|
|
preempts = counter('vllm:num_preemptions_total')
|
|
|
|
ttft = histo_avg('vllm:time_to_first_token_seconds')
|
|
itl = histo_avg('vllm:inter_token_latency_seconds')
|
|
e2e = histo_avg('vllm:e2e_request_latency_seconds')
|
|
tok_s = 1/itl if itl > 0 else 0
|
|
|
|
print()
|
|
print(' REQUESTS:')
|
|
print(f' ├─ Running: {int(running)} Waiting: {int(waiting)}')
|
|
print(f' ├─ Completed: {int(req_ok)} Preemptions: {int(preempts)}')
|
|
print(f' └─ KV Cache: {kv_pct:.1f}%')
|
|
print()
|
|
print(' TOKENS:')
|
|
print(f' ├─ Prompt: {int(prompt_tok):,}')
|
|
print(f' └─ Generated: {int(gen_tok):,}')
|
|
print()
|
|
print(' LATENCY:')
|
|
print(f' ├─ TTFT: {ttft*1000:.0f}ms')
|
|
print(f' ├─ ITL: {itl*1000:.1f}ms')
|
|
print(f' ├─ Tok/s: {tok_s:.1f}')
|
|
print(f' └─ E2E avg: {e2e:.2f}s')
|
|
" "$prom" 2>/dev/null
|
|
else
|
|
echo ""
|
|
echo " METRICS: unreachable"
|
|
fi
|
|
|
|
echo ""
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
echo " Ctrl+C to exit"
|
|
sleep "$INTERVAL"
|
|
done
|