feat: vLLM in K8s via Harbor mirror, pipelines point to internal svc

- vLLM image mirrored to registry.celestium.life/stonks-oracle/vllm-openai - Deployment uses Harbor image (Docker Hub IPv6 unreachable from cluster) - All 3 pipelines use vllm-external.vllm-service.svc.cluster.local:2701 - K8s manifests at infra/kube-vllm/ and synced to ~/sources/kube/vllm
2026-07-03 17:02:31 +00:00
parent 3a9894cd03
commit ecade0dd52
9 changed files with 354 additions and 3 deletions
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+VLLM="http://10.1.1.12:31508"
+GPU_HOST="root@10.1.1.12"
+INTERVAL=3
+
+while true; do
+  clear
+  echo "═══════════════════════════════════════════════════════════════"
+  echo "  vLLM MONITOR (K8s)  $(date '+%Y-%m-%d %H:%M:%S')"
+  echo "═══════════════════════════════════════════════════════════════"
+
+  # GPU
+  gpu=$(ssh -o ConnectTimeout=2 -o BatchMode=yes "$GPU_HOST" \
+    'nvidia-smi --query-gpu=name,temperature.gpu,power.draw,power.limit,memory.used,memory.total,utilization.gpu --format=csv,noheader,nounits' 2>/dev/null)
+  if [ -n "$gpu" ]; then
+    IFS=',' read -r name temp power power_cap mem_used mem_total gpu_util <<< "$gpu"
+    mem_free=$(awk "BEGIN{printf \"%.0f\", $mem_total-$mem_used}")
+    echo ""
+    echo "  GPU: ${name}"
+    echo "  ├─ Temp:  ${temp}°C    Power: ${power}W / ${power_cap}W"
+    echo "  ├─ VRAM:  ${mem_used} / ${mem_total} MiB  (${mem_free} MiB free)"
+    echo "  └─ Util:  ${gpu_util}%"
+  fi
+
+  # vLLM model info
+  models_json=$(curl -sf --max-time 2 "$VLLM/v1/models" 2>/dev/null)
+  if [ -n "$models_json" ]; then
+    echo ""
+    python3 -c "
+import json,sys
+data = json.loads(sys.argv[1])
+for m in data.get('data',[]):
+    print(f'  MODEL: {m[\"id\"]}')
+" "$models_json" 2>/dev/null
+  fi
+
+  # Prometheus metrics from vLLM /metrics endpoint
+  prom=$(curl -sf --max-time 2 "$VLLM/metrics" 2>/dev/null)
+  if [ -n "$prom" ]; then
+    python3 -c "
+import sys
+lines = sys.argv[1].split('\n')
+def gauge(prefix):
+    for l in lines:
+        if l.startswith(prefix) and not l.startswith('#'):
+            return float(l.split()[-1])
+    return 0
+def counter(prefix):
+    return sum(float(l.split()[-1]) for l in lines if l.startswith(prefix) and not l.startswith('#'))
+def histo_avg(prefix):
+    s = counter(prefix + '_sum')
+    c = counter(prefix + '_count')
+    return s/c if c > 0 else 0
+
+running = gauge('vllm:num_requests_running')
+waiting = gauge('vllm:num_requests_waiting')
+kv_pct  = gauge('vllm:gpu_cache_usage_perc') * 100
+prompt_tok = counter('vllm:prompt_tokens_total')
+gen_tok    = counter('vllm:generation_tokens_total')
+req_ok     = counter('vllm:request_success_total')
+preempts   = counter('vllm:num_preemptions_total')
+
+ttft   = histo_avg('vllm:time_to_first_token_seconds')
+itl    = histo_avg('vllm:inter_token_latency_seconds')
+e2e    = histo_avg('vllm:e2e_request_latency_seconds')
+tok_s  = 1/itl if itl > 0 else 0
+
+print()
+print('  REQUESTS:')
+print(f'  ├─ Running:    {int(running)}    Waiting: {int(waiting)}')
+print(f'  ├─ Completed:  {int(req_ok)}     Preemptions: {int(preempts)}')
+print(f'  └─ KV Cache:   {kv_pct:.1f}%')
+print()
+print('  TOKENS:')
+print(f'  ├─ Prompt:     {int(prompt_tok):,}')
+print(f'  └─ Generated:  {int(gen_tok):,}')
+print()
+print('  LATENCY:')
+print(f'  ├─ TTFT:       {ttft*1000:.0f}ms')
+print(f'  ├─ ITL:        {itl*1000:.1f}ms')
+print(f'  ├─ Tok/s:      {tok_s:.1f}')
+print(f'  └─ E2E avg:    {e2e:.2f}s')
+" "$prom" 2>/dev/null
+  else
+    echo ""
+    echo "  METRICS: unreachable"
+  fi
+
+  echo ""
+  echo "═══════════════════════════════════════════════════════════════"
+  echo "  Ctrl+C to exit"
+  sleep "$INTERVAL"
+done