#!/usr/bin/env bash
VLLM="http://10.1.1.12:31508"
GPU_HOST="root@10.1.1.12"
INTERVAL=3

while true; do
  clear
  echo "═══════════════════════════════════════════════════════════════"
  echo "  vLLM MONITOR (K8s)  $(date '+%Y-%m-%d %H:%M:%S')"
  echo "═══════════════════════════════════════════════════════════════"

  # GPU
  gpu=$(ssh -o ConnectTimeout=2 -o BatchMode=yes "$GPU_HOST" \
    'nvidia-smi --query-gpu=name,temperature.gpu,power.draw,power.limit,memory.used,memory.total,utilization.gpu --format=csv,noheader,nounits' 2>/dev/null)
  if [ -n "$gpu" ]; then
    IFS=',' read -r name temp power power_cap mem_used mem_total gpu_util <<< "$gpu"
    mem_free=$(awk "BEGIN{printf \"%.0f\", $mem_total-$mem_used}")
    echo ""
    echo "  GPU: ${name}"
    echo "  ├─ Temp:  ${temp}°C    Power: ${power}W / ${power_cap}W"
    echo "  ├─ VRAM:  ${mem_used} / ${mem_total} MiB  (${mem_free} MiB free)"
    echo "  └─ Util:  ${gpu_util}%"
  fi

  # vLLM model info
  models_json=$(curl -sf --max-time 2 "$VLLM/v1/models" 2>/dev/null)
  if [ -n "$models_json" ]; then
    echo ""
    python3 -c "
import json,sys
data = json.loads(sys.argv[1])
for m in data.get('data',[]):
    print(f'  MODEL: {m[\"id\"]}')
" "$models_json" 2>/dev/null
  fi

  # Prometheus metrics from vLLM /metrics endpoint
  prom=$(curl -sf --max-time 2 "$VLLM/metrics" 2>/dev/null)
  if [ -n "$prom" ]; then
    python3 -c "
import sys
lines = sys.argv[1].split('\n')
def gauge(prefix):
    for l in lines:
        if l.startswith(prefix) and not l.startswith('#'):
            return float(l.split()[-1])
    return 0
def counter(prefix):
    return sum(float(l.split()[-1]) for l in lines if l.startswith(prefix) and not l.startswith('#'))
def histo_avg(prefix):
    s = counter(prefix + '_sum')
    c = counter(prefix + '_count')
    return s/c if c > 0 else 0

running = gauge('vllm:num_requests_running')
waiting = gauge('vllm:num_requests_waiting')
kv_pct  = gauge('vllm:gpu_cache_usage_perc') * 100
prompt_tok = counter('vllm:prompt_tokens_total')
gen_tok    = counter('vllm:generation_tokens_total')
req_ok     = counter('vllm:request_success_total')
preempts   = counter('vllm:num_preemptions_total')

ttft   = histo_avg('vllm:time_to_first_token_seconds')
itl    = histo_avg('vllm:inter_token_latency_seconds')
e2e    = histo_avg('vllm:e2e_request_latency_seconds')
tok_s  = 1/itl if itl > 0 else 0

print()
print('  REQUESTS:')
print(f'  ├─ Running:    {int(running)}    Waiting: {int(waiting)}')
print(f'  ├─ Completed:  {int(req_ok)}     Preemptions: {int(preempts)}')
print(f'  └─ KV Cache:   {kv_pct:.1f}%')
print()
print('  TOKENS:')
print(f'  ├─ Prompt:     {int(prompt_tok):,}')
print(f'  └─ Generated:  {int(gen_tok):,}')
print()
print('  LATENCY:')
print(f'  ├─ TTFT:       {ttft*1000:.0f}ms')
print(f'  ├─ ITL:        {itl*1000:.1f}ms')
print(f'  ├─ Tok/s:      {tok_s:.1f}')
print(f'  └─ E2E avg:    {e2e:.2f}s')
" "$prom" 2>/dev/null
  else
    echo ""
    echo "  METRICS: unreachable"
  fi

  echo ""
  echo "═══════════════════════════════════════════════════════════════"
  echo "  Ctrl+C to exit"
  sleep "$INTERVAL"
done