Celes Renata
a322e00659
- Ollama scaled to 0 — vLLM gets entire 16GB GPU - NuExtract3 (9B) running at 80% GPU utilization - Both OLLAMA_BASE_URL and VLLM_BASE_URL point to vLLM service - Fixed gremlin-1 missing default route (no internet access) - Removed GPU resource limit (time-slicing handles allocation) - Removed init container (direct download works with internet fix)
77 lines
1.9 KiB
YAML
77 lines
1.9 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: vllm
|
|
namespace: vllm-service
|
|
labels:
|
|
app: vllm
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: vllm
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: vllm
|
|
spec:
|
|
runtimeClassName: nvidia
|
|
enableServiceLinks: false
|
|
nodeSelector:
|
|
kubernetes.io/hostname: gremlin-1
|
|
containers:
|
|
- name: vllm
|
|
image: registry.celestium.life/stonks-oracle/vllm-openai:latest
|
|
imagePullPolicy: Always
|
|
args:
|
|
- "--model"
|
|
- "numind/NuExtract3"
|
|
- "--served-model-name"
|
|
- "numind/NuExtract3"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "8000"
|
|
- "--gpu-memory-utilization"
|
|
- "0.80"
|
|
- "--max-model-len"
|
|
- "8192"
|
|
- "--max-num-seqs"
|
|
- "8"
|
|
env:
|
|
- name: HF_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: vllm-secrets
|
|
key: HF_TOKEN
|
|
- name: VLLM_ATTENTION_BACKEND
|
|
value: "FLASHINFER"
|
|
ports:
|
|
- containerPort: 8000
|
|
name: http
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
requests:
|
|
cpu: "2"
|
|
memory: "8Gi"
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8000
|
|
initialDelaySeconds: 120
|
|
periodSeconds: 10
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8000
|
|
initialDelaySeconds: 300
|
|
periodSeconds: 30
|
|
volumeMounts:
|
|
- name: hf-cache
|
|
mountPath: /root/.cache/huggingface
|
|
volumes:
|
|
- name: hf-cache
|
|
persistentVolumeClaim:
|
|
claimName: vllm-hf-cache-pvc
|