Celes Renata
ecade0dd52
- vLLM image mirrored to registry.celestium.life/stonks-oracle/vllm-openai - Deployment uses Harbor image (Docker Hub IPv6 unreachable from cluster) - All 3 pipelines use vllm-external.vllm-service.svc.cluster.local:2701 - K8s manifests at infra/kube-vllm/ and synced to ~/sources/kube/vllm
76 lines
1.9 KiB
YAML
76 lines
1.9 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: vllm
|
|
namespace: vllm-service
|
|
labels:
|
|
app: vllm
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: vllm
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: vllm
|
|
spec:
|
|
runtimeClassName: nvidia
|
|
nodeSelector:
|
|
kubernetes.io/hostname: gremlin-1
|
|
containers:
|
|
- name: vllm
|
|
image: registry.celestium.life/stonks-oracle/vllm-openai:latest
|
|
imagePullPolicy: Always
|
|
args:
|
|
- "--model"
|
|
- "numind/NuExtract3"
|
|
- "--served-model-name"
|
|
- "numind/NuExtract3"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "8000"
|
|
- "--gpu-memory-utilization"
|
|
- "0.45"
|
|
- "--max-model-len"
|
|
- "8192"
|
|
- "--max-num-seqs"
|
|
- "8"
|
|
env:
|
|
- name: HF_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: vllm-secrets
|
|
key: HF_TOKEN
|
|
- name: VLLM_ATTENTION_BACKEND
|
|
value: "FLASHINFER"
|
|
ports:
|
|
- containerPort: 8000
|
|
name: http
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
requests:
|
|
cpu: "2"
|
|
memory: "8Gi"
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8000
|
|
initialDelaySeconds: 120
|
|
periodSeconds: 10
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8000
|
|
initialDelaySeconds: 300
|
|
periodSeconds: 30
|
|
volumeMounts:
|
|
- name: hf-cache
|
|
mountPath: /root/.cache/huggingface
|
|
volumes:
|
|
- name: hf-cache
|
|
persistentVolumeClaim:
|
|
claimName: vllm-hf-cache-pvc
|