Celes Renata
af7b654fb7
- Strategy: Recreate (GPU can only be used by one pod at a time) - nuextract-external targetPort: 8080 (nginx proxy port) - served-model-name: numind/NuExtract3 - DB ai_agents updated to model=numind/NuExtract3 provider=vllm
79 lines
1.9 KiB
YAML
79 lines
1.9 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: vllm
|
|
namespace: vllm-service
|
|
labels:
|
|
app: vllm
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app: vllm
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: vllm
|
|
spec:
|
|
runtimeClassName: nvidia
|
|
enableServiceLinks: false
|
|
nodeSelector:
|
|
kubernetes.io/hostname: gremlin-1
|
|
containers:
|
|
- name: vllm
|
|
image: registry.celestium.life/stonks-oracle/vllm-openai:latest
|
|
imagePullPolicy: Always
|
|
args:
|
|
- "--model"
|
|
- "numind/NuExtract3"
|
|
- "--served-model-name"
|
|
- "numind/NuExtract3"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "8000"
|
|
- "--gpu-memory-utilization"
|
|
- "0.80"
|
|
- "--max-model-len"
|
|
- "8192"
|
|
- "--max-num-seqs"
|
|
- "8"
|
|
env:
|
|
- name: HF_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: vllm-secrets
|
|
key: HF_TOKEN
|
|
- name: VLLM_ATTENTION_BACKEND
|
|
value: "FLASHINFER"
|
|
ports:
|
|
- containerPort: 8000
|
|
name: http
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
requests:
|
|
cpu: "2"
|
|
memory: "8Gi"
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8000
|
|
initialDelaySeconds: 120
|
|
periodSeconds: 10
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8000
|
|
initialDelaySeconds: 300
|
|
periodSeconds: 30
|
|
volumeMounts:
|
|
- name: hf-cache
|
|
mountPath: /root/.cache/huggingface
|
|
volumes:
|
|
- name: hf-cache
|
|
persistentVolumeClaim:
|
|
claimName: vllm-hf-cache-pvc
|