try befier model
This commit is contained in:
@@ -43,40 +43,32 @@ spec:
|
|||||||
apiVersion: apps.nvidia.com/v1alpha1
|
apiVersion: apps.nvidia.com/v1alpha1
|
||||||
kind: NIMService
|
kind: NIMService
|
||||||
metadata:
|
metadata:
|
||||||
name: qwen3-32b-instruct
|
name: qwen36-27b-fp8
|
||||||
namespace: nim-service
|
namespace: nim-service
|
||||||
spec:
|
spec:
|
||||||
env:
|
args:
|
||||||
- name: GPU_MEMORY_UTILIZATION
|
- "--gpu-memory-utilization=0.85"
|
||||||
value: "0.90"
|
- "--served-model-name=qwen36"
|
||||||
- name: NIM_MAX_NUM_SEQS
|
- "--max-model-len=256K"
|
||||||
value: "32"
|
- "--language-model-only"
|
||||||
- name: NIM_MAX_NUM_BATCHED_TOKENS
|
- "--reasoning-parser=qwen3"
|
||||||
value: "16384"
|
- "--enable-auto-tool-choice"
|
||||||
- name: ENABLE_AUTO_TOOL_CHOICE
|
- "--tool-call-parser=qwen3_coder"
|
||||||
value: "true"
|
- "--enable-chunked-prefill"
|
||||||
- name: ENABLE_PREFIX_CACHING
|
- "--max-num-batched-tokens=32768"
|
||||||
value: "true"
|
- "--max-num-seqs=10"
|
||||||
- name: TRUNCATION_SIDE
|
- "--enable-prefix-caching"
|
||||||
value: "left"
|
- '--speculative-config={"method":"mtp","num_speculative_tokens":2}'
|
||||||
- name: VLLM_LOGGING_LEVEL
|
|
||||||
value: "INFO"
|
|
||||||
- name: VLLM_ALLOW_LONG_MAX_MODEL_LEN
|
|
||||||
value: "true"
|
|
||||||
- name: NIM_MAX_MODEL_LEN
|
|
||||||
value: "32768"
|
|
||||||
- name: NIM_ENABLE_KV_CACHE_REUSE
|
|
||||||
value: "1"
|
|
||||||
image:
|
image:
|
||||||
repository: nvcr.io/nim/qwen/qwen3-32b-dgx-spark
|
repository: nvcr.io/nim/nvidia/llm-nim
|
||||||
tag: "1.1.0-variant"
|
tag: "1.12"
|
||||||
pullPolicy: IfNotPresent
|
pullPolicy: IfNotPresent
|
||||||
pullSecrets:
|
pullSecrets:
|
||||||
- ngc-secret
|
- ngc-secret
|
||||||
authSecret: ngc-api-secret
|
authSecret: ngc-api-secret
|
||||||
storage:
|
storage:
|
||||||
nimCache:
|
nimCache:
|
||||||
name: qwen3-32b-instruct
|
name: qwen36-27b-fp8
|
||||||
replicas: 1
|
replicas: 1
|
||||||
resources:
|
resources:
|
||||||
limits:
|
limits:
|
||||||
|
|||||||
Reference in New Issue
Block a user