software-engineering/spark-vllm-docker
2
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Updated Nemotron recipes to use VLLM CUTLASS

Browse Source
This commit is contained in:
Eugene Rakhmatulin
2026-03-22 23:03:24 -07:00
parent 2d749742e4
commit 9e089acf2b
2 changed files with 5 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
recipes/nemotron-3-nano-nvfp4.yaml
View File

@@ -27,15 +27,10 @@ defaults:
gpu_memory_utilization: 0.7 gpu_memory_utilization: 0.7
max_model_len: 262144 max_model_len: 262144
# Environment variables
env:
VLLM_NVFP4_GEMM_BACKEND: "marlin"
VLLM_TEST_FORCE_FP8_MARLIN: "1"
VLLM_MARLIN_USE_ATOMIC_ADD: "1"
# The vLLM serve command template # The vLLM serve command template
command: | command: |
vllm serve nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 \ vllm serve nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 \
--moe-backend cutlass \
--max-model-len {max_model_len} \ --max-model-len {max_model_len} \
--port {port} --host {host} \ --port {port} --host {host} \
--trust-remote-code \ --trust-remote-code \

12
recipes/nemotron-3-super-nvfp4.yaml
View File

@@ -1,8 +1,8 @@
# Recipe: Nemotron-3-Super-NVFP4 # Recipe: Nemotron-3-Super-NVFP4
# Optimized for Marlin backend throughput # Uses VLLM_CUTLASS for NVFP4
recipe_version: "1" recipe_version: "1"
name: Nemotron-3-Super-NVFP4-Marlin-Optimized name: Nemotron-3-Super-NVFP4-CUTLASS-Optimized
description: vLLM serving Nemotron-3-Super-120B using Marlin kernels description: vLLM serving Nemotron-3-Super-120B using CUTLASS kernels
model: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 model: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4
container: vllm-node container: vllm-node
@@ -20,15 +20,11 @@ defaults:
gpu_memory_utilization: 0.7 gpu_memory_utilization: 0.7
max_model_len: 262144 max_model_len: 262144
max_num_seqs: 10 max_num_seqs: 10
env:
VLLM_NVFP4_GEMM_BACKEND: "marlin"
VLLM_TEST_FORCE_FP8_MARLIN: "1"
VLLM_MARLIN_USE_ATOMIC_ADD: "1"
command: | command: |
vllm serve nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 \ vllm serve nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 \
--kv-cache-dtype fp8 \ --kv-cache-dtype fp8 \
-tp {tensor_parallel} \ --moe-backend cutlass \
--trust-remote-code \ --trust-remote-code \
--gpu-memory-utilization {gpu_memory_utilization} \ --gpu-memory-utilization {gpu_memory_utilization} \
--max-model-len {max_model_len} \ --max-model-len {max_model_len} \