software-engineering/spark-vllm-docker
2
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed 3-node Qwen 397B recipe to prevent OOM and use instanttensor

Browse Source
This commit is contained in:
eugr
2026-05-10 22:20:49 -07:00
parent ae8ac815ac
commit ba9dde963f
1 changed files with 1 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
recipes/3x-spark-cluster/qwen3.5-397b-int4-autoround.yaml
View File

@@ -18,9 +18,7 @@ container: vllm-node-tf5
build_args: build_args:
- --tf5 - --tf5
# Mod required to fix ROPE syntax error
mods: mods:
- mods/fix-qwen3.5-autoround
- mods/fix-qwen3.5-chat-template - mods/fix-qwen3.5-chat-template
# Default settings (can be overridden via CLI) # Default settings (can be overridden via CLI)
@@ -34,7 +32,6 @@ defaults:
# Environment variables # Environment variables
env: env:
PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True"
VLLM_MARLIN_USE_ATOMIC_ADD: 1 VLLM_MARLIN_USE_ATOMIC_ADD: 1
# The vLLM serve command template # The vLLM serve command template
@@ -55,7 +52,7 @@ command: |
--chat-template unsloth.jinja \ --chat-template unsloth.jinja \
-tp 1 \ -tp 1 \
-pp {pipeline_parallel} \ -pp {pipeline_parallel} \
--load-format fastsafetensors \ --load-format instanttensor \
--distributed-executor-backend ray --distributed-executor-backend ray