Merge branch '3-node-autodiscover'

2026-03-31 18:22:23 -07:00
parent e7f2ee692f c4860b86a2
commit 23fb7dcc20
11 changed files with 1785 additions and 544 deletions
--- a/.env.example
+++ b/.env.example
@@ -0,0 +1,38 @@
+# Example .env configuration file for spark-vllm-docker
+# Copy this file to .env and customize for your environment
+
+# Cluster configuration
+# CLUSTER_NODES: Comma-separated list of node IPs (first node is the head node)
+CLUSTER_NODES="192.168.177.11,192.168.177.12"
+
+# ETH_IF: Ethernet interface name (optional, auto-detected if not specified)
+ETH_IF="enp1s0f1np1"
+
+# IB_IF: InfiniBand interface name (optional, auto-detected if not specified)
+IB_IF="rocep1s0f1,roceP2p1s0f1"
+
+# LOCAL_IP: Local IP address (optional, auto-detected if not specified)
+# Useful for solo mode or overriding auto-detection
+LOCAL_IP="192.168.177.11"
+
+# MASTER_PORT: Port for cluster coordination (default: 29501)
+MASTER_PORT="29501"
+
+# CONTAINER_NAME: Container name (default: vllm_node)
+# Note: This is a configuration variable, NOT passed as env var to container
+CONTAINER_NAME="vllm_node"
+
+# Container environment variables
+# Any variable starting with CONTAINER_ (except CONTAINER_NAME) will be converted to -e flags
+# Example: CONTAINER_NCCL_DEBUG=INFO becomes -e NCCL_DEBUG=INFO
+CONTAINER_NCCL_DEBUG="INFO"
+CONTAINER_HF_TOKEN="your_huggingface_token_here"
+CONTAINER_NCCL_IGNORE_CPU_AFFINITY="1"
+
+# COPY_HOSTS: Comma-separated list of hosts for build-and-copy.sh (optional)
+# Used by build-and-copy.sh to distribute images across cluster
+COPY_HOSTS="192.168.177.12"
+
+# Additional container environment variables
+# CONTAINER_MAX_JOBS="16"
+# CONTAINER_CUDA_VISIBLE_DEVICES="0,1"
--- a/75
+++ b/75
@@ -14,6 +14,8 @@ ENV MAX_JOBS=${BUILD_JOBS}
 ENV CMAKE_BUILD_PARALLEL_LEVEL=${BUILD_JOBS}
 ENV NINJAFLAGS="-j${BUILD_JOBS}"
 ENV MAKEFLAGS="-j${BUILD_JOBS}"
+ENV DG_JIT_USE_NVRTC=1
+ENV USE_CUDNN=1

 # Set non-interactive frontend to prevent apt prompts
 ENV DEBIAN_FRONTEND=noninteractive
@@ -38,8 +40,8 @@ RUN apt update && \
    curl vim cmake build-essential ninja-build \
    libcudnn9-cuda-13 libcudnn9-dev-cuda-13 \
    python3-dev python3-pip git wget \
-    libnccl-dev libnccl2 libibverbs1 libibverbs-dev rdma-core \
-    ccache \
+    libibverbs1 libibverbs-dev rdma-core \
+    ccache devscripts debhelper fakeroot \
    && rm -rf /var/lib/apt/lists/* \
    && pip install uv

@@ -59,14 +61,19 @@ ENV CCACHE_COMPRESS=1
 ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
 ENV CMAKE_CUDA_COMPILER_LAUNCHER=ccache

-# Setup Workspace
-WORKDIR $VLLM_BASE_DIR
-
 # 2. Set Environment Variables
 ARG TORCH_CUDA_ARCH_LIST="12.1a"
 ENV TORCH_CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST}
 ENV TRITON_PTXAS_PATH=/usr/local/cuda/bin/ptxas

+# Setup Workspace
+WORKDIR $VLLM_BASE_DIR
+
+# Build NCCL with mesh support (TODO: only do it if arch is 12.1) - artifacts will be in /workspace/nccl/build/pkg/deb
+RUN git clone -b dgxspark-3node-ring https://github.com/zyang-dev/nccl.git && \
+    cd nccl && make -j ${BUILD_JOBS} src.build NVCC_GENCODE="-gencode=arch=compute_121,code=sm_121" && \
+    make pkg.debian.build && apt install -y --no-install-recommends --allow-downgrades ./build/pkg/deb/*.deb
+
 # =========================================================
 # STAGE 2: FlashInfer Builder
 # =========================================================
@@ -105,6 +112,26 @@ RUN --mount=type=cache,id=repo-cache,target=/repo-cache \

 WORKDIR /workspace/flashinfer

+ARG FLASHINFER_PRS=""
+
+RUN if [ -n "$FLASHINFER_PRS" ]; then \
+        echo "Applying PRs: $FLASHINFER_PRS"; \
+        for pr in $FLASHINFER_PRS; do \
+            echo "Fetching and applying PR #$pr..."; \
+            curl -fL "https://github.com/flashinfer-ai/flashinfer/pull/${pr}.diff" | git apply -v; \
+        done; \
+    fi
+
+# TEMPORARY patch for NVFP4 crash (PR 2913)
+RUN curl -fsL https://github.com/flashinfer-ai/flashinfer/pull/2913.diff -o pr2913.diff \
+    && if git apply --reverse --check pr2913.diff 2>/dev/null; then \
+         echo "PR #2913 already applied, skipping."; \
+       else \
+         echo "Applying FI PR #2913..."; \
+         git apply -v pr2913.diff; \
+       fi \
+    && rm pr2913.diff
+
 # Apply patch to avoid re-downloading existing cubins
 COPY flashinfer_cache.patch .
 RUN --mount=type=cache,id=uv-cache,target=/root/.cache/uv \
@@ -177,6 +204,16 @@ RUN if [ -n "$VLLM_PRS" ]; then \
        done; \
    fi

+# TEMPORARY PATCH for broken compilation
+# RUN curl -fsL https://patch-diff.githubusercontent.com/raw/vllm-project/vllm/pull/38423.diff -o pr38423.diff \
+#     && if git apply --reverse --check pr38423.diff 2>/dev/null; then \
+#          echo "Patch already applied, skipping."; \
+#        else \
+#          echo "Applying patch..."; \
+#          git apply -v pr38423.diff; \
+#        fi \
+#     && rm pr38423.diff
+
 # Prepare build requirements
 RUN --mount=type=cache,id=uv-cache,target=/root/.cache/uv \
    python3 use_existing_torch.py && \
@@ -194,17 +231,8 @@ RUN --mount=type=cache,id=uv-cache,target=/root/.cache/uv \
 #         patch -p1 < fastsafetensors.patch; \
 #     fi
 # TEMPORARY PATCH for broken vLLM build (unguarded Hopper code) - reverting PR #34758 and #34302
-RUN curl -L https://patch-diff.githubusercontent.com/raw/vllm-project/vllm/pull/34758.diff | patch -p1 -R || echo "Cannot revert PR #34758, skipping"
-RUN curl -L https://patch-diff.githubusercontent.com/raw/vllm-project/vllm/pull/34302.diff | patch -p1 -R || echo "Cannot revert PR #34302, skipping"
-# TEMPORARY PATCH for broken NVFP4 quants
-RUN curl -fsL https://patch-diff.githubusercontent.com/raw/vllm-project/vllm/pull/38126.diff -o pr38126.diff \
-    && if git apply --reverse --check pr38126.diff 2>/dev/null; then \
-         echo "Patch already applied, skipping."; \
-       else \
-         echo "Applying patch..."; \
-         git apply -v pr38126.diff; \
-       fi \
-    && rm pr38126.diff
+# RUN curl -L https://patch-diff.githubusercontent.com/raw/vllm-project/vllm/pull/34758.diff | patch -p1 -R || echo "Cannot revert PR #34758, skipping"
+# RUN curl -L https://patch-diff.githubusercontent.com/raw/vllm-project/vllm/pull/34302.diff | patch -p1 -R || echo "Cannot revert PR #34302, skipping"

 # Final Compilation
 RUN --mount=type=cache,id=ccache,target=/root/.ccache \
@@ -231,6 +259,8 @@ ENV MAX_JOBS=${BUILD_JOBS}
 ENV CMAKE_BUILD_PARALLEL_LEVEL=${BUILD_JOBS}
 ENV NINJAFLAGS="-j${BUILD_JOBS}"
 ENV MAKEFLAGS="-j${BUILD_JOBS}"
+ENV DG_JIT_USE_NVRTC=1
+ENV USE_CUDNN=1

 ENV DEBIAN_FRONTEND=noninteractive
 ENV PIP_BREAK_SYSTEM_PACKAGES=1
@@ -243,13 +273,16 @@ ENV UV_SYSTEM_PYTHON=1
 ENV UV_BREAK_SYSTEM_PACKAGES=1
 ENV UV_LINK_MODE=copy

+# Mount additional packages from base builder image
 # Install runtime dependencies
-RUN apt update && \
+RUN --mount=type=bind,from=base,source=/workspace/vllm/nccl/build/pkg/deb,target=/workspace/nccl-pkg \
+    apt update && \
    apt install -y --no-install-recommends \
    python3 python3-pip python3-dev vim curl git wget \
    libcudnn9-cuda-13 \
-    libnccl-dev libnccl2 libibverbs1 libibverbs-dev rdma-core \
+    libibverbs1 libibverbs-dev rdma-core \
    libxcb1 \
+    && cd /workspace/nccl-pkg && apt install -y --no-install-recommends --allow-downgrades ./*.deb \
    && rm -rf /var/lib/apt/lists/* \
    && pip install uv

@@ -293,5 +326,9 @@ ENV PATH=$VLLM_BASE_DIR:$PATH
 RUN --mount=type=cache,id=uv-cache,target=/root/.cache/uv \
    uv pip install ray[default] fastsafetensors

+# Fix NCCL
+RUN rm /usr/local/lib/python3.12/dist-packages/nvidia/nccl/lib/libnccl.so.2 && \
+    ln -s /usr/lib/aarch64-linux-gnu/libnccl.so.2 /usr/local/lib/python3.12/dist-packages/nvidia/nccl/lib/libnccl.so.2
+    
 # Build metadata (generated by build-and-copy.sh)
-COPY build-metadata.yaml /workspace/build-metadata.yaml
+COPY build-metadata.yaml /workspace/build-metadata.yaml
--- a/README.md
+++ b/README.md
@@ -2,6 +2,7 @@
 # vLLM Docker Optimized for DGX Spark (single or multi-node)

 This repository contains the Docker configuration and startup scripts to run a multi-node vLLM inference cluster using Ray. It supports InfiniBand/RDMA (NCCL) and custom environment configuration for high-performance setups.
+Cluster setup supports direct connect between dual Sparks, connecting via QSFP/RoCE switch and 3-node mesh configuration.

 While it was primarily developed to support multi-node inference, it works just as well on a single node setups.

@@ -31,6 +32,8 @@ We will expand the selection of models we test in the pipeline, but since vLLM i

 If you want to build the latest from main branch, you can specify `--rebuild-vllm` flag. Or you can target a specific vLLM release by setting `--vllm-ref` parameter.

+Similarly, `--rebuild-flashinfer`, `--flashinfer-ref`, and `--apply-flashinfer-pr` control the FlashInfer build in the same way.
+
 ## QUICK START

 ### Build
@@ -52,8 +55,8 @@ Build the container.

 **On DGX Spark cluster:**

-Make sure you connect your Sparks together and enable passwordless SSH as described in NVidia's [Connect Two Sparks Playbook](https://build.nvidia.com/spark/connect-two-sparks/stacked-sparks). 
-You can also check out our new [Networking Guide](docs/NETWORKING.md).
+Make sure you connect your Sparks together and enable passwordless SSH as described in our [Networking Guide](docs/NETWORKING.md). You can also check out NVidia's [Connect Two Sparks Playbook](https://build.nvidia.com/spark/connect-two-sparks/stacked-sparks), but using our guide is the best way to get started.
+**NEW**: the guide now includes instructions on setting up 3-node Spark mesh!

 Then run the following command that will build and distribute image across the cluster.

@@ -67,7 +70,7 @@ An initial build speed depends on your Internet connection speed and whether the

 **On a single node**:

-**NEW** - `launch-cluster.sh` now supports solo mode, which is now a recommended way to run the container on a single Spark:
+`launch-cluster.sh` supports solo mode, which is now a recommended way to run the container on a single Spark:

 ```bash
 ./launch-cluster.sh --solo exec \
@@ -78,23 +81,6 @@ An initial build speed depends on your Internet connection speed and whether the
    --load-format fastsafetensors
 ```

-**To launch using regular `docker run`**
-
-```bash
- docker run \
-  --privileged \
-  --gpus all \
-  -it --rm \
-  --network host --ipc=host \
-  -v  ~/.cache/huggingface:/root/.cache/huggingface \
-  vllm-node \
-  bash -c -i "vllm serve \
-  QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ \
-  --port 8000 --host 0.0.0.0 \
-  --gpu-memory-utilization 0.7 \
-  --load-format fastsafetensors"
-```
-
 **On a cluster**

 It's recommended to download the model on one node and distribute across the cluster using ConnectX interconnect prior to launching. This is to avoid re-downloading the model from the Internet on every node in the cluster.
@@ -127,8 +113,6 @@ This will run the model on all available cluster nodes.

 **Also:** You can use any vLLM container that has "bash" as its default entrypoint with the launch script. It was tested with NGC vLLM, but can work with others too. To use such container in the cluster, you need to specify `--apply-mod use-ngc-vllm` argument to `./launch-cluster.sh`. However, it's recommended to build the container using this repository for best compatibility and most up-to-date features. 

-## CHANGELOG
-
 **IMPORTANT**

 You may want to prune your build cache every once in a while, especially if you've been using these container builds since the beginning.
@@ -149,6 +133,107 @@ Don't do it every time you rebuild, because it will slow down compilation times.

 For periodic maintenance, I recommend using a filter: `docker builder prune --filter until=72h`

+## CHANGELOG
+
+### 2026-03-31
+
+#### Flags to specify Flashinfer ref and apply PRs
+
+`build-and-copy.sh` gains two new flags that mirror the existing vLLM equivalents:
+
+- `--flashinfer-ref <ref>` — build FlashInfer from a specific commit SHA, branch, or tag instead of `main`. Forces a local FlashInfer build (skips prebuilt wheel download).
+- `--apply-flashinfer-pr <pr-num>` — fetch and apply a FlashInfer GitHub PR patch before building. Can be specified multiple times. Forces a local FlashInfer build.
+
+Both flags are incompatible with `--exp-mxfp4`.
+
+#### Default image tag in `build-and-copy.sh`
+
+`build-and-copy.sh` now automatically sets a sensible default image tag when `-t` is not specified:
+
+- `--tf5` / `--pre-tf` - tag defaults to `vllm-node-tf5`
+- `--exp-mxfp4` - tag defaults to `vllm-node-mxfp4`
+- in all other cases - tag defaults to `vllm-node` (no change)
+
+An explicit `-t <tag>` always takes precedence.
+
+#### Support for 3-node mesh setups
+
+Added initial support for setups where 3 Sparks are connected in a ring-like mesh without an additional switch.
+See [Networking Guide](docs/NETWORKING.md) for instructions on how to connect and set up networking in such cluster.
+
+Autodiscover function in both `launch-cluster.sh` and `run-recipe.sh` now can detect mesh setups and configure parameters accordingly.
+
+You can try running a model on all 3 nodes in pipeline-parallel configuration using the following recipe:
+
+```bash
+./run-recipe.sh --discover # force mesh discovery
+./run-recipe.sh recipes/3x-spark-cluster/qwen3.5-397b-int4-autoround.yaml --setup --no-ray --force-build # you can drop --setup and --force-build on subsequent calls
+```
+
+Please note that `--tensor-parallel-size 3` or `-tp 3` is not supported by any commonly used model, so the only two viable options to utilize all three nodes for a single model are:
+
+- `--pipeline-parallel 3` will let you run a model that can't fit on dual Sparks, but without additional speed improvements (total throughtput may improve though).
+- `--data-parallel 3` (possibly with `--enable-expert-parallel`) will let you run a model that can fit on a single Spark, but allow for better concurrency.
+
+You can also run models with `--tensor-parallel 2` in a 3-node configuration - in this case only first two nodes (from autodiscovery/.env or from the CLI parameters) will be utilized.
+
+#### GB10 Verification During Node Discovery
+
+Node discovery now confirms each SSH-reachable peer is a GB10 system before adding it to the cluster:
+Only hosts reporting `NVIDIA GB10` are included. This prevents accidentally adding non-Spark machines that happen to be on the same subnet.
+
+#### Separate COPY_HOSTS Discovery
+
+Autodiscover now determines the host list used for image and model distribution separately from `CLUSTER_NODES`:
+
+- **Non-mesh**: `COPY_HOSTS` mirrors `CLUSTER_NODES` (no change in behaviour).
+- **Mesh**: scans the direct IB-attached `enp1s0f0np0` and `enp1s0f1np1` interfaces (not the OOB ETH interface), so large file transfers use the faster direct InfiniBand path.
+
+`COPY_HOSTS` is saved to `.env` and respected by `build-and-copy.sh`, `hf-download.sh`, and `run-recipe.py`.
+
+#### Interactive Configuration Save in `autodiscover.sh`
+
+`autodiscover.sh` now handles `.env` creation with a guided interactive flow, replacing the previous logic in `run-recipe.py`:
+
+- Runs automatically when `.env` is absent.
+- Asks per-node confirmation for both `CLUSTER_NODES` and `COPY_HOSTS`.
+- Skips if `.env` already exists (use `--setup` to force).
+
+`run-recipe.py` no longer contains its own `.env`-save prompt — it delegates entirely to `autodiscover.sh`.
+
+#### `--setup` Flag in `launch-cluster.sh` and `build-and-copy.sh`
+
+Both scripts now accept `--setup` to force a full autodiscovery run and overwrite the existing `.env`:
+
+```bash
+./launch-cluster.sh --setup exec vllm serve ...
+./build-and-copy.sh --setup -c
+```
+
+This is equivalent to the existing `--setup` in `run-recipe.sh`.
+
+#### `--config` Flag
+
+`hf-download.sh`, `build-and-copy.sh` and `launch-cluster.sh` now accept `--config <file>` to load a custom `.env` configuration file. `COPY_HOSTS` from the config is used for model distribution:
+
+```bash
+./hf-download.sh QuantTrio/MiniMax-M2-AWQ --config /path/to/cluster.env -c --copy-parallel
+```
+
+#### Parallelism-Aware Node Trimming
+
+`launch-cluster.sh` now parses `-tp` / `--tensor-parallel-size`, `-pp` / `--pipeline-parallel-size`, and `-dp` / `--data-parallel-size` from the exec command or launch script and adjusts the active node count accordingly — for both Ray and no-Ray modes.
+
+- If **fewer nodes are needed** than configured, only the required nodes get containers started (excess nodes are left idle).
+- If **more nodes are needed** than available, an error is raised before anything starts.
+
+```
+Note: Command requires 2 node(s) (tp=2 * pp=1 * dp=1); using 2 of 3 configured node(s).
+Error: Command requires 4 nodes (tp=4 * pp=1 * dp=1) but only 3 node(s) are configured.
+```
+
+No flags required — the check is automatic whenever parallelism arguments are present in the command.
+
 ### 2026-03-18

 #### `--master-port` / `--head-port` Parameter
@@ -591,7 +676,8 @@ See (this post on NVIDIA forums)[https://forums.developer.nvidia.com/t/make-glm-
 To use the mod, first build the container with Transformers 5 support (`--pre-tf`) flag, e.g.:

 ```bash
-./build-and-copy.sh -t vllm-node-tf5 --pre-tf -c
+# Image tag defaults to vllm-node-tf5 when --tf5/--pre-tf is used
+./build-and-copy.sh --pre-tf -c
 ```

 Then, to run on a single node:
@@ -641,7 +727,8 @@ It is currently the fastest way to run GPT-OSS on DGX Spark, achieving 60 t/s on
 To use this build, first build the container with `--exp-mxfp4` flag. I recommend using a separate label as it is currently not recommended to use this build for models other than gpt-oss:

 ```bash
-./build-and-copy.sh -t vllm-node-mxfp4 --exp-mxfp4 -c
+# Image tag defaults to vllm-node-mxfp4 when --exp-mxfp4 is used
+./build-and-copy.sh --exp-mxfp4 -c
 ```

 Then, to run on a single Spark:
@@ -885,12 +972,14 @@ Using a different username:

 | Flag | Description |
 | :--- | :--- |
-| `-t, --tag <tag>` | Image tag (default: `vllm-node`) |
+| `-t, --tag <tag>` | Image tag (default: `vllm-node`; auto-set to `vllm-node-tf5` with `--tf5`, `vllm-node-mxfp4` with `--exp-mxfp4`) |
 | `--gpu-arch <arch>` | Target GPU architecture (default: `12.1a`) |
 | `--rebuild-flashinfer` | Skip prebuilt wheel download; force a fresh local FlashInfer build |
 | `--rebuild-vllm` | Force rebuild vLLM from source |
 | `--vllm-ref <ref>` | vLLM commit SHA, branch or tag (default: `main`) |
+| `--flashinfer-ref <ref>` | FlashInfer commit SHA, branch or tag (default: `main`) |
 | `--apply-vllm-pr <pr-num>` | Apply a vLLM PR patch during build. Can be specified multiple times. |
+| `--apply-flashinfer-pr <pr-num>` | Apply a FlashInfer PR patch during build. Can be specified multiple times. |
 | `--tf5` | Install transformers v5 (5.0.0 or higher). Aliases: `--pre-tf, --pre-transformers`. |
 | `--exp-mxfp4` | Build with experimental native MXFP4 support. Alias: `--experimental-mxfp4`. |
 | `-c, --copy-to <hosts>` | Host(s) to copy the image to after building (space- or comma-separated). |
@@ -900,9 +989,13 @@ Using a different username:
 | `-u, --user <user>` | Username for SSH connection (default: current user) |
 | `--full-log` | Enable full Docker build output (`--progress=plain`) |
 | `--no-build` | Skip building, only copy existing image (requires `--copy-to`) |
+| `--network <name>` | Docker network to use during build (e.g. `host`). |
+| `--cleanup` | Remove all cached `.whl` and `*-commit` files from the `wheels/` directory. |
+| `--config <file>` | Path to `.env` configuration file (default: `.env` in script directory) |
+| `--setup` | Force autodiscovery and save configuration to `.env` (even if `.env` already exists) |
 | `-h, --help` | Show help message |

-**IMPORTANT**: When copying to another node, make sure you use the Spark IP assigned to its ConnectX 7 interface (enp1s0f1np1), and not the 10G interface (enP7s7)! If you omit the IP address and use `-c` without addresses, it will use autodiscovery to detect a proper IP address.
+**IMPORTANT**: When copying to another node manually, use the IP assigned to a ConnectX 7 interface (`enp1s0f*`), not the 10G/wireless interfaces. When using `-c` without addresses, autodiscovery selects the correct interface automatically — in mesh mode it uses the direct IB-attached interfaces (`enp1s0f0np0`, `enp1s0f1np1`) for maximum transfer speed.

 ### Copying the container to another Spark node (Manual Method)

@@ -971,9 +1064,12 @@ Assumptions and limitations:
 ### Auto-Detection

 The script attempts to automatically detect:
-*   **Ethernet Interface:** The interface associated with the active InfiniBand device that has an IP address.
-*   **InfiniBand Interface:** The active InfiniBand devices. By default both active RoCE interfaces that correspond to active IB port(s) will be utilized.
-*   **Node Role:** Based on the detected IP address and the list of nodes (defaults to `192.168.177.11` as head and `192.168.177.12` as worker).
+*   **Ethernet Interface (`ETH_IF`):** Determined by the number of active CX7 interfaces:
+    - **2 active** (standard): the `enp*` interface (no capital P) that has an IP address.
+    - **4 active** (mesh topology): `enP7s7` (preferred) or `wlP9s9` (wireless, shown with a warning) — the cluster coordination interface is separate from the CX7 ports in this configuration.
+*   **InfiniBand Interface (`IB_IF`):** All active RoCE devices. In mesh mode this is always `rocep1s0f0,roceP2p1s0f0,rocep1s0f1,roceP2p1s0f1`.
+*   **Cluster peers:** Discovered by scanning the `ETH_IF` subnet for hosts with SSH access **and** a GB10 GPU (`nvidia-smi --query-gpu=name` must return `NVIDIA GB10`).
+*   **Copy hosts (`COPY_HOSTS`):** In standard mode, same as cluster peers. In mesh mode, scanned separately on `enp1s0f0np0` and `enp1s0f1np1` subnets so that image/model transfers use the direct InfiniBand path.

 ### Manual Overrides

@@ -1006,6 +1102,10 @@ You can override the auto-detected values if needed:
 | `--mem-swap-limit-gb` | Memory+swap limit in GB (default: mem-limit + 10, only with `--non-privileged`). |
 | `--pids-limit` | Process limit (default: 4096, only with `--non-privileged`). |
 | `--shm-size-gb` | Shared memory size in GB (default: 64, only with `--non-privileged`). |
+| `--config <file>` | Path to `.env` configuration file (default: `.env` in script directory). |
+| `--setup` | Force autodiscovery and save configuration to `.env` (even if `.env` already exists). |
+| `start \| stop \| status \| exec` | Action to perform (default: `start`). Not compatible with `--launch-script`. |
+| `command` | Command to execute inside the container (only for `exec` action). |

 ### Non-Privileged Mode

@@ -1149,6 +1249,61 @@ You need to make sure you allocate IP addresses to them (no need to allocate IP

 ## 5\. Configuration Details

+### Cluster Configuration (`.env` file)
+
+The scripts share a `.env` file (default: `.env` in the repo directory) for persistent cluster configuration. It is created automatically by autodiscovery — run `--discover` (via `run-recipe.sh`) or `--setup` (via `launch-cluster.sh` / `build-and-copy.sh`) on first use.
+
+**Supported variables:**
+
+| Variable | Description |
+| :--- | :--- |
+| `CLUSTER_NODES` | Comma-separated node IPs used for Ray/vLLM cluster (head node first). |
+| `COPY_HOSTS` | Comma-separated node IPs used for image and model distribution. In mesh mode these are the IPs on the direct IB-attached interfaces, which may differ from `CLUSTER_NODES`. |
+| `LOCAL_IP` | IP address of the local node. |
+| `ETH_IF` | Ethernet interface for cluster coordination (e.g. `enp1s0f1np1` or `enP7s7`). |
+| `IB_IF` | Comma-separated RoCE/IB device names (e.g. `rocep1s0f0,roceP2p1s0f0,rocep1s0f1,roceP2p1s0f1`). |
+| `CONTAINER_*` | Any variable prefixed with `CONTAINER_` (except `CONTAINER_NAME`) is passed as `-e VAR=VALUE` to the container. Example: `CONTAINER_NCCL_DEBUG=INFO` → `-e NCCL_DEBUG=INFO`. |
+
+**Mesh-mode NCCL variables** (written automatically when mesh topology is detected):
+
+```
+CONTAINER_NCCL_NET_PLUGIN=none
+CONTAINER_NCCL_IB_SUBNET_AWARE_ROUTING=1
+CONTAINER_NCCL_IB_MERGE_NICS=0
+```
+
+**Example `.env` for a standard 2-node cluster:**
+
+```
+CLUSTER_NODES=192.168.177.11,192.168.177.12
+COPY_HOSTS=192.168.177.12
+LOCAL_IP=192.168.177.11
+ETH_IF=enp1s0f1np1
+IB_IF=rocep1s0f1,roceP2p1s0f1
+```
+
+To use a custom config file path, pass `--config /path/to/file.env` to any script.
+
+### Autodiscovery Workflow
+
+On first run, if no `.env` is present, the scripts will automatically trigger autodiscovery. You can also run it explicitly:
+
+```bash
+# Via run-recipe.sh
+./run-recipe.sh --discover
+
+# Via launch-cluster.sh or build-and-copy.sh (force re-run even if .env exists)
+./launch-cluster.sh --setup exec vllm serve ...
+./build-and-copy.sh --setup -c
+```
+
+Autodiscovery:
+1. Detects active CX7 interfaces and determines mesh vs. standard topology.
+2. Scans the network for SSH-reachable GB10 peers.
+3. In mesh mode, separately discovers `COPY_HOSTS` on direct IB-attached interfaces.
+4. Prompts for per-node confirmation for both `CLUSTER_NODES` and `COPY_HOSTS`.
+5. Saves the result to `.env`.
+
 ### Environment Persistence

 The script automatically appends exported variables to `~/.bashrc`. If you need to open a second terminal into the running container for debugging, simply run:
@@ -1322,6 +1477,32 @@ The `hf-download.sh` script provides a convenient way to download models from Hu
 ./hf-download.sh -c --copy-parallel QuantTrio/MiniMax-M2-AWQ
 ```

+**Use nodes from `.env` (respects `COPY_HOSTS`):**
+
+```bash
+./hf-download.sh -c QuantTrio/MiniMax-M2-AWQ
+```
+
+When `-c` is given without explicit hosts, the script checks `COPY_HOSTS` in `.env` first, then falls back to autodiscovery. In mesh mode this means transfers go over the direct IB-attached interfaces automatically.
+
+**Use a custom config file:**
+
+```bash
+./hf-download.sh --config /path/to/cluster.env -c QuantTrio/MiniMax-M2-AWQ
+```
+
+**Available options:**
+
+| Flag | Description |
+| :--- | :--- |
+| `<model-name>` | HuggingFace model ID (e.g. `QuantTrio/MiniMax-M2-AWQ`). Required. |
+| `-c, --copy-to <hosts>` | Host(s) to copy the model to after download (space- or comma-separated). Omit hosts to use `COPY_HOSTS` from `.env` or autodiscovery. |
+| `--copy-to-host` | Alias for `--copy-to` (backwards compatibility). |
+| `--copy-parallel` | Copy to all hosts concurrently instead of serially. |
+| `-u, --user <user>` | SSH username for remote copies (default: current user). |
+| `--config <file>` | Path to `.env` configuration file (default: `.env` in script directory). |
+| `-h, --help` | Show help message. |
+
 ### Hardware Architecture

 **Note:** This project targets `12.1a` architecture (NVIDIA GB10 / DGX Spark). If you are using different hardware, you can use `--gpu-arch` flag in `./build-and-copy.sh`.
--- a/autodiscover.sh
+++ b/autodiscover.sh
@@ -1,5 +1,57 @@
 #!/bin/bash

+SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+
+# Load .env file if exists (for shared configuration)
+# This is called early so that DOTENV_* variables are available to all functions
+load_env_if_exists() {
+    local env_file="${CONFIG_FILE:-}"
+    local config_explicit="${CONFIG_FILE_SET:-false}"
+
+    # If CONFIG_FILE is not set, check default location
+    if [[ -z "$env_file" ]]; then
+        env_file="$SCRIPT_DIR/.env"
+        config_explicit="false"
+    fi
+
+    # Validate config file exists if explicitly specified
+    # Exception: if --setup is also specified, the file will be created by the setup procedure
+    if [[ "$config_explicit" == "true" ]] && [[ ! -f "$env_file" ]] && [[ "${FORCE_DISCOVER:-false}" != "true" ]]; then
+        echo "Error: Config file not found: $env_file"
+        exit 1
+    fi
+
+    if [[ -f "$env_file" ]]; then
+        # Load .env variables with DOTENV_ prefix
+        while IFS='=' read -r key value || [[ -n "$key" ]]; do
+            # Skip comments and empty lines
+            [[ "$key" =~ ^[[:space:]]*# ]] && continue
+            [[ -z "$key" ]] && continue
+
+            # Remove leading/trailing whitespace from key
+            key=$(echo "$key" | xargs)
+
+            # Skip if key is empty after trimming
+            [[ -z "$key" ]] && continue
+
+            # Remove quotes from value
+            value="${value%\"}"
+            value="${value#\"}"
+            value="${value%\'}"
+            value="${value#\'}"
+
+            # Export with DOTENV_ prefix
+            export "DOTENV_$key=$value"
+        done < "$env_file"
+    fi
+}
+
+# Load .env file
+load_env_if_exists
+
+# Mesh mode flag (set by detect_interfaces)
+MESH_MODE="false"
+
 # Function to detect IB and Ethernet interfaces
 detect_interfaces() {
    # If both interfaces are already set, nothing to do
@@ -14,60 +66,132 @@ detect_interfaces() {
    fi

    echo "Auto-detecting interfaces..."
-    
+
    # Get all Up interfaces: "rocep1s0f1 port 1 ==> enp1s0f1np1 (Up)"
    # We capture: IB_DEV, NET_DEV
    mapfile -t IB_NET_PAIRS < <(ibdev2netdev | awk '/Up\)/ {print $1 " " $5}')
-    
+
    if [ ${#IB_NET_PAIRS[@]} -eq 0 ]; then
        echo "Error: No active IB interfaces found."
        return 1
    fi

    DETECTED_IB_IFS=()
-    CANDIDATE_ETH_IFS=()
+    ALL_NET_IFS=()

    for pair in "${IB_NET_PAIRS[@]}"; do
        ib_dev=$(echo "$pair" | awk '{print $1}')
        net_dev=$(echo "$pair" | awk '{print $2}')
-        
        DETECTED_IB_IFS+=("$ib_dev")
-        
-        # Check if interface has an IP address
-        if ip addr show "$net_dev" | grep -q "inet "; then
-            CANDIDATE_ETH_IFS+=("$net_dev")
+        ALL_NET_IFS+=("$net_dev")
+    done
+
+    local num_up="${#IB_NET_PAIRS[@]}"
+
+    # --- Sanity checks ---
+
+    # 1. enp* (no capital P) interfaces MUST have an IP
+    for net_dev in "${ALL_NET_IFS[@]}"; do
+        if [[ "$net_dev" =~ ^enp[^P] ]] || [[ "$net_dev" == enp* && "$net_dev" != *P* ]]; then
+            if ! ip addr show "$net_dev" 2>/dev/null | grep -q "inet "; then
+                echo "Error: Interface $net_dev (enp*, no capital P) is Up but has no IP address assigned."
+                return 1
+            fi
        fi
    done

-    # Set IB_IF if not provided
-    if [[ -z "$IB_IF" ]]; then
-        IB_IF=$(IFS=,; echo "${DETECTED_IB_IFS[*]}")
-        echo "  Detected IB_IF: $IB_IF"
-    fi
-
-    # Set ETH_IF if not provided
-    if [[ -z "$ETH_IF" ]]; then
-        if [ ${#CANDIDATE_ETH_IFS[@]} -eq 0 ]; then
-            echo "Error: No active IB-associated interfaces have IP addresses."
+    # 2. No two interfaces with IPs should share the same subnet
+    declare -A SEEN_SUBNETS
+    for net_dev in "${ALL_NET_IFS[@]}"; do
+        local cidr
+        cidr=$(ip -o -f inet addr show "$net_dev" 2>/dev/null | awk '{print $4}' | head -n1)
+        [[ -z "$cidr" ]] && continue
+        # Compute network address using python3
+        local net_addr
+        net_addr=$(python3 -c "import ipaddress; print(str(ipaddress.ip_network('$cidr', strict=False)))" 2>/dev/null)
+        if [[ -n "${SEEN_SUBNETS[$net_addr]}" ]]; then
+            echo "Error: Interfaces $net_dev and ${SEEN_SUBNETS[$net_addr]} share the same subnet ($net_addr). Check network configuration."
            return 1
        fi
-        
-        # Selection logic: Prefer interface without capital 'P'
-        SELECTED_ETH=""
-        for iface in "${CANDIDATE_ETH_IFS[@]}"; do
-            if [[ "$iface" != *"P"* ]]; then
-                SELECTED_ETH="$iface"
-                break
-            fi
-        done
-        
-        # Fallback: Use the first one if all have 'P' or none found yet
-        if [[ -z "$SELECTED_ETH" ]]; then
-            SELECTED_ETH="${CANDIDATE_ETH_IFS[0]}"
+        SEEN_SUBNETS["$net_addr"]="$net_dev"
+    done
+
+    # --- Mode selection ---
+
+    if [[ "$num_up" -eq 2 ]]; then
+        # Non-mesh configuration
+        MESH_MODE="false"
+        echo "  Non-mesh mode: 2 CX7 interfaces active."
+
+        # Set IB_IF if not provided
+        if [[ -z "$IB_IF" ]]; then
+            IB_IF=$(IFS=,; echo "${DETECTED_IB_IFS[*]}")
+            echo "  Detected IB_IF: $IB_IF"
        fi
-        
-        ETH_IF="$SELECTED_ETH"
-        echo "  Detected ETH_IF: $ETH_IF"
+
+        # Set ETH_IF if not provided: prefer interface without capital 'P'
+        if [[ -z "$ETH_IF" ]]; then
+            local selected_eth=""
+            for net_dev in "${ALL_NET_IFS[@]}"; do
+                if ip addr show "$net_dev" 2>/dev/null | grep -q "inet "; then
+                    if [[ "$net_dev" != *P* ]]; then
+                        selected_eth="$net_dev"
+                        break
+                    fi
+                fi
+            done
+            # Fallback: first interface with an IP
+            if [[ -z "$selected_eth" ]]; then
+                for net_dev in "${ALL_NET_IFS[@]}"; do
+                    if ip addr show "$net_dev" 2>/dev/null | grep -q "inet "; then
+                        selected_eth="$net_dev"
+                        break
+                    fi
+                done
+            fi
+            if [[ -z "$selected_eth" ]]; then
+                echo "Error: No active IB-associated interfaces have IP addresses."
+                return 1
+            fi
+            ETH_IF="$selected_eth"
+            echo "  Detected ETH_IF: $ETH_IF"
+        fi
+
+    elif [[ "$num_up" -eq 4 ]]; then
+        # Mesh configuration
+        MESH_MODE="true"
+        echo "  Mesh mode: all 4 CX7 interfaces active."
+
+        # Set IB_IF to all four RoCE interfaces (hardcoded for mesh)
+        if [[ -z "$IB_IF" ]]; then
+            IB_IF="rocep1s0f0,roceP2p1s0f0,rocep1s0f1,roceP2p1s0f1"
+            echo "  Detected IB_IF: $IB_IF"
+        fi
+
+        # Set ETH_IF: check enP7s7 first, then wlP9s9
+        if [[ -z "$ETH_IF" ]]; then
+            if ip addr show enP7s7 2>/dev/null | grep -q "inet "; then
+                ETH_IF="enP7s7"
+                echo "  Detected ETH_IF: $ETH_IF"
+            elif ip addr show wlP9s9 2>/dev/null | grep -q "inet "; then
+                ETH_IF="wlP9s9"
+                echo "  Detected ETH_IF: $ETH_IF"
+                echo "  Warning: using wireless interface (wlP9s9) for cluster coordination. Performance may be limited."
+            else
+                echo "Error: Mesh mode requires enP7s7 or wlP9s9 to be up with an IP address for cluster coordination."
+                return 1
+            fi
+        fi
+
+        # Export mesh NCCL settings directly so launch-cluster.sh picks them up
+        # even if the user declines to save config to .env
+        export DOTENV_CONTAINER_NCCL_NET_PLUGIN=none
+        export DOTENV_CONTAINER_NCCL_IB_SUBNET_AWARE_ROUTING=1
+        export DOTENV_CONTAINER_NCCL_IB_MERGE_NICS=0
+
+    else
+        echo "Error: Unexpected number of active CX7 interfaces ($num_up). Expected 2 (non-mesh) or 4 (mesh)."
+        return 1
    fi
 }

@@ -84,16 +208,51 @@ detect_local_ip() {

    # Get CIDR of the selected ETH_IF
    CIDR=$(ip -o -f inet addr show "$ETH_IF" | awk '{print $4}' | head -n 1)
-    
+
    if [[ -z "$CIDR" ]]; then
        echo "Error: Could not determine IP/CIDR for interface $ETH_IF"
        return 1
    fi
-    
+
    LOCAL_IP=${CIDR%/*}
    echo "  Detected Local IP: $LOCAL_IP ($CIDR)"
 }

+# Scan a subnet for GB10-capable peers via SSH
+# Usage: _scan_subnet_for_gb10 <cidr> <local_ip_to_exclude> <output_file>
+_scan_subnet_for_gb10() {
+    local cidr="$1"
+    local exclude_ip="$2"
+    local out_file="$3"
+
+    if ! command -v python3 &> /dev/null; then
+        echo "Error: python3 not found."
+        return 1
+    fi
+    if ! command -v nc &> /dev/null; then
+        echo "Error: nc (netcat) not found."
+        return 1
+    fi
+
+    local all_ips
+    all_ips=$(python3 -c "import ipaddress, sys; [print(ip) for ip in ipaddress.ip_network(sys.argv[1], strict=False).hosts()]" "$cidr")
+
+    for ip in $all_ips; do
+        [[ "$ip" == "$exclude_ip" ]] && continue
+        (
+            if nc -z -w 1 "$ip" 22 &>/dev/null; then
+                # Check if remote is a GB10 system
+                if ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no -o BatchMode=yes "$ip" \
+                    "nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null" \
+                    2>/dev/null | grep -q "NVIDIA GB10"; then
+                    echo "$ip" >> "$out_file"
+                fi
+            fi
+        ) &
+    done
+    wait
+}
+
 # Function to detect cluster nodes
 detect_nodes() {
    detect_local_ip || return 1
@@ -111,58 +270,182 @@ detect_nodes() {
        return 0
    fi

-    echo "Auto-detecting nodes..."
-    
-    if ! command -v nc &> /dev/null; then
-        echo "Error: nc (netcat) not found. Please install netcat."
-        return 1
-    fi
-    
-    if ! command -v python3 &> /dev/null; then
-        echo "Error: python3 not found. Please install python3."
-        return 1
+    # Try to use CLUSTER_NODES from .env
+    if [[ -n "$DOTENV_CLUSTER_NODES" ]]; then
+        echo "  Using CLUSTER_NODES from .env: $DOTENV_CLUSTER_NODES"
+        PEER_NODES=()
+        IFS=',' read -ra ALL_NODES <<< "$DOTENV_CLUSTER_NODES"
+        for node in "${ALL_NODES[@]}"; do
+            node=$(echo "$node" | xargs)
+            [[ "$node" != "$LOCAL_IP" ]] && PEER_NODES+=("$node")
+        done
+        NODES_ARG="$DOTENV_CLUSTER_NODES"
+        return 0
    fi

-    DETECTED_IPS=("$LOCAL_IP")
+    echo "Auto-detecting nodes on $CIDR (checking for NVIDIA GB10)..."
+
+    local temp_file
+    temp_file=$(mktemp)
+
+    _scan_subnet_for_gb10 "$CIDR" "$LOCAL_IP" "$temp_file"
+
    PEER_NODES=()
-    
-    echo "  Scanning for SSH peers on $CIDR..."
-    
-    # Generate list of IPs using python
-    ALL_IPS=$(python3 -c "import ipaddress, sys; [print(ip) for ip in ipaddress.ip_network(sys.argv[1], strict=False).hosts()]" "$CIDR")
-    
-    TEMP_IPS_FILE=$(mktemp)
-    
-    # Scan in parallel
-    for ip in $ALL_IPS; do
-        # Skip own IP
-        if [[ "$ip" == "$LOCAL_IP" ]]; then continue; fi
-        
-        (
-            # Check port 22 with 1 second timeout
-            if nc -z -w 1 "$ip" 22 &>/dev/null; then
-                echo "$ip" >> "$TEMP_IPS_FILE"
-            fi
-        ) &
-    done
-    
-    # Wait for all background scans to complete
-    wait
-    
-    # Read found IPs
-    if [[ -f "$TEMP_IPS_FILE" ]]; then
+    local detected_ips=("$LOCAL_IP")
+    if [[ -f "$temp_file" ]]; then
        while read -r ip; do
-             DETECTED_IPS+=("$ip")
-             PEER_NODES+=("$ip")
-             echo "  Found peer: $ip"
-        done < "$TEMP_IPS_FILE"
-        rm -f "$TEMP_IPS_FILE"
+            PEER_NODES+=("$ip")
+            detected_ips+=("$ip")
+            echo "  Found GB10 peer: $ip"
+        done < <(sort "$temp_file")
+        rm -f "$temp_file"
    fi
-    
-    # Sort IPs
-    IFS=$'\n' SORTED_IPS=($(sort <<<"${DETECTED_IPS[*]}"))
+
+    # Sort and set NODES_ARG
+    IFS=$'\n' SORTED_IPS=($(sort <<<"${detected_ips[*]}"))
    unset IFS
-    
    NODES_ARG=$(IFS=,; echo "${SORTED_IPS[*]}")
    echo "  Cluster Nodes: $NODES_ARG"
 }
+
+# Function to detect COPY_HOSTS for build/model distribution
+# In non-mesh mode: COPY_PEER_NODES = PEER_NODES (same network)
+# In mesh mode: scan enp* interfaces (direct IB-attached) for GB10 peers
+detect_copy_hosts() {
+    if [[ "$MESH_MODE" == "false" ]]; then
+        COPY_PEER_NODES=("${PEER_NODES[@]}")
+        return 0
+    fi
+
+    # Mesh mode: scan enp1s0f0np0 and enp1s0f1np1 subnets
+    echo "Auto-detecting COPY_HOSTS on direct IB interfaces (mesh mode)..."
+
+    local temp_file
+    temp_file=$(mktemp)
+
+    for iface in enp1s0f0np0 enp1s0f1np1; do
+        local cidr
+        cidr=$(ip -o -f inet addr show "$iface" 2>/dev/null | awk '{print $4}' | head -n1)
+        [[ -z "$cidr" ]] && continue
+        local local_iface_ip="${cidr%/*}"
+        echo "  Scanning $iface ($cidr)..."
+        _scan_subnet_for_gb10 "$cidr" "$local_iface_ip" "$temp_file"
+    done
+
+    # Deduplicate and collect results.
+    # On two-cable setups two IB IPs may belong to the same host; deduplicate by
+    # querying each host's ETH_IF IP as a canonical identity.
+    COPY_PEER_NODES=()
+    declare -A _SEEN_COPY   # keyed by IB IP
+    declare -A _SEEN_HOST   # keyed by ETH_IF IP → first IB IP seen for that host
+    if [[ -f "$temp_file" ]]; then
+        while read -r ip; do
+            [[ -n "${_SEEN_COPY[$ip]}" ]] && continue
+            _SEEN_COPY["$ip"]=1
+            # Resolve canonical host identity via ETH_IF IP
+            local host_ip
+            host_ip=$(ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no -o BatchMode=yes "$ip" \
+                "ip -o -f inet addr show $ETH_IF 2>/dev/null | awk '{print \$4}' | head -n1 | cut -d/ -f1" \
+                </dev/null 2>/dev/null)
+            if [[ -n "$host_ip" && -n "${_SEEN_HOST[$host_ip]}" ]]; then
+                echo "  Skipping $ip (same host as ${_SEEN_HOST[$host_ip]}, ETH_IF: $host_ip)"
+                continue
+            fi
+            [[ -n "$host_ip" ]] && _SEEN_HOST["$host_ip"]="$ip"
+            COPY_PEER_NODES+=("$ip")
+            echo "  Found GB10 copy host: $ip"
+        done < <(sort "$temp_file")
+        rm -f "$temp_file"
+    fi
+}
+
+# Save discovered configuration to .env
+# Skips if .env already exists unless FORCE_DISCOVER=true
+save_config() {
+    local env_file="${CONFIG_FILE:-$SCRIPT_DIR/.env}"
+
+    # Skip if .env exists and not forced
+    if [[ -f "$env_file" && "${FORCE_DISCOVER:-false}" != "true" ]]; then
+        return 0
+    fi
+
+    echo ""
+    local save_prompt="Save discovered configuration to $env_file?"
+    if [[ -f "$env_file" ]]; then
+        save_prompt="Overwrite existing configuration in $env_file?"
+    fi
+    read -r -p "$save_prompt [Y/n]: " response
+    response="${response,,}"
+    if [[ "$response" =~ ^(n|no)$ ]]; then
+        return 0
+    fi
+
+    # Build list of all cluster nodes (local + peers)
+    local all_cluster_nodes=()
+    if [[ -n "$LOCAL_IP" ]]; then
+        all_cluster_nodes+=("$LOCAL_IP")
+    fi
+    for node in "${PEER_NODES[@]}"; do
+        all_cluster_nodes+=("$node")
+    done
+
+    # Per-node confirmation for CLUSTER_NODES
+    echo ""
+    echo "Select nodes for CLUSTER_NODES:"
+    local selected_cluster=()
+    for node in "${all_cluster_nodes[@]}"; do
+        local label="$node"
+        [[ "$node" == "$LOCAL_IP" ]] && label="$node (this machine)"
+        read -r -p "  Include $label? [Y/n]: " r
+        r="${r,,}"
+        if [[ ! "$r" =~ ^(n|no)$ ]]; then
+            selected_cluster+=("$node")
+        fi
+    done
+
+    if [[ "${#selected_cluster[@]}" -eq 0 ]]; then
+        echo "No nodes selected. Aborting save."
+        return 1
+    fi
+
+    # Per-node confirmation for COPY_HOSTS
+    echo ""
+    echo "Select nodes for COPY_HOSTS (build/model distribution):"
+    local selected_copy=()
+    for node in "${COPY_PEER_NODES[@]}"; do
+        read -r -p "  Include $node in COPY_HOSTS? [Y/n]: " r
+        r="${r,,}"
+        if [[ ! "$r" =~ ^(n|no)$ ]]; then
+            selected_copy+=("$node")
+        fi
+    done
+
+    # Write .env
+    {
+        echo "# Auto-generated by autodiscover.sh"
+        echo "CLUSTER_NODES=$(IFS=,; echo "${selected_cluster[*]}")"
+        if [[ "${#selected_copy[@]}" -gt 0 ]]; then
+            echo "COPY_HOSTS=$(IFS=,; echo "${selected_copy[*]}")"
+        fi
+        echo "LOCAL_IP=$LOCAL_IP"
+        echo "ETH_IF=$ETH_IF"
+        echo "IB_IF=$IB_IF"
+        if [[ "$MESH_MODE" == "true" ]]; then
+            echo "# Mesh mode NCCL settings"
+            echo "CONTAINER_NCCL_NET_PLUGIN=none"
+            echo "CONTAINER_NCCL_IB_SUBNET_AWARE_ROUTING=1"
+            echo "CONTAINER_NCCL_IB_MERGE_NICS=0"
+        fi
+    } > "$env_file"
+    echo ""
+    echo "Saved to $env_file"
+}
+
+# Convenience function: run full autodiscovery pipeline
+run_autodiscover() {
+    detect_interfaces || return 1
+    detect_local_ip || return 1
+    detect_nodes || return 1
+    detect_copy_hosts || return 1
+    save_config
+}
--- a/build-and-copy.sh
+++ b/build-and-copy.sh
@@ -6,17 +6,22 @@ START_TIME=$(date +%s)

 # Default values
 IMAGE_TAG="vllm-node"
+IMAGE_TAG_SET=false
 REBUILD_FLASHINFER=false
 REBUILD_VLLM=false
 COPY_HOSTS=()
+COPY_TO_FLAG=false
 SSH_USER="$USER"
 NO_BUILD=false
 VLLM_REF="main"
+VLLM_REF_SET=false
+FLASHINFER_REF="main"
+FLASHINFER_REF_SET=false
 TMP_IMAGE=""
 PARALLEL_COPY=false
 EXP_MXFP4=false
-VLLM_REF_SET=false
 VLLM_PRS=""
+FLASHINFER_PRS=""
 PRE_TRANSFORMERS=false
 FULL_LOG=false
 BUILD_JOBS="16"
@@ -27,6 +32,8 @@ FLASHINFER_RELEASE_TAG="prebuilt-flashinfer-current"
 VLLM_RELEASE_TAG="prebuilt-vllm-current"
 # Space-separated list of GPU architectures for which prebuilt wheels are available
 PREBUILT_WHEELS_SUPPORTED_ARCHS="12.1a"
+CLEANUP_MODE="false"
+CONFIG_FILE=""

 cleanup() {
    if [ -n "$TMP_IMAGE" ] && [ -f "$TMP_IMAGE" ]; then
@@ -262,11 +269,12 @@ if downloads:
 # Help function
 usage() {
    echo "Usage: $0 [OPTIONS]"
-    echo "  -t, --tag <tag>               : Image tag (default: 'vllm-node')"
+    echo "  -t, --tag <tag>               : Image tag (default: 'vllm-node', 'vllm-node-tf5' with --tf5, 'vllm-node-mxfp4' with --exp-mxfp4)"
    echo "  --gpu-arch <arch>             : GPU architecture (default: '12.1a')"
    echo "  --rebuild-flashinfer          : Force rebuild of FlashInfer wheels (ignore cached wheels)"
    echo "  --rebuild-vllm                : Force rebuild of vLLM wheels (ignore cached wheels)"
    echo "  --vllm-ref <ref>              : vLLM commit SHA, branch or tag (default: 'main')"
+    echo "  --flashinfer-ref <ref>        : FlashInfer commit SHA, branch or tag (default: 'main')"
    echo "  -c, --copy-to <hosts>         : Host(s) to copy the image to. Accepts comma or space-delimited lists."
    echo "      --copy-to-host            : Alias for --copy-to (backwards compatibility)."
    echo "      --copy-parallel           : Copy to all hosts in parallel instead of serially."
@@ -275,48 +283,34 @@ usage() {
    echo "  --tf5                         : Install transformers>=5 (aliases: --pre-tf, --pre-transformers)"
    echo "  --exp-mxfp4, --experimental-mxfp4 : Build with experimental native MXFP4 support"
    echo "  --apply-vllm-pr <pr-num>      : Apply a specific PR patch to vLLM source. Can be specified multiple times."
+    echo "  --apply-flashinfer-pr <pr-num>: Apply a specific PR patch to FlashInfer source. Can be specified multiple times."
    echo "  --full-log                    : Enable full build logging (--progress=plain)"
    echo "  --no-build                    : Skip building, only copy image (requires --copy-to)"
    echo "  --network <network>           : Docker network to use during build"
+    echo "  --cleanup                     : Remove all *.whl and *.-commit files in wheels directory"
+    echo "  --config                      : Path to .env configuration file (default: .env in script directory)"
+    echo "  --setup                       : Force autodiscovery and save configuration (even if .env exists)"
    echo "  -h, --help                    : Show this help message"
    exit 1
 }

-# Argument parsing
+# Parse all arguments
+CONFIG_FILE_SET=false
 while [[ "$#" -gt 0 ]]; do
    case $1 in
-        -t|--tag) IMAGE_TAG="$2"; shift ;;
+        -t|--tag) IMAGE_TAG="$2"; IMAGE_TAG_SET=true; shift ;;
        --gpu-arch) GPU_ARCH_LIST="$2"; shift ;;
        --rebuild-flashinfer) REBUILD_FLASHINFER=true ;;
        --rebuild-vllm) REBUILD_VLLM=true ;;
        --vllm-ref) VLLM_REF="$2"; VLLM_REF_SET=true; shift ;;
+        --flashinfer-ref) FLASHINFER_REF="$2"; FLASHINFER_REF_SET=true; shift ;;
        -c|--copy-to|--copy-to-host|--copy-to-hosts)
+            COPY_TO_FLAG=true
            shift
            while [[ "$#" -gt 0 && "$1" != -* ]]; do
                add_copy_hosts "$1"
                shift
            done
-
-            if [ "${#COPY_HOSTS[@]}" -eq 0 ]; then
-                echo "No hosts specified. Using autodiscovery..."
-                source "$(dirname "$0")/autodiscover.sh"
-
-                detect_nodes
-                if [ $? -ne 0 ]; then
-                    echo "Error: Autodiscovery failed."
-                    exit 1
-                fi
-
-                if [ ${#PEER_NODES[@]} -gt 0 ]; then
-                    COPY_HOSTS=("${PEER_NODES[@]}")
-                fi
-
-                if [ "${#COPY_HOSTS[@]}" -eq 0 ]; then
-                     echo "Error: Autodiscovery found no other nodes."
-                     exit 1
-                fi
-                echo "Autodiscovered hosts: ${COPY_HOSTS[*]}"
-            fi
            continue
            ;;
        -j|--build-jobs) BUILD_JOBS="$2"; shift ;;
@@ -337,8 +331,22 @@ while [[ "$#" -gt 0 ]]; do
               exit 1
            fi
            ;;
+        --apply-flashinfer-pr)
+            if [ -n "$2" ] && [[ "$2" != -* ]]; then
+               if [ -n "$FLASHINFER_PRS" ]; then
+                   FLASHINFER_PRS="$FLASHINFER_PRS $2"
+               else
+                   FLASHINFER_PRS="$2"
+               fi
+               shift
+            else
+               echo "Error: --apply-flashinfer-pr requires a PR number."
+               exit 1
+            fi
+            ;;
        --full-log) FULL_LOG=true ;;
        --no-build) NO_BUILD=true ;;
+        --cleanup) CLEANUP_MODE=true ;;
        --network)
            if [ -n "$2" ] && [[ "$2" != -* ]]; then
                NETWORK_ARG="$2"
@@ -348,19 +356,75 @@ while [[ "$#" -gt 0 ]]; do
                exit 1
            fi
            ;;
+        --config) CONFIG_FILE="$2"; CONFIG_FILE_SET=true; shift ;;
+        --setup) FORCE_DISCOVER=true; export FORCE_DISCOVER ;;
        -h|--help) usage ;;
        *) echo "Unknown parameter passed: $1"; usage ;;
    esac
    shift
 done

+# Apply default IMAGE_TAG based on flags if -t was not specified
+if [ "$IMAGE_TAG_SET" = false ]; then
+    if [ "$PRE_TRANSFORMERS" = true ]; then
+        IMAGE_TAG="vllm-node-tf5"
+    elif [ "$EXP_MXFP4" = true ]; then
+        IMAGE_TAG="vllm-node-mxfp4"
+    fi
+fi
+
+# Source autodiscover.sh to load .env file
+source "$(dirname "$0")/autodiscover.sh"
+
+# If --setup: force full autodiscovery and save configuration
+if [[ "${FORCE_DISCOVER:-false}" == "true" ]]; then
+    echo "Running full autodiscovery (--setup)..."
+    detect_interfaces || exit 1
+    detect_local_ip || exit 1
+    detect_nodes || exit 1
+    detect_copy_hosts || exit 1
+    save_config || exit 1
+    # Reload .env so DOTENV_* variables reflect saved config
+    load_env_if_exists
+fi
+
+# Handle COPY_HOSTS from .env or autodiscovery only if -c was explicitly specified
+if [ "$COPY_TO_FLAG" = true ] && [ "${#COPY_HOSTS[@]}" -eq 0 ]; then
+    if [[ -n "$DOTENV_COPY_HOSTS" ]]; then
+        echo "Using COPY_HOSTS from .env: $DOTENV_COPY_HOSTS"
+        IFS=',' read -ra HOSTS_FROM_ENV <<< "$DOTENV_COPY_HOSTS"
+        COPY_HOSTS=("${HOSTS_FROM_ENV[@]}")
+    else
+        echo "No hosts specified. Using autodiscovery..."
+        detect_interfaces || { echo "Error: Interface detection failed."; exit 1; }
+        detect_local_ip || { echo "Error: Local IP detection failed."; exit 1; }
+        detect_nodes || { echo "Error: Node detection failed."; exit 1; }
+        detect_copy_hosts || { echo "Error: Copy host detection failed."; exit 1; }
+
+        if [ "${#COPY_PEER_NODES[@]}" -gt 0 ]; then
+            COPY_HOSTS=("${COPY_PEER_NODES[@]}")
+        fi
+
+        if [ "${#COPY_HOSTS[@]}" -eq 0 ]; then
+            echo "Error: Autodiscovery found no other nodes."
+            exit 1
+        fi
+        echo "Autodiscovered hosts: ${COPY_HOSTS[*]}"
+    fi
+fi
+
 # Validate flag combinations
 if [ -n "$VLLM_PRS" ]; then
    if [ "$EXP_MXFP4" = true ]; then echo "Error: --apply-vllm-pr is incompatible with --exp-mxfp4"; exit 1; fi
 fi

+if [ -n "$FLASHINFER_PRS" ]; then
+    if [ "$EXP_MXFP4" = true ]; then echo "Error: --apply-flashinfer-pr is incompatible with --exp-mxfp4"; exit 1; fi
+fi
+
 if [ "$EXP_MXFP4" = true ]; then
    if [ "$VLLM_REF_SET" = true ]; then echo "Error: --exp-mxfp4 is incompatible with --vllm-ref"; exit 1; fi
+    if [ "$FLASHINFER_REF_SET" = true ]; then echo "Error: --exp-mxfp4 is incompatible with --flashinfer-ref"; exit 1; fi
    if [ "$PRE_TRANSFORMERS" = true ]; then echo "Error: --exp-mxfp4 is incompatible with --tf5"; exit 1; fi
    if [ "$REBUILD_FLASHINFER" = true ]; then echo "Error: --exp-mxfp4 is incompatible with --rebuild-flashinfer"; exit 1; fi
    if [ "$REBUILD_VLLM" = true ]; then echo "Error: --exp-mxfp4 is incompatible with --rebuild-vllm"; exit 1; fi
@@ -372,6 +436,30 @@ if [ "$NO_BUILD" = true ] && [ "${#COPY_HOSTS[@]}" -eq 0 ]; then
    exit 1
 fi

+# Handle cleanup mode
+if [[ "$CLEANUP_MODE" == "true" ]]; then
+    WHEELS_DIR="./wheels"
+    echo "Cleaning up wheels directory..."
+    
+    # Remove all .whl files
+    if compgen -G "$WHEELS_DIR/*.whl" > /dev/null 2>&1; then
+        rm -f "$WHEELS_DIR"/*.whl
+        echo "Removed *.whl files from $WHEELS_DIR"
+    else
+        echo "No *.whl files found in $WHEELS_DIR"
+    fi
+    
+    # Remove all .-commit files
+    if compgen -G "$WHEELS_DIR/.*-commit" > /dev/null 2>&1; then
+        rm -f "$WHEELS_DIR"/.*-commit
+        echo "Removed .*-commit files from $WHEELS_DIR"
+    else
+        echo "No .*-commit files found in $WHEELS_DIR"
+    fi
+    
+    echo "Cleanup complete."
+fi
+
 # Ensure wheels directory exists
 mkdir -p ./wheels

@@ -414,9 +502,21 @@ if [ "$NO_BUILD" = false ]; then
        # ----------------------------------------------------------
        # Phase 1: FlashInfer wheels
        # ----------------------------------------------------------
+        if [ "$FLASHINFER_REF_SET" = true ] || [ -n "$FLASHINFER_PRS" ]; then
+            REBUILD_FLASHINFER=true
+        fi
+
        BUILD_FLASHINFER=false
        if [ "$REBUILD_FLASHINFER" = true ]; then
-            echo "Rebuilding FlashInfer wheels (--rebuild-flashinfer specified)..."
+            if [ "$FLASHINFER_REF_SET" = true ] && [ -n "$FLASHINFER_PRS" ]; then
+                echo "Rebuilding FlashInfer wheels (--flashinfer-ref and --apply-flashinfer-pr specified)..."
+            elif [ "$FLASHINFER_REF_SET" = true ]; then
+                echo "Rebuilding FlashInfer wheels (--flashinfer-ref specified)..."
+            elif [ -n "$FLASHINFER_PRS" ]; then
+                echo "Rebuilding FlashInfer wheels (--apply-flashinfer-pr specified)..."
+            else
+                echo "Rebuilding FlashInfer wheels (--rebuild-flashinfer specified)..."
+            fi
            BUILD_FLASHINFER=true
        elif try_download_wheels "$FLASHINFER_RELEASE_TAG" "flashinfer"; then
            echo "FlashInfer wheels ready."
@@ -438,12 +538,18 @@ if [ "$NO_BUILD" = false ]; then
            FI_CMD=("docker" "build"
                "--target" "flashinfer-export"
                "--output" "type=local,dest=./wheels"
-                "${COMMON_BUILD_FLAGS[@]}")
+                "${COMMON_BUILD_FLAGS[@]}"
+                "--build-arg" "FLASHINFER_REF=$FLASHINFER_REF")

            if [ "$REBUILD_FLASHINFER" = true ]; then
                FI_CMD+=("--build-arg" "CACHEBUST_FLASHINFER=$(date +%s)")
            fi

+            if [ -n "$FLASHINFER_PRS" ]; then
+                echo "Applying FlashInfer PRs: $FLASHINFER_PRS"
+                FI_CMD+=("--build-arg" "FLASHINFER_PRS=$FLASHINFER_PRS")
+            fi
+
            FI_CMD+=(".")

            echo "FlashInfer build command: ${FI_CMD[*]}"
--- a/docs/NETWORKING.md
+++ b/docs/NETWORKING.md
@@ -42,13 +42,54 @@ However, in order to get full bandwidth in NCCL RDMA mode, we need to utilize **
 Also, note that connecting two Sparks using **both** ports won't give you any noticeable advantage in bandwidth, so single connection is sufficient.
 If you connect 3 Sparks by daisy-chaining them, you will only be able to sustain 100G between each pair of Sparks.

-## Connecting more than 2 Sparks in the cluster
+## Connecting 3 Sparks in a mesh cluster without a switch
+
+Three Sparks can be connected together in a cluster without using a separate RoCE switch.
+However, all three Sparks need to be on the same wired network using it's 10G Ethernet port (RG-45, not QSFP). Being on a same wireless network should work too, but it's not recommended and was not tested.
+
+You need to make sure they are connected the following way: port 0 on one Spark should connect to port 1 on another Spark (unlike non-mesh configuration).
+Example diagram:
+
+```mermaid
+block-beta
+    columns 1
+    
+    block:Spark3
+        columns 2
+        Title3["Spark 3"]:2
+        s3p0["Port 0<br>192.168.187.13<br>192.168.188.13"] s3p1["Port 1<br>192.168.197.13<br>192.168.198.13"]
+    end
+    
+    space
+    
+    block:Spark2
+        columns 2
+        Title2["Spark 2"]:2
+        s2p0["Port 0<br>192.168.197.12<br>192.168.198.12"] s2p1["Port 1<br>192.168.177.12<br>192.168.178.13"]
+    end
+    
+    space
+    
+    block:Spark1
+        columns 2
+        Title1["Spark 1"]:2
+        s1p0["Port 0<br>192.168.177.11<br>192.168.178.11"] s1p1["Port 1<br>192.168.187.11<br>192.168.188.11"]
+    end
+
+    s1p0 <--> s2p1
+    s2p0 <--> s3p1
+    s3p0 <--> s1p1
+```
+
+## Connecting more than 2 Sparks in the cluster using a switch

 To connect more than 2 Sparks, you will need a proper switch, for example [Microtik CRS812-DDQ](https://mikrotik.com/product/crs812_ddq).
 Please refer to [this post](https://forums.developer.nvidia.com/t/6x-spark-setup/354399/56) for an example of setting up a 6-8 node Spark cluster.

 ## Network setup

+### For dual Sparks or multiple Sparks using a QSFP switch
+
 Assuming both are connected using rightmost QFSP port (when looking from the back).

 Create `/etc/netplan/40-cx7.yaml` on `spark`:
@@ -65,8 +106,9 @@ network:
    enP2p1s0f1np1:
      dhcp4: no
      dhcp6: no
-      link-local: [ ipv4 ]
+      link-local: []
      mtu: 9000
+      addresses: [192.168.178.11/24]
 ```

 Create `/etc/netplan/40-cx7.yaml` on `spark2`:
@@ -83,16 +125,12 @@ network:
    enP2p1s0f1np1:
      dhcp4: no
      dhcp6: no
-      link-local: [ ipv4 ]
+      link-local: []
      mtu: 9000
+      addresses: [192.168.178.12/24]
 ```

-Please note, that only one interface of the "twin" pair needs an IP address, but MTU needs to be set on both.
-You can also assign a separate address to another "twin" if you want to utilize the second interface independently, but make sure you assign an IP address from a different subnet.
-
-For instance, for the example above, if you want to assign an IP to `enP2p1s0f1np1`, you need to use `192.168.177.12` on `spark`. **DO NOT use the same subnet on both "twins"** - it will confuse autodiscovery and mess up routing.
-
-This will not affect vLLM performance as it will use RDMA over RoCE using both "twins", even if the IP is only set on one.
+**DO NOT use the same subnet on both "twins"** - it will confuse autodiscovery and mess up routing.

 Then run on each node:

@@ -115,6 +153,122 @@ MTU setting (testing):
 sudo ip link set dev enp1s0f1np1 mtu 9000
 ```

+### For 3-node mesh
+
+3-node mesh is configured differently than dual clusters or clusters using a QSFP switch.
+
+Assuming, your Sparks are connected according to the diagram above:
+
+Create `/etc/netplan/40-cx7.yaml` on `spark1`:
+```yaml
+network:
+  version: 2
+  ethernets:
+    enp1s0f0np0:
+      dhcp4: no
+      dhcp6: no        # Explicitly disable DHCPv6
+      link-local: []   # Restrict link-local addresses to static IPv4 only
+      mtu: 9000
+      addresses: [192.168.177.11/24]
+    enP2p1s0f0np0:
+      dhcp4: no
+      dhcp6: no
+      link-local: []
+      mtu: 9000
+      addresses: [192.168.178.11/24]
+    enp1s0f1np1:
+      dhcp4: no
+      dhcp6: no        # Explicitly disable DHCPv6
+      link-local: []   # Restrict link-local addresses to static IPv4 only
+      mtu: 9000
+      addresses: [192.168.187.11/24]
+    enP2p1s0f1np1:
+      dhcp4: no
+      dhcp6: no
+      link-local: []
+      mtu: 9000
+      addresses: [192.168.188.11/24]
+```
+
+Create `/etc/netplan/40-cx7.yaml` on `spark2`:
+```yaml
+network:
+  version: 2
+  ethernets:
+    enp1s0f0np0:
+      dhcp4: no
+      dhcp6: no        # Explicitly disable DHCPv6
+      link-local: []   # Restrict link-local addresses to static IPv4 only
+      mtu: 9000
+      addresses: [192.168.197.12/24]
+    enP2p1s0f0np0:
+      dhcp4: no
+      dhcp6: no
+      link-local: []
+      mtu: 9000
+      addresses: [192.168.198.12/24]
+    enp1s0f1np1:
+      dhcp4: no
+      dhcp6: no        # Explicitly disable DHCPv6
+      link-local: []   # Restrict link-local addresses to static IPv4 only
+      mtu: 9000
+      addresses: [192.168.177.12/24]
+    enP2p1s0f1np1:
+      dhcp4: no
+      dhcp6: no
+      link-local: []
+      mtu: 9000
+      addresses: [192.168.178.12/24]
+```
+
+Create `/etc/netplan/40-cx7.yaml` on `spark3`:
+```yaml
+network:
+  version: 2
+  ethernets:
+    enp1s0f0np0:
+      dhcp4: no
+      dhcp6: no        # Explicitly disable DHCPv6
+      link-local: []   # Restrict link-local addresses to static IPv4 only
+      mtu: 9000
+      addresses: [192.168.187.13/24]
+    enP2p1s0f0np0:
+      dhcp4: no
+      dhcp6: no
+      link-local: []
+      mtu: 9000
+      addresses: [192.168.188.13/24]
+    enp1s0f1np1:
+      dhcp4: no
+      dhcp6: no        # Explicitly disable DHCPv6
+      link-local: []   # Restrict link-local addresses to static IPv4 only
+      mtu: 9000
+      addresses: [192.168.197.13/24]
+    enP2p1s0f1np1:
+      dhcp4: no
+      dhcp6: no
+      link-local: []
+      mtu: 9000
+      addresses: [192.168.198.13/24]
+```
+
+Then run (on each Spark):
+
+```bash
+sudo chmod 600 /etc/netplan/40-cx7.yaml
+sudo netplan apply
+```
+
+### Passwordless SSH and benchmarks
+
+Set up passwordless ssh. On the first spark:
+
+```bash
+wget https://raw.githubusercontent.com/NVIDIA/dgx-spark-playbooks/refs/heads/main/nvidia/connect-two-sparks/assets/discover-sparks
+chmod +x discover-sparks
+./discover-sparks
+```
+
 **Benchmark connection (use perftest package):**

 Run the receiver on `spark2` node:
@@ -196,7 +350,9 @@ ib_write_lat 192.168.177.12 -d rocep1s0f1 --report_gbits -R --force-link IB
 ---------------------------------------------------------------------------------------
 ```

-## NCCL Setup
+## NCCL Tests
+
+### Dual Sparks or Sparks via QSFP switch

 From https://build.nvidia.com/spark/nccl/stacked-sparks

@@ -240,3 +396,51 @@ mpirun -np 2 -H 192.168.177.11:1,192.168.177.12:1 \
  $HOME/nccl-tests/build/all_gather_perf -b 16G -e 16G -f 2

 ```
+
+### 3-node mesh
+
+```bash
+# Install dependencies and build NCCL
+sudo apt-get update && sudo apt-get install -y libopenmpi-dev
+git clone -b dgxspark-3node-ring https://github.com/zyang-dev/nccl.git ~/nccl/
+cd ~/nccl/
+make -j src.build NVCC_GENCODE="-gencode=arch=compute_121,code=sm_121"
+
+# Set environment variables
+export CUDA_HOME="/usr/local/cuda"
+export MPI_HOME="/usr/lib/aarch64-linux-gnu/openmpi"
+export NCCL_HOME="$HOME/nccl/build/"
+export LD_LIBRARY_PATH="$NCCL_HOME/lib:$CUDA_HOME/lib64/:$MPI_HOME/lib:$LD_LIBRARY_PATH"
+```
+
+Build NCCL Test Suite:
+
+```bash
+# Clone and build NCCL tests
+git clone https://github.com/NVIDIA/nccl-tests.git ~/nccl-tests/
+cd ~/nccl-tests/
+make MPI=1
+```
+
+Test on both nodes (replace spark1, spark2, spark3 with the actual hostnames or IP address on non-QSFP interface):
+
+```bash
+# Set environment variables
+export CUDA_HOME="/usr/local/cuda"
+export MPI_HOME="/usr/lib/aarch64-linux-gnu/openmpi"
+export NCCL_HOME="$HOME/nccl_spark_cluster/build/"
+export LD_LIBRARY_PATH="$NCCL_HOME/lib:$CUDA_HOME/lib64/:$MPI_HOME/lib:$LD_LIBRARY_PATH"
+
+# For 3-node mesh we have to use 10G interface for OOB communication!
+export UCX_NET_DEVICES=enP7s7
+export NCCL_SOCKET_IFNAME=enP7s7
+export OMPI_MCA_btl_tcp_if_include=enP7s7
+export NCCL_IB_HCA=rocep1s0f0,roceP2p1s0f0,rocep1s0f1,roceP2p1s0f1
+export NCCL_IB_DISABLE=0
+
+# Run the all_gather performance test across both nodes
+mpirun -np 3 -H spark1:1,spark2:1,spark3:1 \
+  --mca plm_rsh_agent "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \
+  -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH -x NCCL_IB_MERGE_NICS=0 -x NCCL_NET_PLUGIN=none -x NCCL_IB_SUBNET_AWARE_ROUTING=1 \
+  $HOME/nccl-tests/build/all_gather_perf -b 16G -e 16G -f 3
+```
--- a/hf-download.sh
+++ b/hf-download.sh
@@ -7,6 +7,8 @@ HUB_PATH="${HF_HOME:-$HOME/.cache/huggingface}/hub"
 COPY_HOSTS=()
 SSH_USER="$USER"
 PARALLEL_COPY=false
+CONFIG_FILE=""
+CONFIG_FILE_SET=false

 # Help function
 usage() {
@@ -16,6 +18,7 @@ usage() {
    echo "      --copy-to-host          : Alias for --copy-to (backwards compatibility)."
    echo "      --copy-parallel         : Copy to all hosts in parallel instead of serially."
    echo "  -u, --user <user>           : Username for ssh commands (default: \$USER)"
+    echo "  --config <file>             : Path to .env configuration file (default: .env in script directory)"
    echo "  -h, --help                  : Show this help message"
    exit 1
 }
@@ -37,11 +40,11 @@ copy_model_to_host() {
    local host="$1"
    local model_name="$2"
    local model_dir="$3"
-    
+
    echo "Copying model '$model_name' to ${SSH_USER}@${host}..."
    local host_copy_start host_copy_end host_copy_time
    host_copy_start=$(date +%s)
-    
+
    if rsync -av --mkpath --progress "$model_dir" "${SSH_USER}@${host}:$HUB_PATH/"; then
        host_copy_end=$(date +%s)
        host_copy_time=$((host_copy_end - host_copy_start))
@@ -53,44 +56,24 @@ copy_model_to_host() {
 }

 # Argument parsing
+COPY_TO_FLAG=false
 while [[ "$#" -gt 0 ]]; do
    case $1 in
        -c|--copy-to|--copy-to-host|--copy-to-hosts)
+            COPY_TO_FLAG=true
            shift
            # Consume arguments until the next flag or end of args
            while [[ "$#" -gt 0 && "$1" != -* ]]; do
                add_copy_hosts "$1"
                shift
            done
-
-            # If no hosts specified, use autodiscovery
-            if [ "${#COPY_HOSTS[@]}" -eq 0 ]; then
-                echo "No hosts specified. Using autodiscovery..."
-                source "$(dirname "$0")/autodiscover.sh"
-                
-                detect_nodes
-                if [ $? -ne 0 ]; then
-                    echo "Error: Autodiscovery failed."
-                    exit 1
-                fi
-                
-                # Use PEER_NODES directly
-                if [ ${#PEER_NODES[@]} -gt 0 ]; then
-                    COPY_HOSTS=("${PEER_NODES[@]}")
-                fi
-                
-                if [ "${#COPY_HOSTS[@]}" -eq 0 ]; then
-                     echo "Error: Autodiscovery found no other nodes."
-                     exit 1
-                fi
-                echo "Autodiscovered hosts: ${COPY_HOSTS[*]}"
-            fi
            continue
            ;;
        --copy-parallel) PARALLEL_COPY=true ;;
        -u|--user) SSH_USER="$2"; shift ;;
+        --config) CONFIG_FILE="$2"; CONFIG_FILE_SET=true; shift ;;
        -h|--help) usage ;;
-        *) 
+        *)
            # If positional argument is provided
            if [ -z "${MODEL_NAME:-}" ]; then
                MODEL_NAME="$1"
@@ -103,12 +86,47 @@ while [[ "$#" -gt 0 ]]; do
    shift
 done

+# Export config so autodiscover.sh picks it up
+export CONFIG_FILE CONFIG_FILE_SET
+
+# Source autodiscover.sh to load .env (for DOTENV_COPY_HOSTS) and make detection functions available
+source "$(dirname "$0")/autodiscover.sh"
+
 # Validate model name is provided
 if [ -z "${MODEL_NAME:-}" ]; then
    echo "Error: Model name is required."
    usage
 fi

+# Resolve COPY_HOSTS if --copy-to was given without hosts, or use .env
+if [ "$COPY_TO_FLAG" = true ] && [ "${#COPY_HOSTS[@]}" -eq 0 ]; then
+    # --copy-to was specified but no hosts given: use .env or autodiscover
+    if [[ -n "$DOTENV_COPY_HOSTS" ]]; then
+        echo "Using COPY_HOSTS from .env: $DOTENV_COPY_HOSTS"
+        IFS=',' read -ra HOSTS_FROM_ENV <<< "$DOTENV_COPY_HOSTS"
+        COPY_HOSTS=("${HOSTS_FROM_ENV[@]}")
+    else
+        echo "No hosts specified. Using autodiscovery..."
+        detect_interfaces || { echo "Error: Interface detection failed."; exit 1; }
+        detect_local_ip || { echo "Error: Local IP detection failed."; exit 1; }
+        detect_nodes || { echo "Error: Node detection failed."; exit 1; }
+        detect_copy_hosts || { echo "Error: Copy host detection failed."; exit 1; }
+
+        if [ "${#COPY_PEER_NODES[@]}" -gt 0 ]; then
+            COPY_HOSTS=("${COPY_PEER_NODES[@]}")
+        fi
+
+        if [ "${#COPY_HOSTS[@]}" -eq 0 ]; then
+            echo "Error: Autodiscovery found no other nodes."
+            exit 1
+        fi
+        echo "Autodiscovered copy hosts: ${COPY_HOSTS[*]}"
+    fi
+elif [ "$COPY_TO_FLAG" = false ] && [ "${#COPY_HOSTS[@]}" -eq 0 ] && [[ -n "$DOTENV_COPY_HOSTS" ]]; then
+    # No --copy-to flag but .env has COPY_HOSTS — don't auto-copy; user must request it explicitly
+    : # intentional no-op; user didn't ask for copy
+fi
+
 # Check if uvx is installed
 if ! command -v uvx &> /dev/null; then
    echo "Error: 'uvx' command not found."
@@ -231,4 +249,4 @@ if [ "$COPY_TIME" -gt 0 ]; then
 fi
 echo "Total:     $(printf '%02d:%02d:%02d' $((TOTAL_TIME/3600)) $((TOTAL_TIME%3600/60)) $((TOTAL_TIME%60)))"
 echo "========================================="
-echo "Done downloading $MODEL_NAME."
+echo "Done downloading $MODEL_NAME."
--- a/launch-cluster.sh
+++ b/launch-cluster.sh
@@ -24,12 +24,13 @@ CONTAINER_NAME="$DEFAULT_CONTAINER_NAME"
 COMMAND_TO_RUN=""
 DAEMON_MODE="false"
 CHECK_CONFIG="false"
-ACTION="start"
+ACTION=""
 CLUSTER_WAS_RUNNING="false"
 MOD_PATHS=()
 MOD_TYPES=()
 LAUNCH_SCRIPT_PATH=""
 SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+CONFIG_FILE=""  # Will be set to default after argument parsing

 ACTIONS_ARG=""
 SOLO_MODE="false"
@@ -67,9 +68,31 @@ usage() {
    echo "  --mem-swap-limit-gb Memory+swap limit in GB (default: mem-limit + 10, only with --non-privileged)"
    echo "  --pids-limit    Process limit (default: 4096, only with --non-privileged)"
    echo "  --shm-size-gb   Shared memory size in GB (default: 64, only with --non-privileged)"
+    echo "  --config        Path to .env configuration file (default: .env in script directory)
+  --setup/--discover  Force autodiscovery and save configuration (even if .env exists)"
    echo "  action          start | stop | status | exec (Default: start). Not compatible with --launch-script."
    echo "  command         Command to run (only for 'exec' action). Not compatible with --launch-script."
    echo ""
+    echo "Supported .env file variables:"
+    echo "  CLUSTER_NODES       Comma-separated list of node IPs"
+    echo "  ETH_IF              Ethernet interface name"
+    echo "  IB_IF               InfiniBand interface name"
+    echo "  MASTER_PORT         Port for cluster coordination (default: 29501)"
+    echo "  CONTAINER_NAME      Container name (default: vllm_node)"
+    echo "  LOCAL_IP            Local IP address (for solo mode or override auto-detection)"
+    echo "  CONTAINER_*         Any variable starting with CONTAINER_ (except CONTAINER_NAME)"
+    echo "                      becomes -e flag. Example: CONTAINER_NCCL_DEBUG=INFO -> -e NCCL_DEBUG=INFO"
+    echo ""
+    echo "Example .env file:"
+    echo "  CLUSTER_NODES=192.168.1.1,192.168.1.2"
+    echo "  ETH_IF=eth0"
+    echo "  IB_IF=ib0"
+    echo "  MASTER_PORT=29501"
+    echo "  CONTAINER_NAME=vllm_node"
+    echo "  LOCAL_IP=192.168.1.1"
+    echo "  CONTAINER_NCCL_DEBUG=INFO"
+    echo "  CONTAINER_HF_TOKEN=abc123"
+    echo ""
    echo "Launch Script Usage:"
    echo "  $0 --launch-script examples/my-script.sh   # Script copied to container and executed"
    echo "  $0 --launch-script /path/to/script.sh      # Uses absolute path to script"
@@ -108,6 +131,8 @@ while [[ "$#" -gt 0 ]]; do
        --shm-size-gb) SHM_SIZE_GB="$2"; shift ;;
        -d) DAEMON_MODE="true" ;;
        -h|--help) usage ;;
+        --config) CONFIG_FILE="$2"; shift ;;
+        --setup|--discover) FORCE_DISCOVER=true; export FORCE_DISCOVER ;;
        start|stop|status) 
            if [[ -n "$LAUNCH_SCRIPT_PATH" ]]; then
                echo "Error: Action '$1' is not compatible with --launch-script. Please omit the action or not use --launch-script."
@@ -133,6 +158,115 @@ while [[ "$#" -gt 0 ]]; do
    shift
 done

+# Set .env file path (use default if not specified)
+if [[ -z "$CONFIG_FILE" ]]; then
+    CONFIG_FILE="$SCRIPT_DIR/.env"
+    CONFIG_FILE_SET=false
+else
+    CONFIG_FILE_SET=true
+fi
+
+# Load .env file
+if [[ -f "$CONFIG_FILE" ]]; then
+    echo "Loading configuration from .env file..."
+    
+    # Validate .env file syntax
+    if ! python3 -c "
+import sys
+import re
+
+env_file = '$CONFIG_FILE'
+seen_keys = set()
+
+with open(env_file, 'r') as f:
+    for line_num, line in enumerate(f, 1):
+        line = line.strip()
+        # Skip empty lines and comments
+        if not line or line.startswith('#'):
+            continue
+        
+        # Check for key=value format
+        if '=' not in line:
+            print(f'Error: Invalid syntax at line {line_num}: missing \"=\"')
+            sys.exit(1)
+        
+        key = line.split('=', 1)[0].strip()
+        
+        # Validate key format (alphanumeric + underscore)
+        if not re.match(r'^[A-Za-z_][A-Za-z0-9_]*$', key):
+            print(f'Error: Invalid key format at line {line_num}: {key}')
+            sys.exit(1)
+        
+        # Check for duplicates
+        if key in seen_keys:
+            print(f'Error: Duplicate key at line {line_num}: {key}')
+            sys.exit(1)
+        
+        seen_keys.add(key)
+
+sys.exit(0)
+" 2>/dev/null; then
+        echo "Error: Invalid .env file syntax. Aborting."
+        exit 1
+    fi
+    
+    # Load .env variables with DOTENV_ prefix
+    while IFS='=' read -r key value || [[ -n "$key" ]]; do
+        # Skip comments and empty lines
+        [[ "$key" =~ ^[[:space:]]*# ]] && continue
+        [[ -z "$key" ]] && continue
+        
+        # Remove leading/trailing whitespace from key
+        key=$(echo "$key" | xargs)
+        
+        # Skip if key is empty after trimming
+        [[ -z "$key" ]] && continue
+        
+        # Remove quotes and whitespace from value using Python for proper shlex handling
+        value=$(python3 -c "
+import shlex
+import sys
+value = '''$value'''
+# Strip whitespace
+value = value.strip()
+# Remove surrounding quotes if present
+if (value.startswith('\"') and value.endswith('\"')) or (value.startswith(\"'\" ) and value.endswith(\"'\")):
+    value = value[1:-1]
+print(value)
+")
+        
+        # Export with DOTENV_ prefix
+        export "DOTENV_$key=$value"
+    done < "$CONFIG_FILE"
+    
+    echo "Loaded .env variables: $(compgen -v DOTENV_ | tr '\n' ' ')"
+fi
+
+# Apply .env configuration (CLI args take precedence)
+if [[ -z "$NODES_ARG" && -n "$DOTENV_CLUSTER_NODES" ]]; then
+    NODES_ARG="$DOTENV_CLUSTER_NODES"
+fi
+
+if [[ -z "$ETH_IF" && -n "$DOTENV_ETH_IF" ]]; then
+    ETH_IF="$DOTENV_ETH_IF"
+fi
+
+if [[ -z "$IB_IF" && -n "$DOTENV_IB_IF" ]]; then
+    IB_IF="$DOTENV_IB_IF"
+fi
+
+if [[ -z "$MASTER_PORT" || "$MASTER_PORT" == "29501" ]] && [[ -n "$DOTENV_MASTER_PORT" ]]; then
+    MASTER_PORT="$DOTENV_MASTER_PORT"
+fi
+
+if [[ -z "$CONTAINER_NAME" || "$CONTAINER_NAME" == "vllm_node" ]] && [[ -n "$DOTENV_CONTAINER_NAME" ]]; then
+    CONTAINER_NAME="$DOTENV_CONTAINER_NAME"
+fi
+
+if [[ -n "$DOTENV_LOCAL_IP" ]]; then
+    export LOCAL_IP="$DOTENV_LOCAL_IP"
+fi
+
 # Validate non-privileged mode flags
 if [[ "$NON_PRIVILEGED_MODE" == "true" ]]; then
    # Set default swap limit if not specified
@@ -163,6 +297,26 @@ if [[ -n "$NCCL_DEBUG_VAL" ]]; then
    esac
 fi

+# Add container environment variables from .env (CONTAINER_* pattern)
+# Excludes CONTAINER_NAME which is a configuration variable, not an env var
+for env_var in $(compgen -v DOTENV_CONTAINER_); do
+    # Skip CONTAINER_NAME as it's a configuration variable
+    [[ "$env_var" == "DOTENV_CONTAINER_NAME" ]] && continue
+    
+    # Get the value
+    value="${!env_var}"
+    
+    # Extract the actual env var name (remove DOTENV_CONTAINER_ prefix)
+    actual_var="${env_var#DOTENV_CONTAINER_}"
+    
+    # Properly escape the value for shell using Python
+    escaped_value=$(python3 -c "import shlex; print(shlex.quote('$value'))")
+    
+    # Add to docker args
+    DOCKER_ARGS="$DOCKER_ARGS -e $actual_var=$escaped_value"
+    echo "Adding container env: $actual_var"
+done
+
 # Add build job parallelization environment variables if BUILD_JOBS is set
 if [[ -n "$BUILD_JOBS" ]]; then
    DOCKER_ARGS="$DOCKER_ARGS -e MAX_JOBS=$BUILD_JOBS"
@@ -214,7 +368,7 @@ if [[ -n "$LAUNCH_SCRIPT_PATH" ]]; then
    LAUNCH_SCRIPT_MODE="true"

    # If launch script is specified, default action to exec unless explicitly set to stop/status
-    if [[ "$ACTION" == "start" ]]; then
+    if [[ -z "$ACTION" || "$ACTION" == "start" ]]; then
        ACTION="exec"
    fi
 fi
@@ -259,13 +413,33 @@ done
 # Source autodiscover module
 source "$(dirname "$0")/autodiscover.sh"

-if [[ "$SOLO_MODE" == "true" ]]; then
-    if [[ -n "$NODES_ARG" ]]; then
-        echo "Error: --solo is incompatible with -n/--nodes."
-        exit 1
+if [[ "${FORCE_DISCOVER:-false}" == "true" ]]; then
+    # --setup: force full autodiscovery and save configuration
+    echo "Running full autodiscovery (--setup)..."
+    # Clear pre-loaded values so detect functions run fresh instead of short-circuiting
+    ETH_IF="" IB_IF="" NODES_ARG="" LOCAL_IP=""
+    detect_interfaces || exit 1
+    detect_local_ip || exit 1
+    detect_nodes || exit 1
+    detect_copy_hosts || exit 1
+    save_config || exit 1
+    # Reload .env so DOTENV_* variables reflect saved config
+    load_env_if_exists
+    [[ -z "$NODES_ARG" && -n "$DOTENV_CLUSTER_NODES" ]] && NODES_ARG="$DOTENV_CLUSTER_NODES"
+    [[ -z "$ETH_IF" && -n "$DOTENV_ETH_IF" ]] && ETH_IF="$DOTENV_ETH_IF"
+    [[ -z "$IB_IF" && -n "$DOTENV_IB_IF" ]] && IB_IF="$DOTENV_IB_IF"
+    # If no action was specified, setup was the only intent — exit cleanly
+    if [[ -z "$ACTION" && "$LAUNCH_SCRIPT_MODE" != "true" ]]; then
+        exit 0
    fi
+fi
+
+if [[ "$SOLO_MODE" == "true" ]]; then
    # Solo mode: skip node detection, just get local IP
-    LOCAL_IP="127.0.0.1"
+    # Use LOCAL_IP from .env if set, otherwise default to 127.0.0.1
+    if [[ -z "$LOCAL_IP" ]]; then
+        LOCAL_IP="127.0.0.1"
+    fi
    NODES_ARG="$LOCAL_IP"
    PEER_NODES=()
    echo "Solo mode enabled. Skipping node detection."
@@ -337,6 +511,12 @@ if [[ "$ACTION" == "start" || "$ACTION" == "exec" || "$CHECK_CONFIG" == "true" ]
    fi
 fi

+if [[ -z "$ACTION" && "$LAUNCH_SCRIPT_MODE" != "true" && "$CHECK_CONFIG" != "true" ]]; then
+    echo "Error: No action specified. Use: start | stop | status | exec"
+    usage
+    exit 1
+fi
+
 if [[ "$CHECK_CONFIG" == "true" ]]; then
    echo "Configuration Check Complete."
    echo "  Image Name: $IMAGE_NAME"
@@ -552,6 +732,32 @@ apply_mod_to_container() {
    fi
 }

+# Parse -tp/-pp/-dp (and long forms) from a text string (command or script content).
+# Sets TP_SIZE, PP_SIZE, DP_SIZE, PARALLELISM_FOUND globals.
+# Only acts when at least one parallelism flag is present.
+parse_parallelism_from_text() {
+    local text="$1"
+    TP_SIZE=1; PP_SIZE=1; DP_SIZE=1
+    PARALLELISM_FOUND=false
+
+    # Normalize --flag=value to --flag value for uniform word-by-word parsing
+    local normalized
+    normalized=$(echo "$text" | sed 's/\(--[a-z-]*\)=/\1 /g')
+
+    local prev=""
+    for word in $normalized; do
+        case "$prev" in
+            -tp|--tensor-parallel-size)
+                [[ "$word" =~ ^[0-9]+$ ]] && TP_SIZE="$word" && PARALLELISM_FOUND=true ;;
+            -pp|--pipeline-parallel-size)
+                [[ "$word" =~ ^[0-9]+$ ]] && PP_SIZE="$word" && PARALLELISM_FOUND=true ;;
+            -dp|--data-parallel-size)
+                [[ "$word" =~ ^[0-9]+$ ]] && DP_SIZE="$word" && PARALLELISM_FOUND=true ;;
+        esac
+        prev="$word"
+    done
+}
+
 # Build a patched copy of the launch script on the host for a specific node.
 # Strips --distributed-executor-backend and appends multi-node args.
 # Prints the path of the temp file (caller must delete it).
@@ -797,6 +1003,29 @@ exec_no_ray_cluster() {
 }

 if [[ "$ACTION" == "exec" ]]; then
+    # Trim (or error on) PEER_NODES based on declared parallelism, for any multi-node exec
+    if [[ "$SOLO_MODE" != "true" && ${#PEER_NODES[@]} -gt 0 ]]; then
+        if [[ "$LAUNCH_SCRIPT_MODE" == "true" ]]; then
+            cmd_text=$(cat "$LAUNCH_SCRIPT_PATH" 2>/dev/null || true)
+        else
+            cmd_text="$COMMAND_TO_RUN"
+        fi
+        parse_parallelism_from_text "$cmd_text"
+
+        if [[ "$PARALLELISM_FOUND" == "true" ]]; then
+            required_nodes=$(( TP_SIZE * PP_SIZE * DP_SIZE ))
+            total_nodes=$(( 1 + ${#PEER_NODES[@]} ))
+
+            if [[ "$required_nodes" -gt "$total_nodes" ]]; then
+                echo "Error: Command requires $required_nodes nodes (tp=$TP_SIZE * pp=$PP_SIZE * dp=$DP_SIZE) but only $total_nodes node(s) are configured."
+                exit 1
+            elif [[ "$required_nodes" -lt "$total_nodes" ]]; then
+                echo "Note: Command requires $required_nodes node(s) (tp=$TP_SIZE * pp=$PP_SIZE * dp=$DP_SIZE); using $required_nodes of $total_nodes configured node(s)."
+                PEER_NODES=("${PEER_NODES[@]:0:$(( required_nodes - 1 ))}")
+            fi
+        fi
+    fi
+
    start_cluster
    echo "Executing command: $COMMAND_TO_RUN"

--- a/recipes/3x-spark-cluster/qwen3.5-397b-int4-autoround.yaml
+++ b/recipes/3x-spark-cluster/qwen3.5-397b-int4-autoround.yaml
@@ -0,0 +1,61 @@
+# Recipe: Qwen3.5-122B-A10B-INT4-Autoround
+# Qwen3.5-122B model in Intel INT4-Autoround quantization
+# Important: set memory utilization in GB, not percentage! Requires --no-ray to fit full context on two sparks.
+# If you experience node shutdown, please limit GPU clocks on the affected node (or both): `sudo nvidia-smi -lgc 200,2150`
+
+recipe_version: "1"
+name: Qwen3.5-397B-INT4-Autoround (PP=3)
+description: Recipe for Qwen3.5-397B-INT4-Autoround to run on 3-node mesh in pipeline-parallel mode
+
+# HuggingFace model to download (optional, for --download-model)
+model: Intel/Qwen3.5-397B-A17B-int4-AutoRound
+
+cluster_only: true
+
+# Container image to use
+container: vllm-node-tf5
+
+build_args:
+  - --tf5
+
+# Mod required to fix ROPE syntax error
+mods:
+  - mods/fix-qwen3.5-autoround
+  - mods/fix-qwen3.5-chat-template
+
+# Default settings (can be overridden via CLI)
+defaults:
+  port: 8000
+  host: 0.0.0.0
+  pipeline_parallel: 3
+  gpu_memory_utilization: 0.7
+  max_model_len: 262144
+  max_num_batched_tokens: 16384
+
+# Environment variables
+env: 
+  PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" 
+  VLLM_MARLIN_USE_ATOMIC_ADD: 1
+
+# The vLLM serve command template
+command: |
+  vllm serve Intel/Qwen3.5-397B-A17B-int4-AutoRound \
+    --max-model-len {max_model_len} \
+    --max-num-seqs 10 \
+    --kv-cache-dtype fp8 \
+    --gpu-memory-utilization {gpu_memory_utilization} \
+    --port {port} \
+    --host {host} \
+    --enable-prefix-caching \
+    --enable-auto-tool-choice \
+    --tool-call-parser qwen3_coder \
+    --reasoning-parser qwen3 \
+    --max-num-batched-tokens {max_num_batched_tokens} \
+    --trust-remote-code \
+    --chat-template unsloth.jinja \
+    -tp 1 \
+    -pp {pipeline_parallel} \
+    --load-format fastsafetensors \
+    --distributed-executor-backend ray
+
+  
--- a/recipes/README.md
+++ b/recipes/README.md
@@ -44,12 +44,16 @@ The recipe runner can automatically discover cluster nodes:
 ```

 When you run `--discover`, it:
-1. Scans the network for nodes with SSH access
-2. Prompts you to select which nodes to include
-3. Saves the configuration to `.env`
+1. Detects active CX7 interfaces and determines mesh vs. standard topology.
+2. Scans the network for peers that are both SSH-reachable **and** have an NVIDIA GB10 GPU.
+3. In mesh mode, separately discovers `COPY_HOSTS` on the direct IB-attached interfaces.
+4. Prompts for per-node confirmation for `CLUSTER_NODES` and `COPY_HOSTS`.
+5. Saves the full configuration (including mesh NCCL settings if applicable) to `.env`.

 Future recipe runs will automatically use nodes from `.env` unless you specify `-n` or `--solo`.

+When distributing the container image or model files, the runner uses `COPY_HOSTS` from `.env` (which may differ from `CLUSTER_NODES` in mesh mode) to ensure transfers go over the fastest available path.
+
 ## Workflow Modes

 ### Solo Mode (Single Node)
@@ -169,6 +173,7 @@ Usage: ./run-recipe.sh [OPTIONS] [RECIPE]
 Cluster discovery:
  --discover                  Auto-detect cluster nodes and save to .env
  --show-env                  Show current .env configuration
+  --config FILE               Path to .env configuration file (default: .env in repo directory)

 Recipe overrides:
  --port PORT                 Override port
@@ -186,10 +191,25 @@ Setup options:

 Launch options:
  --solo                      Run in solo mode (single node, no Ray)
+  --no-ray                    Multi-node without Ray (PyTorch distributed backend)
  -n, --nodes IPS             Comma-separated node IPs (first = head)
  -d, --daemon                Run in daemon mode
  -t, --container IMAGE       Override container from recipe
+  --name NAME                 Override container name
  --nccl-debug LEVEL          NCCL debug level (VERSION, WARN, INFO, TRACE)
+  --master-port PORT          Cluster coordination port: Ray head port or PyTorch
+                              distributed master port (default: 29501).
+                              Alias: --head-port
+  --eth-if IFACE              Override Ethernet interface
+  --ib-if IFACE               Override InfiniBand interface
+  -e VAR=VALUE                Pass environment variable to container (repeatable)
+  -j N                        Number of parallel build jobs
+  --no-cache-dirs             Do not mount ~/.cache/vllm, ~/.cache/flashinfer, ~/.triton
+  --non-privileged            Run container without --privileged
+  --mem-limit-gb N            Memory limit in GB (only with --non-privileged)
+  --mem-swap-limit-gb N       Memory+swap limit in GB (only with --non-privileged)
+  --pids-limit N              Process limit (only with --non-privileged)
+  --shm-size-gb N             Shared memory size in GB (only with --non-privileged)

 Extra vLLM arguments:
  -- ARGS...                  Pass additional arguments directly to vLLM
@@ -261,10 +281,18 @@ command: |

 ```
 ┌─────────────────────────────────────────────────────────┐
+│  autodiscover.sh                                        │
+│  - Interface detection (standard / mesh topology)       │
+│  - GB10 peer verification via SSH                       │
+│  - CLUSTER_NODES and COPY_HOSTS discovery               │
+│  - Interactive .env save with per-node confirmation     │
+└──────────────────────────┬──────────────────────────────┘
+                           │ sourced by
+                           ▼
+┌─────────────────────────────────────────────────────────┐
 │  run-recipe.sh / run-recipe.py                          │
 │  - Parses YAML recipe                                   │
-│  - Auto-discovers cluster nodes (--discover)            │
-│  - Loads nodes from .env                                │
+│  - Loads / triggers cluster discovery (--discover)      │
 │  - Handles --setup (build + download + run)             │
 │  - Generates launch script from template                │
 │  - Applies CLI overrides                                │
@@ -274,15 +302,15 @@ command: |
 ┌──────────────────────┐  ┌───────────────────────────────┐
 │  build-and-copy.sh   │  │  hf-download.sh               │
 │  - Docker build      │  │  - HuggingFace model download │
-│  - Copy to workers   │  │  - Rsync to workers           │
+│  - Copy to COPY_HOSTS│  │  - Rsync to COPY_HOSTS        │
 └──────────────────────┘  └───────────────────────────────┘
-           │ 
+           │
           │ then calls (for run)
           ▼
 ┌─────────────────────────────────────────────────────────┐
 │  launch-cluster.sh                                      │
 │  - Cluster orchestration                                │
-│  - Container lifecycle                                  │
+│  - Container lifecycle (trimmed to required node count) │
 │  - Mod application                                      │
 │  - Launch script execution                              │
 └─────────────────────────────────────────────────────────┘
--- a/run-recipe.py
+++ b/run-recipe.py