How-To Setup Slurm Block Scheduling to Maximize Throughput Using NVLink Domain for GB200 SKU

Last Updated: Jan 15, 2026

Introduction

The NVIDIA GB200 NVL72 is equipped with NVLink for faster GPU to GPU communication. When using these machines in a Slurm cluster, it is recommended to set up a block-scheduling topology in Slurm to maximize throughput by using the NVLink domain.

Prerequisites

• NVIDIA GB200 NVL72
• Slurm - v25.05 and above
• Crusoe CLI

Step-by-Step Instructions

Step 1: Modify the terraform for Slurm to create VMs with 3 character padding

• To achieve this, the main.tf file in the Slurm repo will need the following change:

resource "crusoe_compute_instance" "slurm_compute_node" {
  count = var.slurm_compute_node_count
  name  = "slurm-compute-node-${format("%03d", count.index)}" <=== change to insert padding
}

Step 2: List all the GB200 VMs in the cluster

• Query all running GB200 VMs in your project and save the output in JSON format

crusoe compute vms list --types gb200-186gb-nvl-ib.4x --states STATE_RUNNING -f json > slurm.json

Step 3: Create a topology.py file using the below Python script

 #!/usr/bin/env python3

import json
import re
import sys
import collections

def node_list_to_bracket_notation(nodes):
    """
    Convert a list of nodes like 'slurm-compute-node-001', 'slurm-compute-node-002', 
    'slurm-compute-node-003', 'slurm-compute-node-005'
    to a compact form like 'slurm-compute-node-[001-003,005]'

    This version is modified to correctly handle the zero-padding and
    bracket formatting seen in the user's example output.
    """
    if not nodes:
        return ""

    # Extract the prefix and node numbers
    prefix_pattern = re.compile(r'(.*?)(\d+)$')
    node_prefix = None
    node_numbers = []
    pad_width = 1  # Default padding if none is detected

    for node in nodes:
        match = prefix_pattern.match(node)
        if match:
            prefix, number_str = match.groups()
            number = int(number_str)
            
            if node_prefix is None:
                # This is the first node, establish prefix and padding
                node_prefix = prefix
                pad_width = len(number_str) # Capture padding from first node
            elif prefix != node_prefix:
                # If we have different prefixes, fall back to comma-separated list
                print(f"Warning: Mixed node prefixes ('{node_prefix}', '{prefix}'). Defaulting to comma list.", file=sys.stderr)
                return ','.join(sorted(nodes))
                
            node_numbers.append(number)
        else:
            # If any node doesn't match the pattern, fall back
            print(f"Warning: Node '{node}' did not match pattern. Defaulting to comma list.", file=sys.stderr)
            return ','.join(sorted(nodes))

    if not node_prefix:
         # This should only happen if list is not empty but no nodes matched
         return ','.join(sorted(nodes))

    # Sort node numbers
    node_numbers.sort()

    # Group consecutive numbers into ranges
    ranges = []
    if not node_numbers:
         return node_prefix # Should be unreachable if nodes list was valid
         
    start = node_numbers[0]
    prev = start

    for num in node_numbers[1:] + [None]:  # Add None as sentinel to handle the last range
        if num is None or num > prev + 1:
            # End of a range
            if prev == start:
                # Single number in range
                ranges.append(f"{start:0{pad_width}d}")
            else:
                # A proper range
                ranges.append(f"{start:0{pad_width}d}-{prev:0{pad_width}d}")
            
            if num is not None:
                start = num
        prev = num if num is not None else prev
        
    # Construct the final string, always using brackets as per the example
    return f"{node_prefix}[{','.join(ranges)}]"

def generate_topology(input_file, output_file):
    """
    Reads VM data from input_file, groups by pod_id, 
    and writes a Slurm topology file to output_file.
    """
    
    # 1. Read the provided JSON file
    print(f"Reading {input_file}...")
    try:
        with open(input_file, 'r') as f:
            vms_data = json.load(f)
    except json.JSONDecodeError as e:
        print(f"Error: Failed to decode JSON from {input_file}.", file=sys.stderr)
        print(f"Details: {e}", file=sys.stderr)
        sys.exit(1)
    except FileNotFoundError:
        print(f"Error: Input file not found: {input_file}", file=sys.stderr)
        sys.exit(1)
    except Exception as e:
        print(f"Error reading file {input_file}: {e}", file=sys.stderr)
        sys.exit(1)

    # 2. Identify pod_id and group VMs
    pods = collections.defaultdict(list)
    
    # Ensure vms_data is a list
    if not isinstance(vms_data, list):
         print(f"Error: Expected JSON file to contain a list of VMs, but found {type(vms_data)}.", file=sys.stderr)
         sys.exit(1)

    for vm in vms_data:
        pod_id = vm.get('pod_id')
        vm_name = vm.get('name')
        
        if pod_id and vm_name:
            pods[pod_id].append(vm_name)
        else:
            print(f"Warning: Skipping VM entry with missing 'pod_id' or 'name': {vm}", file=sys.stderr)

    if not pods:
        print("Warning: No VMs with 'pod_id' and 'name' were found in the JSON file.", file=sys.stderr)

    # 3. Format and prepare output lines
    output_lines = []
    
    # Sort by pod_id to ensure a consistent output order (block01, block02, etc.)
    sorted_pod_ids = sorted(pods.keys())
    
    for i, pod_id in enumerate(sorted_pod_ids, 1):
        block_name = f"block{i:02d}"
        node_list = pods[pod_id]
        
        # Get the formatted node string (e.g., "slurm-compute-node-[001-017]")
        formatted_nodes = node_list_to_bracket_notation(node_list)
        
        output_lines.append(f"BlockName={block_name} Nodes={formatted_nodes}")

    # 4. Output the file
    header = [
        "##################################################################",
        "# Slurm's network topology configuration file for use with the",
        "# topology/block plugin",
        "##################################################################"
    ]
    
    try:
        with open(output_file, 'w') as f:
            for line in header:
                f.write(line + "\n")
            f.write("\n") # Add a blank line
            
            for line in output_lines:
                f.write(line + "\n")
            
            f.write("BlockSizes=18\n")
            
        print(f"Successfully generated {output_file}")
        
    except IOError as e:
        print(f"Error: Could not write to output file {output_file}.", file=sys.stderr)
        print(f"Details: {e}", file=sys.stderr)
        sys.exit(1)

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print(f"Usage: {sys.argv[0]} <input_json_file>")
        sys.exit(1)
        
    input_filename = sys.argv[1]
    output_filename = "topology.conf" # Output file as requested
    
    generate_topology(input_filename, output_filename)

Step 4: Run the python script 

python3 topology.py slurm.json

Example output:

# python3 topology.py slurm.json
Reading slurm.json...
Successfully generated topology.conf

# cat topology.conf
....
BlockName=block01 Nodes=slurm-compute-node-[000-017]
BlockName=block02 Nodes=slurm-compute-node-[018-035]
BlockName=block03 Nodes=slurm-compute-node-[036-053]
BlockName=block04 Nodes=slurm-compute-node-[054-071]
BlockName=block05 Nodes=slurm-compute-node-[072-089]
BlockName=block06 Nodes=slurm-compute-node-[090-107]
BlockName=block07 Nodes=slurm-compute-node-[108-125]
BlockName=block08 Nodes=slurm-compute-node-[126-143]
BlockName=block09 Nodes=slurm-compute-node-[144-161]
BlockName=block10 Nodes=slurm-compute-node-[162-179]
BlockSizes=18

Step 5: Copy the created topology.conf file to the /etc/slurm directory on the Slurm head node

Example:

# cat /etc/slurm/topology.conf
...
##################################################################
# Slurm's network topology configuration file for use with the
# topology/block plugin
##################################################################
BlockName=block01 Nodes=slurm-compute-node-[000-017]
BlockName=block02 Nodes=slurm-compute-node-[018-035]
BlockName=block03 Nodes=slurm-compute-node-[036-053]
BlockName=block04 Nodes=slurm-compute-node-[054-071]
BlockName=block05 Nodes=slurm-compute-node-[072-089]
BlockName=block06 Nodes=slurm-compute-node-[090-107]

Step 6: Edit the slurm.conf file under /etc/slurm directory on the head node to set the TopologyPlugin to block topology instead of the default.

Example:

# cat /etc/slurm/slurm.conf | grep -i topology
TopologyPlugin=topology/block

Step 7: Reconfigure the Slurm cluster to enforce block topology. To do so, run the following command from the head node

# sudo -i scontrol reconfigure

Step 8: Verify the topology is set correctly by running the following command from the login/head node

Example:

ubuntu@slurm-head-node-0:~$ scontrol show topology
BlockName=block01 BlockIndex=0 Nodes=slurm-compute-node-[000-017] BlockSize=18
BlockName=block02 BlockIndex=1 Nodes=slurm-compute-node-[018-035] BlockSize=18
BlockName=block03 BlockIndex=2 Nodes=slurm-compute-node-[036-053] BlockSize=18
BlockName=block04 BlockIndex=3 Nodes=slurm-compute-node-[054-071] BlockSize=18
BlockName=block05 BlockIndex=4 Nodes=slurm-compute-node-[072-089] BlockSize=18
BlockName=block06 BlockIndex=5 Nodes=slurm-compute-node-[090-107] BlockSize=18
BlockName=block07 BlockIndex=6 Nodes=slurm-compute-node-[108-125] BlockSize=18
BlockName=block08 BlockIndex=7 Nodes=slurm-compute-node-[126-143] BlockSize=18
BlockName=block09 BlockIndex=8 Nodes=slurm-compute-node-[144-161] BlockSize=18
BlockName=block10 BlockIndex=9 Nodes=slurm-compute-node-[162-179] BlockSize=18

Additional Resources

https://slurm.schedmd.com/SLUG24/NVIDIA-Craig_Tierney.pdf