{
  "hardware_profiles": {
    "h100": {
      "brand": "NVIDIA",
      "generation": "hopper",
      "compute_arch": "sm90",
      "display_name": "H100",
      "description": "NVIDIA H100 80GB NVLink (8-GPU node)",
      "gpu_count": 8,
      "vram_gb": 640,
      "multi_node": false
    },
    "h200": {
      "brand": "NVIDIA",
      "generation": "hopper",
      "compute_arch": "sm90",
      "display_name": "H200",
      "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node",
      "gpu_count": 8,
      "vram_gb": 1128,
      "multi_node": false
    },
    "b200": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm100",
      "display_name": "B200",
      "description": "NVIDIA B200 SXM 180 GB HBM3e · 8-GPU HGX B200 node",
      "gpu_count": 8,
      "vram_gb": 1440,
      "multi_node": false
    },
    "gb200": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm100",
      "display_name": "GB200 NVL4",
      "description": "NVIDIA GB200 Grace-Blackwell compute tray · 192 GB HBM3e/GPU · 4 Blackwell GPUs (NVL72 tray unit)",
      "gpu_count": 4,
      "vram_gb": 768,
      "multi_node": false
    },
    "b300": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm103",
      "display_name": "B300",
      "description": "NVIDIA B300 SXM 268 GB HBM3e · 8-GPU HGX B300 node (Blackwell Ultra)",
      "gpu_count": 8,
      "vram_gb": 2144,
      "multi_node": false
    },
    "gb300": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm103",
      "display_name": "GB300 NVL4",
      "description": "NVIDIA GB300 Grace-Blackwell Ultra compute tray · 288 GB HBM3e/GPU · 4 GPUs (NVL72 tray unit)",
      "gpu_count": 4,
      "vram_gb": 1152,
      "multi_node": false
    },
    "vr200": {
      "brand": "NVIDIA",
      "generation": "rubin",
      "compute_arch": "sm107",
      "display_name": "VR200 NVL4",
      "description": "NVIDIA Vera Rubin compute tray · 288 GB HBM4/GPU · 4 Rubin GPUs (NVL72 tray unit)",
      "gpu_count": 4,
      "vram_gb": 1152,
      "multi_node": false,
      "restricted": true,
      "docker_image": "vllm/vllm-openai:cu134-nightly",
      "install": {
        "pip": false,
        "docker": {
          "note": "Rubin support is a preview: only this nightly image (CUDA 13.4, PyTorch 2.15) runs on it, and there is no pip wheel yet."
        }
      }
    },
    "dgx_station_gb300": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm103",
      "display_name": "DGX Station (GB300)",
      "description": "NVIDIA DGX Station · 1× GB300 Grace-Blackwell Ultra Superchip · 252 GB HBM3e (desktop workstation)",
      "gpu_count": 1,
      "vram_gb": 252,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "dgx_spark_gb10": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm121",
      "display_name": "DGX Spark (GB10)",
      "description": "NVIDIA DGX Spark · 1× GB10 Grace Blackwell Superchip · 128 GB unified memory",
      "gpu_count": 1,
      "vram_gb": 128,
      "multi_node": false,
      "restricted": true
    },
    "rtx_pro_6000": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX Pro 6000",
      "description": "NVIDIA RTX Pro 6000 · 1× RTX Pro 6000 GPU · 96 GB VRAM",
      "gpu_count": 1,
      "vram_gb": 96,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_pro_6000_2x": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX Pro 6000 2x",
      "description": "NVIDIA RTX Pro 6000 · 2× RTX Pro 6000 GPU · 192 GB VRAM",
      "gpu_count": 2,
      "vram_gb": 192,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_pro_6000_4x": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX Pro 6000 4x",
      "description": "NVIDIA RTX Pro 6000 · 4× RTX Pro 6000 GPU · 384 GB VRAM",
      "gpu_count": 4,
      "vram_gb": 384,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_pro_5000": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX PRO 5000",
      "description": "NVIDIA RTX PRO 5000 Blackwell · 72 GB VRAM",
      "gpu_count": 1,
      "vram_gb": 72,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_pro_5000_4x": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX PRO 5000 Blackwell 4x",
      "description": "NVIDIA RTX PRO 5000 Blackwell · 4× 72 GB VRAM · PCIe, no NVLink",
      "gpu_count": 4,
      "vram_gb": 288,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_pro_6000_8x": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX Pro 6000 8x",
      "description": "NVIDIA RTX Pro 6000 · 8× RTX Pro 6000 GPU · 768 GB VRAM",
      "gpu_count": 8,
      "vram_gb": 768,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_5090": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX 5090",
      "description": "NVIDIA RTX 5090 · 1× RTX 5090 GPU · 32 GB VRAM",
      "gpu_count": 1,
      "vram_gb": 32,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_4090_2x": {
      "brand": "NVIDIA",
      "generation": "ada",
      "compute_arch": "sm89",
      "display_name": "RTX 4090",
      "description": "NVIDIA RTX 4090 · 2× RTX 4090 GPU · 24 GB VRAM/GPU",
      "gpu_count": 2,
      "vram_gb": 48,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_5090_2x": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX 5090",
      "description": "NVIDIA RTX 5090 · 2× RTX 5090 GPU · 32 GB VRAM/GPU",
      "gpu_count": 2,
      "vram_gb": 64,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "l40s": {
      "brand": "NVIDIA",
      "generation": "ada",
      "compute_arch": "sm89",
      "display_name": "L40S",
      "description": "NVIDIA L40S · 1× L40S GPU · 48 GB GDDR6 ECC (PCIe, no NVLink)",
      "gpu_count": 1,
      "vram_gb": 48,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "mi300x": {
      "brand": "AMD",
      "generation": "amd",
      "compute_arch": "gfx942",
      "display_name": "MI300X",
      "description": "AMD Instinct MI300X 192 GB HBM3 · 8-GPU OAM node (CDNA 3)",
      "gpu_count": 8,
      "vram_gb": 1536,
      "multi_node": false
    },
    "mi325x": {
      "brand": "AMD",
      "generation": "amd",
      "compute_arch": "gfx942",
      "display_name": "MI325X",
      "description": "AMD Instinct MI325X 256 GB HBM3e · 8-GPU OAM node (CDNA 3)",
      "gpu_count": 8,
      "vram_gb": 2048,
      "multi_node": false
    },
    "mi350x": {
      "brand": "AMD",
      "generation": "amd",
      "compute_arch": "gfx950",
      "display_name": "MI350X",
      "description": "AMD Instinct MI350X 288 GB HBM3e · 8-GPU OAM node (CDNA 4)",
      "gpu_count": 8,
      "vram_gb": 2304,
      "multi_node": false
    },
    "mi355x": {
      "brand": "AMD",
      "generation": "amd",
      "compute_arch": "gfx950",
      "display_name": "MI355X",
      "description": "AMD Instinct MI355X 288 GB HBM3e · 8-GPU OAM node (CDNA 4)",
      "gpu_count": 8,
      "vram_gb": 2304,
      "multi_node": false
    },
    "trillium": {
      "brand": "Google",
      "generation": "tpu",
      "display_name": "TPU v6e (Trillium)",
      "description": "Google Cloud TPU v6e Trillium · 32 GB HBM/chip · served via vLLM TPU",
      "gpu_count": 8,
      "vram_gb": 256,
      "multi_node": false,
      "restricted": true
    },
    "ironwood": {
      "brand": "Google",
      "generation": "tpu",
      "display_name": "TPU7x",
      "description": "Google Cloud TPU v7 Ironwood · 206 GB HBM3e/chip (96 GiB per TensorCore) · served via vLLM TPU",
      "gpu_count": 8,
      "vram_gb": 824,
      "chips_per_node": 4,
      "slice_topology": "2x2x{n}",
      "multi_node": false,
      "restricted": true
    },
    "xeon6": {
      "brand": "Intel",
      "generation": "cpu",
      "display_name": "Xeon 6",
      "description": "Intel Xeon 6 CPU NUMA node · x86 CPU backend",
      "gpu_count": 4,
      "vram_gb": 1024,
      "multi_node": false,
      "restricted": true
    },
    "xeon5": {
      "brand": "Intel",
      "generation": "cpu",
      "display_name": "Xeon 5",
      "description": "Intel Xeon 5 CPU NUMA node · x86 CPU backend",
      "gpu_count": 4,
      "vram_gb": 1024,
      "multi_node": false,
      "restricted": true
    },
    "arc_pro_b60": {
      "brand": "Intel",
      "generation": "xpu",
      "display_name": "Arc Pro B60",
      "description": "Intel Arc Pro B60 24 GB · 4-card node · vLLM XPU backend",
      "gpu_count": 4,
      "vram_gb": 96,
      "multi_node": false,
      "restricted": true
    },
    "arc_pro_b70": {
      "brand": "Intel",
      "generation": "xpu",
      "display_name": "Arc Pro B70",
      "description": "Intel Arc Pro B70 32 GB · 4-card node · vLLM XPU backend",
      "gpu_count": 4,
      "vram_gb": 128,
      "multi_node": false,
      "restricted": true
    },
    "ascend_950pr": {
      "brand": "Huawei",
      "generation": "npu",
      "display_name": "Ascend 950PR",
      "description": "Huawei Ascend 950PR · 1× NPU · 128 GB HiBL 1.0 HBM · vLLM Ascend backend",
      "gpu_count": 1,
      "vram_gb": 128,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "ascend_950dt": {
      "brand": "Huawei",
      "generation": "npu",
      "display_name": "Ascend 950DT",
      "description": "Huawei Ascend 950DT · 8× NPU node · 96 GB HBM each · vLLM Ascend backend",
      "gpu_count": 8,
      "vram_gb": 768,
      "multi_node": false,
      "restricted": true
    },
    "ascend_910c": {
      "brand": "Huawei",
      "generation": "npu",
      "display_name": "Ascend 910C",
      "description": "Huawei Ascend 910C · 16× NPU · 64 GB HBM each · Atlas 800 A3 · vLLM Ascend backend",
      "gpu_count": 16,
      "vram_gb": 1024,
      "multi_node": false,
      "restricted": true
    },
    "ascend_950dt_8x": {
      "brand": "Huawei",
      "generation": "npu",
      "display_name": "Ascend 950DT 8x",
      "description": "Huawei Ascend 950DT · 8× NPU · 96 GB HBM each · Atlas A5 · vLLM Ascend backend",
      "gpu_count": 8,
      "vram_gb": 768,
      "multi_node": false,
      "restricted": true
    },
    "ascend_950pr_8x": {
      "brand": "Huawei",
      "generation": "npu",
      "display_name": "Ascend 950PR 8x",
      "description": "Huawei Ascend 950PR · 8× NPU · 128 GB HiBL 1.0 HBM each · Atlas A5 · vLLM Ascend backend",
      "gpu_count": 8,
      "vram_gb": 1024,
      "multi_node": false,
      "restricted": true
    },
    "810e": {
      "brand": "Zhenwu",
      "generation": "ppu",
      "display_name": "810E",
      "description": "Zhenwu 810E · 16× PPU · 96 GB per PPU",
      "gpu_count": 16,
      "vram_gb": 1536,
      "multi_node": true,
      "restricted": true,
      "install": {
        "pip": false
      }
    },
    "m890": {
      "brand": "Zhenwu",
      "generation": "ppu",
      "display_name": "M890",
      "description": "Zhenwu M890 · 8× PPU · 144 GB per PPU",
      "gpu_count": 8,
      "vram_gb": 1152,
      "multi_node": true,
      "restricted": true,
      "install": {
        "pip": false
      }
    }
  },
  "tasks": [
    {
      "id": "text",
      "display_name": "Text",
      "description": "Text-only models: chat, reasoning, code generation, tool calling"
    },
    {
      "id": "multimodal",
      "display_name": "Multimodal",
      "description": "Vision, audio, or other modalities beyond text"
    },
    {
      "id": "omni",
      "display_name": "Omni",
      "description": "Any-to-any models: text + vision + audio + generation"
    },
    {
      "id": "embedding",
      "display_name": "Embedding",
      "description": "Embedding, reranker, classifier, scoring models"
    }
  ],
  "strategies": [
    "single_node_tp",
    "single_node_tep",
    "single_node_dep",
    "multi_node_tp",
    "multi_node_tep",
    "multi_node_dep",
    "multi_node_tp_pp",
    "multi_node_tp_dp",
    "pd_cluster"
  ],
  "frontends": {
    "rust": {
      "label": "Rust",
      "env": {
        "VLLM_USE_RUST_FRONTEND": "1"
      }
    },
    "python": {
      "label": "Python"
    }
  },
  "kv_offload_groups": {
    "offloading": {
      "display_name": "Offloading Connector",
      "label": "Offloading",
      "order": 2,
      "sub_row_label": "Offload Tiers",
      "docs_url": "https://docs.vllm.ai/en/stable/features/kv_offloading_usage",
      "tooltip": "Offloading Connector — vLLM's built-in KV offload path: extends the prefix cache into host DRAM, optionally backed by further tiers. Composes with the selected strategy.",
      "description": "vLLM's built-in OffloadingConnector extends the prefix cache beyond GPU memory: completed KV blocks are copied out by DMA alongside compute and promoted back on a hit. A CPU DRAM tier is always present and is the only tier with GPU access; optional secondary tiers sit behind it and stage their transfers through it. Pick the tiers below."
    }
  },
  "kv_offload": {
    "simple": {
      "display_name": "Simple (CPU offload)",
      "label": "Simple",
      "order": 1,
      "description": "Offload KV cache to CPU DRAM on each serving node via SimpleCPUOffloadConnector — one flag, no extra infrastructure. ~220 GiB of host RAM used per GPU rank.",
      "args": [
        "--kv-transfer-config",
        "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":236223201280,\"lazy_offload\":false}}"
      ],
      "hardware": {
        "tpu": "unsupported"
      },
      "pd": {
        "roles": [
          "prefill"
        ]
      }
    },
    "offloading_cpu": {
      "display_name": "CPU DRAM only",
      "label": "CPU",
      "group": "offloading",
      "order": 1,
      "requires_opt_in": true,
      "brands": [
        "NVIDIA",
        "AMD"
      ],
      "description": "Single CPU DRAM tier, under LRU eviction. cpu_bytes_to_use is the total across all workers — not per rank, as Simple's knob is — and should exceed the aggregate GPU KV cache, or the CPU tier just mirrors what the GPU already holds.",
      "args": [
        "--kv-transfer-config",
        "{\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":549755813888,\"blocks_per_chunk\":4}}"
      ]
    },
    "offloading_fs": {
      "display_name": "CPU DRAM + filesystem",
      "label": "CPU + Filesystem",
      "group": "offloading",
      "order": 2,
      "requires_opt_in": true,
      "brands": [
        "NVIDIA",
        "AMD"
      ],
      "description": "Adds a filesystem tier behind the CPU tier, so cached prefixes outlive DRAM capacity and survive a restart. The CPU tier becomes a smaller staging tier; the filesystem carries the capacity. Point root_dir at fast local NVMe, or at a shared mount to pool one cache across instances. Object-store (obj) and RDMA-peer (p2p) tiers configure the same way in secondary_tiers.",
      "env": {
        "PYTHONHASHSEED": "0"
      },
      "args": [
        "--kv-transfer-config",
        "{\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":34359738368,\"blocks_per_chunk\":4,\"eviction_policy\":\"lru\",\"secondary_tiers\":[{\"type\":\"fs\",\"root_dir\":\"/mnt/kv_cache\",\"n_read_threads\":32,\"n_write_threads\":16}]}}"
      ]
    },
    "lmcache": {
      "display_name": "LMCache",
      "order": 4,
      "description": "Offload KV cache to a shared CPU-DRAM pool via LMCacheMPConnector — prefix caching beyond GPU KV capacity, one pool per node (not per-rank segments). Adds an LMCache Server tab; run it in a separate terminal before vllm serve.",
      "strategies": [
        "single_node_tp",
        "single_node_tep",
        "single_node_dep"
      ],
      "args": [
        "--kv-transfer-config",
        "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":5555,\"lmcache.mp.mq_timeout\":300}}"
      ],
      "install": "uv pip install \"lmcache==0.5.1\"",
      "docs_url": "https://docs.lmcache.ai/",
      "companion": {
        "label": "LMCache Server",
        "description": "Run on the same node in a separate terminal, before vllm serve — owns the shared CPU-DRAM KV pool.",
        "command": "# --port must match \"lmcache.mp.port\" in --kv-transfer-config.\n# Size --l1-size-gb to ~75% of the host DRAM you can dedicate — pinned\n# memory is unswappable; the pool grows lazily from --l1-init-size-gb.\n# --max-gpu-workers 1 avoids concurrent-GPU-transfer stalls under heavy\n# async-load pressure; CPU-side workers stay at 8.\nlmcache server \\\n  --port 5555 \\\n  --http-port 8080 \\\n  --l1-size-gb 512 \\\n  --l1-init-size-gb 20 \\\n  --max-gpu-workers 1 \\\n  --max-cpu-workers 8 \\\n  --chunk-size 1024 \\\n  --l1-align-bytes 16384 \\\n  --eviction-trigger-watermark 0.85 \\\n  --eviction-ratio 0.10 \\\n  --eviction-policy LRU \\\n  --supported-transfer-mode lmcache_driven\n"
      }
    },
    "iaxl": {
      "display_name": "IAXL KVShrink",
      "label": "IAXL",
      "order": 5,
      "requires_opt_in": true,
      "brands": [
        "Intel",
        "NVIDIA"
      ],
      "hardware": {
        "xpu": "unsupported",
        "gb200": "unsupported",
        "gb300": "unsupported",
        "dgx_station_gb300": "unsupported",
        "dgx_spark_gb10": "unsupported",
        "vr200": "unsupported"
      },
      "strategies": [
        "single_node_tp"
      ],
      "docker": false,
      "description": "intel-accel-for-llm's KVShrinkConnector offloads KV cache to a compressed host-DRAM store on the Intel Xeon host. Intel QAT compresses and decompresses the KV blocks by default, so setvars.sh needs at least TP_SIZE QAT devices (in-tree QAT driver with qatlib, and POLICY=0 in /etc/sysconfig/qat so the process sees every device it assigns) and stops the launch without them. To compress on IAA instead, export IAXL_QAT_ZIP_ENABLE=0 IAXL_IAA_ZIP_ENABLE=1 before the serve block; to spread fewer QAT devices over the ranks, export KVSHRINK_QAT_DEVICES yourself (comma-separated device indices per rank, | between ranks). Prefix caching is turned off so prefix hits come from KVShrink.",
      "docs_url": "https://github.com/intel/intel-accel-for-llm#kvshrink-vllm-example",
      "env": {
        "TP_SIZE": "{tp_size}"
      },
      "setup": "# setvars.sh sizes QAT/IAA/DSA and CPU per rank for TP_SIZE; vLLM wants libtcmalloc + libiomp5 preloaded\nexport IAXL_LD_PRELOAD=\"/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:$(python3 -c 'import sys; print(sys.prefix)')/lib/libiomp5.so\"\ncd \"$HOME/intel-accel-for-llm\" && source setvars.sh && export LD_PRELOAD=\"$IAXL_LD_PRELOAD${LD_PRELOAD:+:$LD_PRELOAD}\"\n",
      "args": [
        "--kv-transfer-config",
        "{\"kv_connector\":\"KVShrinkConnector\",\"kv_connector_module_path\":\"kvshrink.kvshrink_connector\",\"kv_role\":\"kv_both\"}",
        "--no-enable-prefix-caching"
      ],
      "hardware_overrides": {
        "cpu": {
          "description": "intel-accel-for-llm's KVShrinkConnector offloads KV cache to a compressed DRAM store on this Xeon, beside vLLM CPU inference (a DEVICE=cpu build has no DSA path). Intel QAT compresses and decompresses the KV blocks by default, so setvars.sh needs at least TP_SIZE QAT devices (in-tree QAT driver with qatlib, and POLICY=0 in /etc/sysconfig/qat so the process sees every device it assigns) and stops the launch without them. To compress on IAA instead, export IAXL_QAT_ZIP_ENABLE=0 IAXL_IAA_ZIP_ENABLE=1 before the serve block; to spread fewer QAT devices over the ranks, export KVSHRINK_QAT_DEVICES yourself (comma-separated device indices per rank, | between ranks). Each TP rank keeps a 16 GB compressed pool (IAXL_DDR_POOL_SIZE_GB), well below IAXL's default because vLLM's CPU engine already reserves most of its NUMA node's memory for its own KV cache; raise it if RAM allows. Prefix caching is turned off so prefix hits come from KVShrink. setvars.sh binds vLLM and IAXL to the same CPUs by default; for best performance export disjoint per-rank CPU lists on one NUMA node (VLLM_CPU_OMP_THREADS_BIND / KVSHRINK_CPU_OMP_THREADS_BIND) before the serve block — see the CPU inference guide.",
          "docs_url": "https://github.com/intel/intel-accel-for-llm/blob/main/doc/usage/vllm-cpu-inference.en.md",
          "install": [
            {
              "note": "IAXL build prerequisites (Debian/Ubuntu, as in IAXL's docker/Dockerfile.dev): the native extension builds QPL and qatlib from source",
              "command": "sudo apt-get install -y build-essential cmake pkg-config autoconf automake autoconf-archive libtool nasm yasm libboost-all-dev libudev-dev liblz4-dev libsqlite3-dev zlib1g-dev uuid-dev libjson-c-dev libssl-dev libnuma-dev libtcmalloc-minimal4 iproute2"
            },
            {
              "note": "IAXL source checkout, pinned to the commit this option was tested with (no PyPI wheel or published vLLM image includes IAXL yet); the serve command sources setvars.sh from here",
              "command": "git clone https://github.com/intel/intel-accel-for-llm.git \"$HOME/intel-accel-for-llm\" && git -C \"$HOME/intel-accel-for-llm\" checkout -q 28f2d3e1a15721763912d0dd7ec5d4b1f76bc959"
            },
            {
              "note": "IAXL CPU build (DEVICE=cpu) against the installed torch, with qatlib for the in-tree QAT driver; requirements.txt adds the build tools and intel-openmp (libiomp5). It also installs IAXL's kvshrink vLLM plugin, which loads in every vLLM process in this environment but only acts when KVShrinkConnector is configured.",
              "command": "cd \"$HOME/intel-accel-for-llm\"\nuv pip install -r requirements.txt\nIAXL_VERSION=$(sed -n 's/^export IAXL_VERSION=${IAXL_VERSION:-\\(.*\\)}$/\\1/p' setvars.sh) DEVICE=cpu IAXL_QATLIB=intree uv pip install --no-build-isolation -e ."
            }
          ],
          "env": {
            "DEVICE": "cpu",
            "IAXL_DDR_POOL_SIZE_GB": "16"
          }
        },
        "nvidia": {
          "description": "intel-accel-for-llm's KVShrinkConnector offloads KV cache to a compressed host-DRAM store on the GPU server's Intel Xeon host (4th Gen Xeon Scalable or later) — not Grace- or Vera-based or other non-Xeon hosts. Intel QAT compresses and decompresses the KV blocks by default, so setvars.sh needs at least TP_SIZE QAT devices (in-tree QAT driver with qatlib, and POLICY=0 in /etc/sysconfig/qat so the process sees every device it assigns) and stops the launch without them. To compress on IAA instead, export IAXL_QAT_ZIP_ENABLE=0 IAXL_IAA_ZIP_ENABLE=1 before the serve block; to spread fewer QAT devices over the ranks, export KVSHRINK_QAT_DEVICES yourself (comma-separated device indices per rank, | between ranks). Each rank's pool keeps IAXL's default budget: 80% of host RAM divided across ranks, up to 80% of a NUMA node per rank. To also move KV fragments between GPU memory and host DRAM with Intel DSA through GDRCopy (no GPU SMs or CPU cores used), install the GDRCopy driver (tools/install_gdr_driver.sh), configure DSA work queues and set IAXL_DSA_GD_ENABLE=1 — on GPUs with P2P DMA support only. Prefix caching is turned off so prefix hits come from KVShrink. The build needs the CUDA toolkit installed at /usr/local/cuda.",
          "install": [
            {
              "note": "IAXL build prerequisites (Ubuntu 22.04/24.04, as in IAXL's docker/Dockerfile.dev; GDRCopy publishes its library for those releases only): the native extension builds QPL and qatlib from source; UCX needs the RDMA libraries. Needs the CUDA toolkit at /usr/local/cuda.",
              "command": "sudo apt-get install -y build-essential cmake pkg-config autoconf automake autoconf-archive libtool nasm yasm libboost-all-dev libudev-dev liblz4-dev libsqlite3-dev zlib1g-dev uuid-dev libjson-c-dev libssl-dev libnuma-dev libtcmalloc-minimal4 iproute2 rdma-core libibverbs-dev librdmacm-dev"
            },
            {
              "note": "GDRCopy 2.6-1 user-space library (gdrapi.h + libgdrapi), which the DEVICE=cuda build links even with DSA off; the gdrdrv kernel driver is only needed to enable DSA",
              "command": "GDR_UBUNTU=$(. /etc/os-release && echo \"${VERSION_ID/./_}\")\ncurl -fLo /tmp/libgdrapi.deb \"https://developer.download.nvidia.com/compute/redist/gdrcopy/CUDA%2013.3/ubuntu${GDR_UBUNTU}/x64/libgdrapi_2.6-1_amd64.Ubuntu${GDR_UBUNTU}.deb\"\nsudo apt-get install -y /tmp/libgdrapi.deb"
            },
            {
              "note": "UCX v1.21.0 in its own prefix, for NIXL (as IAXL's docker/Dockerfile.dev builds it)",
              "command": "git clone --depth 1 --branch v1.21.0 https://github.com/openucx/ucx.git \"$HOME/ucx\" && cd \"$HOME/ucx\"\n./autogen.sh\n./contrib/configure-release-mt --prefix=/usr/local/ucx --enable-shared --disable-static --disable-doxygen-doc --enable-optimizations --enable-cma --enable-devel-headers --with-cuda=/usr/local/cuda --with-verbs --with-dm\nmake -j\"$(nproc)\" && sudo make install-strip\necho /usr/local/ucx/lib | sudo tee /etc/ld.so.conf.d/ucx.conf >/dev/null && sudo ldconfig"
            },
            {
              "note": "NIXL v1.2.0 on that UCX, into /usr/local where IAXL's CMake looks for it (as IAXL's docker/Dockerfile.dev builds it); meson, ninja and pybind11 (for NIXL's Python bindings) come from pip",
              "command": "uv pip install meson ninja pybind11\ngit clone --depth 1 --branch v1.2.0 https://github.com/ai-dynamo/nixl.git \"$HOME/nixl\" && cd \"$HOME/nixl\"\nmeson setup build -Ducx_path=/usr/local/ucx -Dinstall_headers=true -Denable_plugins=UCX -Ddisable_gds_backend=true -Dbuild_docs=false --prefix=/usr/local\nninja -C build && sudo \"$(command -v ninja)\" -C build install && sudo ldconfig"
            },
            {
              "note": "IAXL source checkout, pinned to the commit this option was tested with (no PyPI wheel or published vLLM image includes IAXL yet); the serve command sources setvars.sh from here",
              "command": "git clone https://github.com/intel/intel-accel-for-llm.git \"$HOME/intel-accel-for-llm\" && git -C \"$HOME/intel-accel-for-llm\" checkout -q 28f2d3e1a15721763912d0dd7ec5d4b1f76bc959"
            },
            {
              "note": "IAXL CUDA build (DEVICE=cuda) against the installed torch, with qatlib for the in-tree QAT driver; requirements.txt adds the build tools and intel-openmp (libiomp5). It also installs IAXL's kvshrink vLLM plugin, which loads in every vLLM process in this environment but only acts when KVShrinkConnector is configured.",
              "command": "cd \"$HOME/intel-accel-for-llm\"\nuv pip install -r requirements.txt\nIAXL_VERSION=$(sed -n 's/^export IAXL_VERSION=${IAXL_VERSION:-\\(.*\\)}$/\\1/p' setvars.sh) DEVICE=cuda IAXL_QATLIB=intree uv pip install --no-build-isolation -e ."
            }
          ],
          "env": {
            "DEVICE": "cuda",
            "IAXL_DSA_GD_ENABLE": "0"
          }
        }
      }
    }
  }
}