/bin/sh -c #(nop) ARG RELEASE

/bin/sh -c #(nop) ARG LAUNCHPAD_BUILD_ARCH

/bin/sh -c #(nop) LABEL org.opencontainers.image.ref.name=ubuntu

/bin/sh -c #(nop) LABEL org.opencontainers.image.version=22.04

/bin/sh -c #(nop) ADD file:32d41b6329e8f89fa4ac92ef97c04b7cfd5e90fb74e1509c3e27d7c91195b7c7 in /

/bin/sh -c #(nop) CMD ["/bin/bash"]

LABEL maintainer=dl.mlsedevops@amd.com

ARG ROCM_VERSION=5.3

ARG AMDGPU_VERSION=5.3

ARG APT_PREF

RUN |3 ROCM_VERSION=7.0.2 AMDGPU_VERSION=7.0.2 APT_PREF=Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600 /bin/sh -c echo "$APT_PREF" > /etc/apt/preferences.d/rocm-pin-600 # buildkit

RUN |3 ROCM_VERSION=7.0.2 AMDGPU_VERSION=7.0.2 APT_PREF=Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600 /bin/sh -c apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl libnuma-dev gnupg && curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - && printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ jammy main" | tee /etc/apt/sources.list.d/rocm.list && printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu jammy main" | tee /etc/apt/sources.list.d/amdgpu.list && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends sudo libelf1 kmod file python3-dev python3-pip rocm-dev rocm-libs build-essential && apt-get clean && rm -rf /var/lib/apt/lists/* # buildkit

RUN |3 ROCM_VERSION=7.0.2 AMDGPU_VERSION=7.0.2 APT_PREF=Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600 /bin/sh -c groupadd -g 109 render # buildkit

SHELL [/bin/bash -eo pipefail -c]

ARG TARGETPLATFORM=linux/amd64

ARG TARGETOS=linux

ARG TARGETARCH=amd64

ENV DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8

RUN |3 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 /bin/bash -eo pipefail -c # Tools # Refresh apt-get update -y && apt-get install -y --no-install-recommends \ software-properties-common apt-transport-https \ ca-certificates gnupg2 lsb-release gnupg-agent \ && apt-get update -y \ && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ && apt-get update -y # Install apt-get install -y --no-install-recommends \ ca-certificates build-essential binutils bash openssl \ curl wget aria2 \ git git-lfs \ unzip xz-utils \ tzdata locales \ iproute2 iputils-ping ifstat net-tools dnsutils pciutils ipmitool \ procps sysstat htop \ tini vim jq bc tree # Update locale localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 # Update timezone rm -f /etc/localtime \ && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ && echo "Asia/Shanghai" > /etc/timezone \ && dpkg-reconfigure --frontend noninteractive tzdata # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* \ && rm -rf /var/cache/apt # buildkit

RUN |3 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 /bin/bash -eo pipefail -c # GCC # Upgrade GCC if the Ubuntu version is lower than 21.04. source /etc/os-release if (( $(echo "${VERSION_ID} >= 21.04" | bc -l) )); then echo "Skipping GCC upgrade for ${VERSION_ID}..." exit 0 fi # Install apt-get install -y --no-install-recommends \ gcc-11 g++-11 gfortran-11 gfortran # Update alternatives if [[ -f /etc/alternatives/gcov-dump ]]; then update-alternatives --remove-all gcov-dump; fi; update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-11 10 if [[ -f /etc/alternatives/lto-dump ]]; then update-alternatives --remove-all lto-dump; fi; update-alternatives --install /usr/bin/lto-dump lto-dump /usr/bin/lto-dump-11 10 if [[ -f /etc/alternatives/gcov ]]; then update-alternatives --remove-all gcov; fi; update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-11 10 if [[ -f /etc/alternatives/gcc ]]; then update-alternatives --remove-all gcc; fi; update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 if [[ -f /etc/alternatives/gcc-nm ]]; then update-alternatives --remove-all gcc-nm; fi; update-alternatives --install /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-11 10 if [[ -f /etc/alternatives/cpp ]]; then update-alternatives --remove-all cpp; fi; update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 10 if [[ -f /etc/alternatives/g++ ]]; then update-alternatives --remove-all g++; fi; update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 10 if [[ -f /etc/alternatives/gcc-ar ]]; then update-alternatives --remove-all gcc-ar; fi; update-alternatives --install /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 10 if [[ -f /etc/alternatives/gcov-tool ]]; then update-alternatives --remove-all gcov-tool; fi; update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-11 10 if [[ -f /etc/alternatives/gcc-ranlib ]]; then update-alternatives --remove-all gcc-ranlib; fi; update-alternatives --install /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 10 if [[ -f /etc/alternatives/gfortran ]]; then update-alternatives --remove-all gfortran; fi; update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 10 # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* \ && rm -rf /var/cache/apt # buildkit

RUN |3 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 /bin/bash -eo pipefail -c # C buildkit # Install apt-get install -y --no-install-recommends \ make ninja-build pkg-config ccache curl --retry 3 --retry-connrefused -fL "https://github.com/Kitware/CMake/releases/download/v3.31.7/cmake-3.31.7-linux-$(uname -m).tar.gz" | tar -zx -C /usr --strip-components 1 # Install dependencies apt-get install -y --no-install-recommends \ perl-openssl-defaults perl yasm \ zlib1g zlib1g-dev libbz2-dev libffi-dev libgdbm-dev libgdbm-compat-dev \ openssl libssl-dev libsqlite3-dev lcov libomp-dev \ libblas-dev liblapack-dev libopenblas-dev libblas3 liblapack3 libhdf5-dev \ libxml2 libxslt1-dev libgl1-mesa-glx libgmpxx4ldbl \ libncurses5-dev libreadline6-dev libsqlite3-dev \ liblzma-dev lzma lzma-dev tk-dev uuid-dev libmpdec-dev \ ffmpeg libjpeg-dev libpng-dev libtiff-dev libwebp-dev \ libnuma-dev libjemalloc-dev # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* \ && rm -rf /var/cache/apt # buildkit

ARG PYTHON_VERSION=3.12

ENV PYTHON_VERSION=3.12

RUN |4 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 PYTHON_VERSION=3.12 /bin/bash -eo pipefail -c # Python if (( $(echo "$(python3 --version | cut -d' ' -f2 | cut -d'.' -f1,2) == ${PYTHON_VERSION}" | bc -l) )); then echo "Skipping Python upgrade for ${PYTHON_VERSION}..." if [[ -z "$(ldconfig -v 2>/dev/null | grep libpython${PYTHON_VERSION})" ]]; then PYTHON_LIB_PREFIX=$(python3 -c "import sys; print(sys.base_prefix);") echo "${PYTHON_LIB_PREFIX}/lib" >> /etc/ld.so.conf.d/python3.conf echo "${PYTHON_LIB_PREFIX}/lib64" >> /etc/ld.so.conf.d/python3.conf EXPORT_PYTHON_LIB="export LD_LIBRARY_PATH=${PYTHON_LIB_PREFIX}/lib:${PYTHON_LIB_PREFIX}/lib64:\${LD_LIBRARY_PATH}" echo "${EXPORT_PYTHON_LIB}" >> /etc/profile echo "${EXPORT_PYTHON_LIB}" >> ~/.bashrc fi exit 0 fi # Add deadsnakes PPA for Python versions for i in 1 2 3; do add-apt-repository -y ppa:deadsnakes/ppa && break || { echo "Attempt $i failed, retrying in 5s..."; sleep 5; } done apt-get update -y # Install apt-get install -y --no-install-recommends \ python${PYTHON_VERSION} \ python${PYTHON_VERSION}-dev \ python${PYTHON_VERSION}-venv \ python${PYTHON_VERSION}-lib2to3 \ python${PYTHON_VERSION}-gdbm \ python${PYTHON_VERSION}-tk \ libibverbs-dev # Update alternatives if [[ -f /etc/alternatives/python3 ]]; then update-alternatives --remove-all python3; fi; update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 if [[ -f /etc/alternatives/python ]]; then update-alternatives --remove-all python; fi; update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 curl -sS "https://bootstrap.pypa.io/get-pip.py" | python${PYTHON_VERSION} if [[ -f /etc/alternatives/2to3 ]]; then update-alternatives --remove-all 2to3; fi; update-alternatives --install /usr/bin/2to3 2to3 /usr/bin/2to3${PYTHON_VERSION} 1 || true if [[ -f /etc/alternatives/pydoc3 ]]; then update-alternatives --remove-all pydoc3; fi; update-alternatives --install /usr/bin/pydoc3 pydoc3 /usr/bin/pydoc${PYTHON_VERSION} 1 || true if [[ -f /etc/alternatives/idle3 ]]; then update-alternatives --remove-all idle3; fi; update-alternatives --install /usr/bin/idle3 idle3 /usr/bin/idle${PYTHON_VERSION} 1 || true if [[ -f /etc/alternatives/python3-config ]]; then update-alternatives --remove-all python3-config; fi; update-alternatives --install /usr/bin/python3-config python3-config /usr/bin/python${PYTHON_VERSION}-config 1 || true # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* \ && rm -rf /var/cache/apt # buildkit

ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_ROOT_USER_ACTION=ignore PIPX_HOME=/root/.local/share/pipx PIPX_LOCAL_VENVS=/root/.local/share/pipx/venvs UV_NO_CACHE=1 UV_HTTP_TIMEOUT=500 UV_INDEX_STRATEGY=unsafe-best-match

RUN |4 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 PYTHON_VERSION=3.12 /bin/bash -eo pipefail -c # Buildkit cat <<EOT >/tmp/requirements.txt build cmake<4 ninja<1.11 setuptools<80 setuptools-scm packaging<25 wheel pybind11<3 Cython psutil pipx uv EOT pip install -r /tmp/requirements.txt # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* # buildkit

ARG ROCM_VERSION=7.0.2

ARG ROCM_ARCHS

ENV ROCM_HOME=/opt/rocm ROCM_VERSION=7.0.2 ROCM_ARCHS=

RUN |6 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 PYTHON_VERSION=3.12 ROCM_VERSION=7.0.2 ROCM_ARCHS= /bin/bash -eo pipefail -c # Toolkit uv pip install --system amdsmi==${ROCM_VERSION} # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* # buildkit

SHELL [/bin/bash -eo pipefail -c]

ARG TARGETPLATFORM=linux/amd64

ARG TARGETOS=linux

ARG TARGETARCH=amd64

ENV UV_SYSTEM_PYTHON=1 UV_PRERELEASE=allow

ARG VLLM_TRITON_VERSION=3.5.0

ARG VLLM_TORCH_SOURCE=radeon

ARG VLLM_TORCH_VERSION=2.8.0

ARG VLLM_TORCH_ROCM_VERSION=7.0.2

ENV VLLM_TRITON_VERSION=3.5.0 VLLM_TORCH_VERSION=2.8.0 VLLM_TORCH_ROCM_VERSION=7.0.2

RUN |7 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 VLLM_TRITON_VERSION=3.5.0 VLLM_TORCH_SOURCE=radeon VLLM_TORCH_VERSION=2.8.0 VLLM_TORCH_ROCM_VERSION=7.0.2 /bin/bash -eo pipefail -c # Torch # Install from radeon source. if [[ "${VLLM_TORCH_SOURCE}" == "radeon" ]]; then IFS="." read -r PYTHON_MAJOR PYTHON_MINOR <<< "${PYTHON_VERSION}" WHEEL_SUFFIX="cp${PYTHON_MAJOR}${PYTHON_MINOR}-linux_$(uname -m).whl" TRITON_URL="$(curl -sf "https://repo.radeon.com/rocm/manylinux/rocm-rel-${ROCM_VERSION}/" | grep "triton" | grep -v lw | grep -v dev | grep "${WHEEL_SUFFIX}" | tail -n 1 | sed -n 's/.*href="\([^"]*\)".*/\1/p')" TORCH_URL="$(curl -sf "https://repo.radeon.com/rocm/manylinux/rocm-rel-${ROCM_VERSION}/" | grep "torch-${VLLM_TORCH_VERSION}" | grep -v lw | grep -v dev | grep "${WHEEL_SUFFIX}" | tail -n 1 | sed -n 's/.*href="\([^"]*\)".*/\1/p')" TORCHVISION_URL="$(curl -sf "https://repo.radeon.com/rocm/manylinux/rocm-rel-${ROCM_VERSION}/" | grep "torchvision" | grep -v lw | grep -v dev | grep "${WHEEL_SUFFIX}" | tail -n 1 | sed -n 's/.*href="\([^"]*\)".*/\1/p')" TORCHAUDIO_URL="$(curl -sf "https://repo.radeon.com/rocm/manylinux/rocm-rel-${ROCM_VERSION}/" | grep "torchaudio-${VLLM_TORCH_VERSION}" | grep -v lw | grep -v dev | grep "${WHEEL_SUFFIX}" | tail -n 1 | sed -n 's/.*href="\([^"]*\)".*/\1/p')" if [[ -z "${TORCH_URL}" || -z "${TORCHVISION_URL}" || -z "${TORCHAUDIO_URL}" ]]; then echo "Failed to get ROCM PyTorch wheels for version ${VLLM_TORCH_VERSION} from radeon source." exit 1 fi uv pip install "https://repo.radeon.com/rocm/manylinux/rocm-rel-${ROCM_VERSION}/${TRITON_URL}" uv pip install "https://repo.radeon.com/rocm/manylinux/rocm-rel-${ROCM_VERSION}/${TORCH_URL}" uv pip install "https://repo.radeon.com/rocm/manylinux/rocm-rel-${ROCM_VERSION}/${TORCHVISION_URL}" uv pip install "https://repo.radeon.com/rocm/manylinux/rocm-rel-${ROCM_VERSION}/${TORCHAUDIO_URL}" uv pip install \ numpy scipy exit 0 fi # Install from pytorch source. cat <<EOT >/tmp/requirements.txt triton=${VLLM_TRITON_VERSION} torch==${VLLM_TORCH_VERSION} torchvision torchaudio EOT IFS="." read -r ROCM_MAJOR ROCM_MINOR ROCM_PATCH <<< "${VLLM_TORCH_ROCM_VERSION}" if [[ "${TARGETARCH}" == "amd64" ]]; then uv pip install --index-url https://download.pytorch.org/whl/rocm${ROCM_MAJOR}.${ROCM_MINOR} \ -r /tmp/requirements.txt else uv pip install --extra-index-url https://download.pytorch.org/whl/cpu/ \ -r /tmp/requirements.txt fi uv pip install \ numpy scipy # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* # buildkit

ARG CMAKE_MAX_JOBS

ARG VLLM_VERSION=0.11.0

ENV VLLM_VERSION=0.11.0

RUN |9 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 VLLM_TRITON_VERSION=3.5.0 VLLM_TORCH_SOURCE=radeon VLLM_TORCH_VERSION=2.8.0 VLLM_TORCH_ROCM_VERSION=7.0.2 CMAKE_MAX_JOBS= VLLM_VERSION=0.11.0 /bin/bash -eo pipefail -c # vLLM IFS="." read -r ROCM_MAJOR ROCM_MINOR ROCM_PATCH <<< "${VLLM_TORCH_ROCM_VERSION}" CMAKE_MAX_JOBS="${CMAKE_MAX_JOBS}" if [[ -z "${CMAKE_MAX_JOBS}" ]]; then CMAKE_MAX_JOBS="$(( $(nproc) / 2 ))" fi if (( $(echo "${CMAKE_MAX_JOBS} > 8" | bc -l) )); then CMAKE_MAX_JOBS="8" fi VL_ROCM_ARCHS="$(echo ${ROCM_ARCHS} | sed -e 's/;gfx1[0-9]\{3\}//g')" if [[ -z "${VL_ROCM_ARCHS}" ]]; then if (( $(echo "${ROCM_MAJOR}.${ROCM_MINOR} <= 6.2" | bc -l) )); then VL_ROCM_ARCHS="gfx900;gfx906;gfx908;gfx90a;gfx941;gfx942;gfx1030;gfx1100;gfx1101" elif (( $(echo "${ROCM_MAJOR}.${ROCM_MINOR} < 7.0" | bc -l) )); then VL_ROCM_ARCHS="gfx900;gfx906;gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201" else VL_ROCM_ARCHS="gfx900;gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201;gfx1151" fi fi export MAX_JOBS="${CMAKE_MAX_JOBS}" export COMPILE_CUSTOM_KERNELS=1 export PYTORCH_ROCM_ARCH="${VL_ROCM_ARCHS}" # Install EXTRA_INDEX_URL="https://download.pytorch.org/whl/rocm${ROCM_MAJOR}.${ROCM_MINOR}" if [[ "${TARGETARCH}" == "arm64" ]]; then EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu/" fi git -C /tmp clone --recursive --shallow-submodules \ --depth 1 --branch v${VLLM_VERSION} --single-branch \ https://github.com/vllm-project/vllm.git vllm-${VLLM_VERSION} VLLM_TARGET_DEVICE="rocm" \ uv pip install --no-build-isolation --verbose --extra-index-url ${EXTRA_INDEX_URL} \ /tmp/vllm-${VLLM_VERSION} # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* \ && ccache --clear --clean # buildkit

RUN |9 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 VLLM_TRITON_VERSION=3.5.0 VLLM_TORCH_SOURCE=radeon VLLM_TORCH_VERSION=2.8.0 VLLM_TORCH_ROCM_VERSION=7.0.2 CMAKE_MAX_JOBS= VLLM_VERSION=0.11.0 /bin/bash -eo pipefail -c # FlashAttention if [[ ! -d /flashattention/workspace ]]; then echo "Skipping FlashAttention installation for ${TARGETARCH}..." exit 0 fi # Install uv pip install --no-build-isolation --no-deps \ /flashattention/workspace/*.whl # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* # buildkit

RUN |9 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 VLLM_TRITON_VERSION=3.5.0 VLLM_TORCH_SOURCE=radeon VLLM_TORCH_VERSION=2.8.0 VLLM_TORCH_ROCM_VERSION=7.0.2 CMAKE_MAX_JOBS= VLLM_VERSION=0.11.0 /bin/bash -eo pipefail -c # AITER if [[ ! -d /aiter/workspace ]]; then echo "Skipping AITER installation for ${TARGETARCH}..." exit 0 fi # Install uv pip install --no-build-isolation \ /aiter/workspace/*.whl # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* # buildkit

ARG VLLM_LMCACHE_VERSION=0.3.9post1

ENV VLLM_LMCACHE_VERSION=0.3.9post1

RUN |10 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 VLLM_TRITON_VERSION=3.5.0 VLLM_TORCH_SOURCE=radeon VLLM_TORCH_VERSION=2.8.0 VLLM_TORCH_ROCM_VERSION=7.0.2 CMAKE_MAX_JOBS= VLLM_VERSION=0.11.0 VLLM_LMCACHE_VERSION=0.3.9post1 /bin/bash -eo pipefail -c # LMCache # Ref https://github.com/LMCache/LMCache/blob/5afe9688b3519074b9915e7b3acf871328250150/docs/source/getting_started/installation.rst?plain=1#L184-L237. IFS="." read -r ROCM_MAJOR ROCM_MINOR ROCM_PATCH <<< "${VLLM_TORCH_ROCM_VERSION}" CMAKE_MAX_JOBS="${CMAKE_MAX_JOBS}" if [[ -z "${CMAKE_MAX_JOBS}" ]]; then CMAKE_MAX_JOBS="$(( $(nproc) / 2 ))" fi if (( $(echo "${CMAKE_MAX_JOBS} > 8" | bc -l) )); then CMAKE_MAX_JOBS="8" fi LC_ROCM_ARCHS="$(echo ${ROCM_ARCHS} | sed -e 's/;gfx1[0-9]\{3\}//g')" if [[ -z "${LC_ROCM_ARCHS}" ]]; then if (( $(echo "${ROCM_MAJOR}.${ROCM_MINOR} <= 6.2" | bc -l) )); then LC_ROCM_ARCHS="gfx900;gfx906;gfx908;gfx90a;gfx941;gfx942;gfx1030;gfx1100;gfx1101" elif (( $(echo "${ROCM_MAJOR}.${ROCM_MINOR} < 7.0" | bc -l) )); then LC_ROCM_ARCHS="gfx900;gfx906;gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201" else LC_ROCM_ARCHS="gfx900;gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201;gfx1151" fi fi export MAX_JOBS="${CMAKE_MAX_JOBS}" export PYTORCH_ROCM_ARCH="${LC_ROCM_ARCHS}" export TORCH_DONT_CHECK_COMPILER_ABI=1 export CXX=hipcc export BUILD_WITH_HIP=1 cat <<EOT >/tmp/requirements.txt aiofile==3.9.0 aiofiles==25.1.0 awscrt==0.28.2 cufile-python==0.2.0 nvtx==0.2.13 sortedcontainers==2.4.0 EOT uv pip install \ -r /tmp/requirements.txt # Install LMCache git -C /tmp clone --recursive --shallow-submodules \ --depth 1 --branch v${VLLM_LMCACHE_VERSION} --single-branch \ https://github.com/LMCache/LMCache.git lmcache if [[ "${TARGETARCH}" == "arm64" ]]; then sed -i "s/^infinistore$/infinistore; platform_machine == 'x86_64'/" /tmp/lmcache/requirements/common.txt fi uv pip install --no-build-isolation --no-deps --verbose \ /tmp/lmcache # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* \ && ccache --clear --clean # buildkit

RUN |10 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 VLLM_TRITON_VERSION=3.5.0 VLLM_TORCH_SOURCE=radeon VLLM_TORCH_VERSION=2.8.0 VLLM_TORCH_ROCM_VERSION=7.0.2 CMAKE_MAX_JOBS= VLLM_VERSION=0.11.0 VLLM_LMCACHE_VERSION=0.3.9post1 /bin/bash -eo pipefail -c # Ray # Install Ray Client and Default RAY_VERSION=$(pip show ray | grep Version: | cut -d' ' -f 2) cat <<EOT >/tmp/requirements.txt ray[client]==${RAY_VERSION} ray[default]==${RAY_VERSION} EOT uv pip install \ -r /tmp/requirements.txt # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* # buildkit

RUN |10 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 VLLM_TRITON_VERSION=3.5.0 VLLM_TORCH_SOURCE=radeon VLLM_TORCH_VERSION=2.8.0 VLLM_TORCH_ROCM_VERSION=7.0.2 CMAKE_MAX_JOBS= VLLM_VERSION=0.11.0 VLLM_LMCACHE_VERSION=0.3.9post1 /bin/bash -eo pipefail -c # Postprocess # Review uv pip tree \ --package vllm \ --package flash_attn \ --package triton \ --package aiter \ --package torch \ --package lmcache # buildkit

ENV RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1 RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES=1 TOKENIZERS_PARALLELISM=false SAFETENSORS_FAST_GPU=1 HIP_FORCE_DEV_KERNARG=1

WORKDIR /

ENTRYPOINT ["tini" "--"]

SHELL [/bin/bash -eo pipefail -c]

ARG TARGETPLATFORM=linux/amd64

ARG TARGETOS=linux

ARG TARGETARCH=amd64

ENV UV_SYSTEM_PYTHON=1 UV_PRERELEASE=allow

RUN |3 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 /bin/bash -eo pipefail -c # Tools # Install apt-get update -y && apt-get install -y --no-install-recommends \ libgrpc-dev \ libgrpc++-dev \ libprotobuf-dev \ protobuf-compiler \ protobuf-compiler-grpc \ libtool \ autoconf \ librdmacm-dev \ rdma-core \ rdmacm-utils \ infiniband-diags \ ibverbs-utils \ libibverbs-dev \ openssh-server \ openmpi-bin \ openmpi-common \ libopenmpi-dev \ zip \ unzip # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* \ && rm -rf /var/cache/apt # buildkit

SHELL [/bin/bash -eo pipefail -c]

ARG TARGETPLATFORM=linux/amd64

ARG TARGETOS=linux

ARG TARGETARCH=amd64

ENV UV_SYSTEM_PYTHON=1 UV_PRERELEASE=allow

ARG CMAKE_MAX_JOBS

ARG SGLANG_VERSION=0.5.4.post3

ENV SGLANG_VERSION=0.5.4.post3

RUN |5 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 CMAKE_MAX_JOBS= SGLANG_VERSION=0.5.4.post3 /bin/bash -eo pipefail -c # SGLang IFS="." read -r ROCM_MAJOR ROCM_MINOR ROCM_PATCH <<< "${VLLM_TORCH_ROCM_VERSION}" CMAKE_MAX_JOBS="${CMAKE_MAX_JOBS}" if [[ -z "${CMAKE_MAX_JOBS}" ]]; then CMAKE_MAX_JOBS="$(( $(nproc) / 2 ))" fi if (( $(echo "${CMAKE_MAX_JOBS} > 8" | bc -l) )); then CMAKE_MAX_JOBS="8" fi SG_ROCM_ARCHS="$(echo ${ROCM_ARCHS} | sed -e 's/;gfx1[0-9]\{3\}//g')" if [[ -z "${SG_ROCM_ARCHS}" ]]; then if (( $(echo "${ROCM_MAJOR}.${ROCM_MINOR} <= 6.2" | bc -l) )); then SG_ROCM_ARCHS="gfx900;gfx906;gfx908;gfx90a;gfx941;gfx942;gfx1030;gfx1100;gfx1101" elif (( $(echo "${ROCM_MAJOR}.${ROCM_MINOR} < 7.0" | bc -l) )); then SG_ROCM_ARCHS="gfx900;gfx906;gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201" else SG_ROCM_ARCHS="gfx900;gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201;gfx1151" fi fi export MAX_JOBS="${CMAKE_MAX_JOBS}" export PYTORCH_ROCM_ARCH="${SG_ROCM_ARCHS}" export AMDGPU_TARGET="gfx950" echo "Building SGLang with the following environment variables:" env # Download SGLang git -C /tmp clone --recursive --shallow-submodules \ --depth 1 --branch v${SGLANG_VERSION} --single-branch \ https://github.com/sgl-project/sglang.git sglang-${SGLANG_VERSION} # Install SGLang # Copy config files to support MI300X in virtualized environments (MI300X_VF). # Symlinks will not be created in image build. # See https://github.com/sgl-project/sglang/blob/09938e1f82334a98c124ae3352d893480963775f/docker/rocm.Dockerfile#L197C2-L200. find /tmp/sglang-${SGLANG_VERSION}/python/sglang/srt/layers/quantization/configs/ \ /tmp/sglang-${SGLANG_VERSION}/python/sglang/srt/layers/moe/fused_moe_triton/configs/ \ -type f -name '*MI300X*' | xargs -I {} sh -c 'vf_config=$(echo "$1" | sed "s/MI300X/MI300X_VF/"); cp "$1" "$vf_config"' -- {} pushd /tmp/sglang-${SGLANG_VERSION}/python \ && if [[ -f pyproject_other.toml ]]; then mv pyproject_other.toml pyproject.toml; fi \ && uv pip install --verbose --no-deps .[all_hip] # Install SGLang Kernel pushd /tmp/sglang-${SGLANG_VERSION}/sgl-kernel \ && if [[ -f pyproject_rocm.toml ]]; then mv pyproject_rocm.toml pyproject.toml; fi \ && python setup_rocm.py install # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* \ && ccache --clear --clean # buildkit

RUN |5 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 CMAKE_MAX_JOBS= SGLANG_VERSION=0.5.4.post3 /bin/bash -eo pipefail -c # SGlang Router # Install Rust curl --retry 3 --retry-connrefused --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y export PATH="/root/.cargo/bin:${PATH}" \ && rustc --version \ && cargo --version # Install build tools uv pip install \ setuptools-rust maturin # Install SGLang Router git -C /tmp clone --recursive --shallow-submodules \ --depth 1 --branch v${SGLANG_VERSION} --single-branch \ https://github.com/sgl-project/sglang.git sglang pushd /tmp/sglang/sgl-router \ && ulimit -n 65536 && maturin build --release --features vendored-openssl --out dist \ && tree -hs /tmp/sglang/sgl-router/dist \ && uv pip install --force-reinstall /tmp/sglang/sgl-router/dist/*.whl # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* \ && rm -rf /root/.cargo \ && rm -rf /root/.rustup \ && sed -i '$d' /root/.profile \ && sed -i '$d' /root/.bashrc \ && ccache --clear --clean # buildkit

RUN |5 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 CMAKE_MAX_JOBS= SGLANG_VERSION=0.5.4.post3 /bin/bash -eo pipefail -c # MoonCake if [[ ! -d /mooncake/workspace ]]; then echo "Skipping MoonCake installation for ${TARGETARCH}..." exit 0 fi # Install uv pip install --no-build-isolation \ /mooncake/workspace/*.whl # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* # buildkit

RUN |5 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 CMAKE_MAX_JOBS= SGLANG_VERSION=0.5.4.post3 /bin/bash -eo pipefail -c # Fast-Hadamard-Transform uv pip install \ wheel==0.45.1 CMAKE_MAX_JOBS="${CMAKE_MAX_JOBS}" if [[ -z "${CMAKE_MAX_JOBS}" ]]; then CMAKE_MAX_JOBS="$(( $(nproc) / 2 ))" fi if (( $(echo "${CMAKE_MAX_JOBS} > 8" | bc -l) )); then CMAKE_MAX_JOBS="8" fi export MAX_JOBS="${CMAKE_MAX_JOBS}" export FAST_HADAMARD_TRANSFORM_SKIP_CUDA_BUILD="TRUE" echo "Building Fast-Hadamard-Transform with the following environment variables:" env # Install Fast-Hadamard-Transform git -C /tmp clone --recursive --shallow-submodules \ --branch rocm --single-branch \ https://github.com/jeffdaily/fast-hadamard-transform.git fast-hadamard-transform pushd /tmp/fast-hadamard-transform \ && python setup.py install # Cleanup rm -rf /var/tmp/* \ && rm -rf /tmp/* \ && ccache --clear --clean # buildkit

RUN |5 TARGETPLATFORM=linux/amd64 TARGETOS=linux TARGETARCH=amd64 CMAKE_MAX_JOBS= SGLANG_VERSION=0.5.4.post3 /bin/bash -eo pipefail -c # Postprocess # Review uv pip tree \ --package sglang \ --package sglang-router \ --package sgl-kernel \ --package flashinfer-python \ --package triton \ --package vllm \ --package torch \ --package deep-ep # buildkit

ENV HIP_FORCE_DEV_KERNARG=1 HSA_NO_SCRATCH_RECLAIM=1 SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 SGLANG_INT4_WEIGHT=0 SGLANG_MOE_PADDING=1 SGLANG_ROCM_DISABLE_LINEARQUANT=0 SGLANG_ROCM_FUSED_DECODE_MLA=1 SGLANG_SET_CPU_AFFINITY=1 SGLANG_USE_AITER=1 SGLANG_USE_ROCM700A=1 NCCL_MIN_NCHANNELS=112 VLLM_FP8_PADDING=1 VLLM_FP8_ACT_PADDING=1 VLLM_FP8_WEIGHT_PADDING=1 VLLM_FP8_REDUCE_CONV=1 TORCHINDUCTOR_MAX_AUTOTUNE=1 TORCHINDUCTOR_MAX_AUTOTUNE_POINTWISE=1

WORKDIR /

ENTRYPOINT ["tini" "--"]