From b3ff0889e638f991687f82f180645d965c77ff2b Mon Sep 17 00:00:00 2001
From: axsapronov <a@sapronov.me>
Date: Fri, 10 Jul 2026 10:05:55 +0700
Subject: [PATCH 1/2] feat: add gguff plugin to vllm runtime

---
 pack/cann/Dockerfile.vllm | 17 +++++++++++++++++
 pack/cuda/Dockerfile.vllm | 17 +++++++++++++++++
 pack/dtk/Dockerfile.vllm  | 17 +++++++++++++++++
 pack/hggc/Dockerfile.vllm | 17 +++++++++++++++++
 pack/maca/Dockerfile.vllm | 17 +++++++++++++++++
 pack/rocm/Dockerfile.vllm | 17 +++++++++++++++++
 6 files changed, 102 insertions(+)

diff --git a/pack/cann/Dockerfile.vllm b/pack/cann/Dockerfile.vllm
index 176c258..ada6a8d 100644
--- a/pack/cann/Dockerfile.vllm
+++ b/pack/cann/Dockerfile.vllm
@@ -9,6 +9,7 @@ ARG VLLM_LMCACHE_VERSION=0.4.3
 ARG VLLM_LMCACHE_ASCEND_VERSION=${VLLM_LMCACHE_VERSION}
 ARG VLLM_MOONCAKE_VERSION=0.3.11
 ARG VLLM_OMNI_COMMIT=4a24a517
+ARG VLLM_GGUF_PLUGIN_VERSION=0.0.3
 
 FROM ${CANN_BASE_IMAGE} AS vllm-build
 SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
@@ -255,6 +256,7 @@ SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
+ARG VLLM_GGUF_PLUGIN_VERSION
 
 ## Install Ray
 
@@ -403,6 +405,7 @@ EOT
     # Review
     uv pip tree \
         --package vllm \
+        --package vllm-gguf-plugin \
         --package vllm-ascend \
         --package vllm-omni \
         --package torch \
@@ -429,6 +432,20 @@ RUN --mount=type=bind,target=/workspace,rw <<EOF
     fi
 EOF
 
+## Install GGUF
+
+RUN <<EOF
+    # GGUF
+    # https://docs.vllm.ai/en/v0.24.0/features/quantization/gguf/
+
+    uv pip install \
+        vllm-gguf-plugin==${VLLM_GGUF_PLUGIN_VERSION}
+
+    # Cleanup
+    rm -rf /var/tmp/* \
+        && rm -rf /tmp/*
+EOF
+
 ## Dependencies
 
 RUN <<EOF
diff --git a/pack/cuda/Dockerfile.vllm b/pack/cuda/Dockerfile.vllm
index 558196c..a9b2172 100644
--- a/pack/cuda/Dockerfile.vllm
+++ b/pack/cuda/Dockerfile.vllm
@@ -7,6 +7,7 @@ ARG VLLM_VERSION=0.24.0
 ARG VLLM_TORCH_VERSION=2.11.0
 ARG VLLM_TORCH_CUDA_VERSION=${CUDA_VERSION}
 ARG VLLM_OMNI_COMMIT=a560ed1
+ARG VLLM_GGUF_PLUGIN_VERSION=0.0.3
 
 FROM ${VLLM_BASE_IMAGE} AS vllm-build
 SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
@@ -194,6 +195,7 @@ SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
+ARG VLLM_GGUF_PLUGIN_VERSION
 
 ## Install Ray
 
@@ -284,6 +286,7 @@ RUN <<EOF
     # Review
     uv pip tree \
         --package vllm \
+        --package vllm-gguf-plugin \
         --package vllm-omni \
         --package flashinfer-python \
         --package torch \
@@ -309,6 +312,20 @@ RUN --mount=type=bind,target=/workspace,rw <<EOF
     fi
 EOF
 
+## Install GGUF
+
+RUN <<EOF
+    # GGUF
+    # https://docs.vllm.ai/en/v0.24.0/features/quantization/gguf/
+
+    uv pip install \
+        vllm-gguf-plugin==${VLLM_GGUF_PLUGIN_VERSION}
+
+    # Cleanup
+    rm -rf /var/tmp/* \
+        && rm -rf /tmp/*
+EOF
+
 ## Dependencies
 
 RUN <<EOF
diff --git a/pack/dtk/Dockerfile.vllm b/pack/dtk/Dockerfile.vllm
index 4c7a096..52b167a 100644
--- a/pack/dtk/Dockerfile.vllm
+++ b/pack/dtk/Dockerfile.vllm
@@ -31,6 +31,7 @@ ARG DTK_ARCHS
 ARG VLLM_VERSION=0.18.1
 ARG VLLM_TORCH_VERSION=2.10.0
 ARG VLLM_LMCACHE_VERSION=0.5.0
+ARG VLLM_GGUF_PLUGIN_VERSION=0.0.3
 ARG VLLM_BASE_IMAGE=gpustack/runner:dtk${DTK_VERSION}${DTK_VERSION_EXTRA}-vllm${VLLM_VERSION}-python${PYTHON_VERSION}-vanilla
 
 # Stage Bake Runtime
@@ -276,6 +277,7 @@ SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
+ARG VLLM_GGUF_PLUGIN_VERSION
 
 ENV UV_SYSTEM_PYTHON=1 \
     UV_PRERELEASE=allow \
@@ -323,6 +325,20 @@ RUN <<EOF
         && rm -rf /tmp/*
 EOF
 
+## Install GGUF
+
+RUN <<EOF
+    # GGUF
+    # https://docs.vllm.ai/en/v0.24.0/features/quantization/gguf/
+
+    uv pip install \
+        vllm-gguf-plugin==${VLLM_GGUF_PLUGIN_VERSION}
+
+    # Cleanup
+    rm -rf /var/tmp/* \
+        && rm -rf /tmp/*
+EOF
+
 ## Postprocess
 
 RUN <<EOF
@@ -331,6 +347,7 @@ RUN <<EOF
     # Review
     uv pip tree \
         --package vllm \
+        --package vllm-gguf-plugin \
         --package torch \
         --package lmcache
 EOF
diff --git a/pack/hggc/Dockerfile.vllm b/pack/hggc/Dockerfile.vllm
index 5a0a8d9..b6b2a88 100644
--- a/pack/hggc/Dockerfile.vllm
+++ b/pack/hggc/Dockerfile.vllm
@@ -27,6 +27,7 @@ ARG HGGC_ARCHS
 ARG VLLM_VERSION=0.18.0
 ARG VLLM_TORCH_VERSION=2.9.0
 ARG VLLM_BASE_IMAGE=gpustack/runner:hggc${HGGC_VERSION}${HGGC_VERSION_EXTRA}-vllm${VLLM_VERSION}-python${PYTHON_VERSION}-vanilla
+ARG VLLM_GGUF_PLUGIN_VERSION=0.0.3
 
 # Stage Bake Runtime
 #
@@ -281,6 +282,7 @@ SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
+ARG VLLM_GGUF_PLUGIN_VERSION
 
 ENV UV_SYSTEM_PYTHON=1 \
     UV_PRERELEASE=allow \
@@ -302,6 +304,20 @@ ENV VLLM_VERSION=${VLLM_VERSION}
 
 ## Nothing to do, as the vLLM has installed in global.
 
+## Install GGUF
+
+RUN <<EOF
+    # GGUF
+    # https://docs.vllm.ai/en/v0.24.0/features/quantization/gguf/
+
+    uv pip install \
+        vllm-gguf-plugin==${VLLM_GGUF_PLUGIN_VERSION}
+
+    # Cleanup
+    rm -rf /var/tmp/* \
+        && rm -rf /tmp/*
+EOF
+
 ## Postprocess
 
 RUN <<EOF
@@ -310,6 +326,7 @@ RUN <<EOF
     # Review
     uv pip tree \
         --package vllm \
+        --package vllm-gguf-plugin \
         --package flash-attn \
         --package torch
 EOF
diff --git a/pack/maca/Dockerfile.vllm b/pack/maca/Dockerfile.vllm
index 2cd8f9c..c900963 100644
--- a/pack/maca/Dockerfile.vllm
+++ b/pack/maca/Dockerfile.vllm
@@ -27,6 +27,7 @@ ARG MACA_ARCHS
 ARG VLLM_VERSION=0.21.0
 ARG VLLM_TORCH_VERSION=2.8.0
 ARG VLLM_BASE_IMAGE=gpustack/runner:maca${MACA_VERSION}${MACA_VERSION_EXTRA}-vllm${VLLM_VERSION}-python${PYTHON_VERSION}-vanilla
+ARG VLLM_GGUF_PLUGIN_VERSION=0.0.3
 
 # Stage Bake Runtime
 #
@@ -269,6 +270,7 @@ SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
+ARG VLLM_GGUF_PLUGIN_VERSION
 
 ENV UV_SYSTEM_PYTHON=1 \
     UV_PRERELEASE=allow \
@@ -290,6 +292,20 @@ ENV VLLM_VERSION=${VLLM_VERSION}
 
 ## Nothing to do, as the vLLM has installed in global.
 
+## Install GGUF
+
+RUN <<EOF
+    # GGUF
+    # https://docs.vllm.ai/en/v0.24.0/features/quantization/gguf/
+
+    uv pip install \
+        vllm-gguf-plugin==${VLLM_GGUF_PLUGIN_VERSION}
+
+    # Cleanup
+    rm -rf /var/tmp/* \
+        && rm -rf /tmp/*
+EOF
+
 ## Postprocess
 
 RUN <<EOF
@@ -303,6 +319,7 @@ RUN <<EOF
     # Review
     uv pip tree \
         --package vllm \
+        --package vllm-gguf-plugin \
         --package vllm-metax \
         --package flash-attn \
         --package flashinfer \
diff --git a/pack/rocm/Dockerfile.vllm b/pack/rocm/Dockerfile.vllm
index 4440109..0f6eb9c 100644
--- a/pack/rocm/Dockerfile.vllm
+++ b/pack/rocm/Dockerfile.vllm
@@ -9,6 +9,7 @@ ARG VLLM_TORCH_ROCM_VERSION=${ROCM_VERSION}
 ARG VLLM_LMCACHE_VERSION=0.5.0
 ARG VLLM_MOONCAKE_VERSION=0.3.11.post1
 ARG VLLM_OMNI_COMMIT=a560ed1
+ARG VLLM_GGUF_PLUGIN_VERSION=0.0.3
 
 FROM ${VLLM_BASE_IMAGE} AS vllm-build
 SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
@@ -313,6 +314,7 @@ SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
+ARG VLLM_GGUF_PLUGIN_VERSION
 
 ## Install Ray
 
@@ -399,6 +401,7 @@ RUN <<EOF
     # Review
     uv pip tree \
         --package vllm \
+        --package vllm-gguf-plugin \
         --package vllm-omni \
         --package flash-attn \
         --package triton \
@@ -423,6 +426,20 @@ RUN --mount=type=bind,target=/workspace,rw <<EOF
     fi
 EOF
 
+## Install GGUF
+
+RUN <<EOF
+    # GGUF
+    # https://docs.vllm.ai/en/v0.24.0/features/quantization/gguf/
+
+    uv pip install \
+        vllm-gguf-plugin==${VLLM_GGUF_PLUGIN_VERSION}
+
+    # Cleanup
+    rm -rf /var/tmp/* \
+        && rm -rf /tmp/*
+EOF
+
 ## Dependencies
 
 RUN <<EOF

From 9543481dfa0d834d00c16fe12b4797d0293928c0 Mon Sep 17 00:00:00 2001
From: axsapronov <a@sapronov.me>
Date: Fri, 10 Jul 2026 10:21:08 +0700
Subject: [PATCH 2/2] fix: don't install vllm-gguf-plugin for old vllm images

---
 pack/cann/Dockerfile.vllm | 17 -----------------
 pack/dtk/Dockerfile.vllm  | 17 -----------------
 pack/hggc/Dockerfile.vllm | 17 -----------------
 pack/maca/Dockerfile.vllm | 17 -----------------
 4 files changed, 68 deletions(-)

diff --git a/pack/cann/Dockerfile.vllm b/pack/cann/Dockerfile.vllm
index ada6a8d..176c258 100644
--- a/pack/cann/Dockerfile.vllm
+++ b/pack/cann/Dockerfile.vllm
@@ -9,7 +9,6 @@ ARG VLLM_LMCACHE_VERSION=0.4.3
 ARG VLLM_LMCACHE_ASCEND_VERSION=${VLLM_LMCACHE_VERSION}
 ARG VLLM_MOONCAKE_VERSION=0.3.11
 ARG VLLM_OMNI_COMMIT=4a24a517
-ARG VLLM_GGUF_PLUGIN_VERSION=0.0.3
 
 FROM ${CANN_BASE_IMAGE} AS vllm-build
 SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
@@ -256,7 +255,6 @@ SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
-ARG VLLM_GGUF_PLUGIN_VERSION
 
 ## Install Ray
 
@@ -405,7 +403,6 @@ EOT
     # Review
     uv pip tree \
         --package vllm \
-        --package vllm-gguf-plugin \
         --package vllm-ascend \
         --package vllm-omni \
         --package torch \
@@ -432,20 +429,6 @@ RUN --mount=type=bind,target=/workspace,rw <<EOF
     fi
 EOF
 
-## Install GGUF
-
-RUN <<EOF
-    # GGUF
-    # https://docs.vllm.ai/en/v0.24.0/features/quantization/gguf/
-
-    uv pip install \
-        vllm-gguf-plugin==${VLLM_GGUF_PLUGIN_VERSION}
-
-    # Cleanup
-    rm -rf /var/tmp/* \
-        && rm -rf /tmp/*
-EOF
-
 ## Dependencies
 
 RUN <<EOF
diff --git a/pack/dtk/Dockerfile.vllm b/pack/dtk/Dockerfile.vllm
index 52b167a..4c7a096 100644
--- a/pack/dtk/Dockerfile.vllm
+++ b/pack/dtk/Dockerfile.vllm
@@ -31,7 +31,6 @@ ARG DTK_ARCHS
 ARG VLLM_VERSION=0.18.1
 ARG VLLM_TORCH_VERSION=2.10.0
 ARG VLLM_LMCACHE_VERSION=0.5.0
-ARG VLLM_GGUF_PLUGIN_VERSION=0.0.3
 ARG VLLM_BASE_IMAGE=gpustack/runner:dtk${DTK_VERSION}${DTK_VERSION_EXTRA}-vllm${VLLM_VERSION}-python${PYTHON_VERSION}-vanilla
 
 # Stage Bake Runtime
@@ -277,7 +276,6 @@ SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
-ARG VLLM_GGUF_PLUGIN_VERSION
 
 ENV UV_SYSTEM_PYTHON=1 \
     UV_PRERELEASE=allow \
@@ -325,20 +323,6 @@ RUN <<EOF
         && rm -rf /tmp/*
 EOF
 
-## Install GGUF
-
-RUN <<EOF
-    # GGUF
-    # https://docs.vllm.ai/en/v0.24.0/features/quantization/gguf/
-
-    uv pip install \
-        vllm-gguf-plugin==${VLLM_GGUF_PLUGIN_VERSION}
-
-    # Cleanup
-    rm -rf /var/tmp/* \
-        && rm -rf /tmp/*
-EOF
-
 ## Postprocess
 
 RUN <<EOF
@@ -347,7 +331,6 @@ RUN <<EOF
     # Review
     uv pip tree \
         --package vllm \
-        --package vllm-gguf-plugin \
         --package torch \
         --package lmcache
 EOF
diff --git a/pack/hggc/Dockerfile.vllm b/pack/hggc/Dockerfile.vllm
index b6b2a88..5a0a8d9 100644
--- a/pack/hggc/Dockerfile.vllm
+++ b/pack/hggc/Dockerfile.vllm
@@ -27,7 +27,6 @@ ARG HGGC_ARCHS
 ARG VLLM_VERSION=0.18.0
 ARG VLLM_TORCH_VERSION=2.9.0
 ARG VLLM_BASE_IMAGE=gpustack/runner:hggc${HGGC_VERSION}${HGGC_VERSION_EXTRA}-vllm${VLLM_VERSION}-python${PYTHON_VERSION}-vanilla
-ARG VLLM_GGUF_PLUGIN_VERSION=0.0.3
 
 # Stage Bake Runtime
 #
@@ -282,7 +281,6 @@ SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
-ARG VLLM_GGUF_PLUGIN_VERSION
 
 ENV UV_SYSTEM_PYTHON=1 \
     UV_PRERELEASE=allow \
@@ -304,20 +302,6 @@ ENV VLLM_VERSION=${VLLM_VERSION}
 
 ## Nothing to do, as the vLLM has installed in global.
 
-## Install GGUF
-
-RUN <<EOF
-    # GGUF
-    # https://docs.vllm.ai/en/v0.24.0/features/quantization/gguf/
-
-    uv pip install \
-        vllm-gguf-plugin==${VLLM_GGUF_PLUGIN_VERSION}
-
-    # Cleanup
-    rm -rf /var/tmp/* \
-        && rm -rf /tmp/*
-EOF
-
 ## Postprocess
 
 RUN <<EOF
@@ -326,7 +310,6 @@ RUN <<EOF
     # Review
     uv pip tree \
         --package vllm \
-        --package vllm-gguf-plugin \
         --package flash-attn \
         --package torch
 EOF
diff --git a/pack/maca/Dockerfile.vllm b/pack/maca/Dockerfile.vllm
index c900963..2cd8f9c 100644
--- a/pack/maca/Dockerfile.vllm
+++ b/pack/maca/Dockerfile.vllm
@@ -27,7 +27,6 @@ ARG MACA_ARCHS
 ARG VLLM_VERSION=0.21.0
 ARG VLLM_TORCH_VERSION=2.8.0
 ARG VLLM_BASE_IMAGE=gpustack/runner:maca${MACA_VERSION}${MACA_VERSION_EXTRA}-vllm${VLLM_VERSION}-python${PYTHON_VERSION}-vanilla
-ARG VLLM_GGUF_PLUGIN_VERSION=0.0.3
 
 # Stage Bake Runtime
 #
@@ -270,7 +269,6 @@ SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
-ARG VLLM_GGUF_PLUGIN_VERSION
 
 ENV UV_SYSTEM_PYTHON=1 \
     UV_PRERELEASE=allow \
@@ -292,20 +290,6 @@ ENV VLLM_VERSION=${VLLM_VERSION}
 
 ## Nothing to do, as the vLLM has installed in global.
 
-## Install GGUF
-
-RUN <<EOF
-    # GGUF
-    # https://docs.vllm.ai/en/v0.24.0/features/quantization/gguf/
-
-    uv pip install \
-        vllm-gguf-plugin==${VLLM_GGUF_PLUGIN_VERSION}
-
-    # Cleanup
-    rm -rf /var/tmp/* \
-        && rm -rf /tmp/*
-EOF
-
 ## Postprocess
 
 RUN <<EOF
@@ -319,7 +303,6 @@ RUN <<EOF
     # Review
     uv pip tree \
         --package vllm \
-        --package vllm-gguf-plugin \
         --package vllm-metax \
         --package flash-attn \
         --package flashinfer \