kimi-k3 : the MoonViT-3d vision tower and full-size loading fixes

conversion/base.py

@@ -104,6 +104,11 @@ def repack_mxfp4_blocks(packed: Tensor, scale: Tensor) -> np.ndarray:
     if tuple(s.shape) != (rows, n_blocks):
         raise ValueError(f"MXFP4 scale shape {tuple(s.shape)} does not match {(rows, n_blocks)}")
 
+    # 0xff is the NaN E8M0 exponent; caught later without the tensor name
+    n_bad = int((s == 0xFF).sum())
+    if n_bad:
+        raise ValueError(f"invalid E8M0 scale byte 0xff in {n_bad} MXFP4 block(s)")
+
     src = p.reshape(rows, n_blocks, 16)
     lo = src & 0x0F           # elements 0, 2, 4, ...
     hi = (src >> 4) & 0x0F    # elements 1, 3, 5, ...

conversion/kimivl.py

@@ -168,3 +168,79 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
             name = name.replace("mm_projector.linear_", "mm_projector.proj.linear_", 1)
 
         yield from super().modify_tensors(data_torch, name, bid)
+
+
+@ModelBase.register("KimiK3ForConditionalGeneration")
+class KimiK3VisionModel(MmprojModel):
+    """Kimi-K3 MoonViT-3d vision tower (image path).
+
+    Structurally the Kimi-K2.5 tower with RMSNorm, no biases, a non-square fused QKV
+    (qkv_hidden_size 1536 vs vt_hidden_size 1024) and a post-norm patchmergerv2 projector.
+    Video is out of scope: for t == 1 the temporal pool and temporal position term vanish.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        assert self.hparams_vision is not None, "Kimi-K3 requires vision_config in config.json"
+        self.merge_kernel_size = tuple(self.hparams_vision.get("merge_kernel_size", [2, 2]))
+        self.patch_size = self.hparams_vision.get("patch_size", 14)
+        pos_emb_h = self.hparams_vision.get("init_pos_emb_height", 64)
+        self.hparams_vision["image_size"] = pos_emb_h * self.patch_size
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        assert self.hparams_vision is not None
+        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.KIMIK3)
+
+        # qkv width != n_embd, so the runtime cannot derive d_head
+        n_head = self.hparams_vision["vt_num_attention_heads"]
+        qkv_hidden = self.hparams_vision.get("qkv_hidden_size") or self.hparams_vision["vt_hidden_size"]
+        assert qkv_hidden % n_head == 0, f"qkv_hidden_size {qkv_hidden} not divisible by {n_head} heads"
+        self.gguf_writer.add_vision_head_dim(qkv_hidden // n_head)
+
+        self.gguf_writer.add_vision_use_gelu(True)  # activation_func is gelu_pytorch_tanh
+        self.gguf_writer.add_vision_attention_layernorm_eps(
+            self.hparams_vision.get("projector_ln_eps", 1e-5))
+        self.gguf_writer.add_vision_projector_scale_factor(self.merge_kernel_size[0])
+
+        in_patch_limit = self.preprocessor_config.get("media_proc_cfg", {}).get(
+            "in_patch_limit", self.preprocessor_config.get("in_patch_limit", 16384))
+        pixels_per_patch = self.patch_size ** 2
+        self.gguf_writer.add_vision_min_pixels(8 * pixels_per_patch)
+        self.gguf_writer.add_vision_max_pixels(in_patch_limit * pixels_per_patch)
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        name, _ = item
+        if not name.startswith(("vision_tower.", "mm_projector.")):
+            return None
+        return super().filter_tensors(item)
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        assert self.hparams_vision is not None
+        n_head = self.hparams_vision["vt_num_attention_heads"]
+
+        if "wqkv" in name and "weight" in name:
+            # de-interleave Q/K so the runtime can use build_rope_2d(interleave_freq=false)
+            out_dim = data_torch.shape[0]
+            qkv_dim = out_dim // 3
+            head_dim = qkv_dim // n_head
+            wq, wk, wv = (data_torch[:qkv_dim], data_torch[qkv_dim:2 * qkv_dim], data_torch[2 * qkv_dim:])
+
+            def deinterleave(w: Tensor) -> Tensor:
+                return (w.reshape(n_head, head_dim // 4, 2, 2, w.shape[-1])
+                         .permute(0, 2, 1, 3, 4)
+                         .reshape(w.shape[0], w.shape[-1]))
+
+            data_torch = torch.cat([deinterleave(wq), deinterleave(wk), wv], dim=0)
+
+        if "pos_emb.weight" in name:
+            # kept 3D: the runtime reads grid extents from ne[1]/ne[2]
+            pass
+
+        if "mm_projector.proj.0." in name:
+            name = name.replace(".proj.0.", ".proj.linear_1.")
+        elif "mm_projector.proj.2." in name:
+            name = name.replace(".proj.2.", ".proj.linear_2.")
+
+        yield from super().modify_tensors(data_torch, name, bid)

gguf-py/gguf/constants.py

@@ -5190,6 +5190,7 @@ class VisionProjectorType:
     KIMIVL = "kimivl"
     PADDLEOCR = "paddleocr"
     KIMIK25 = "kimik25"
+    KIMIK3 = "kimik3"
     LIGHTONOCR = "lightonocr"
     COGVLM = "cogvlm"
     JANUS_PRO = "janus_pro"

src/models/kimi-k3.cpp

@@ -35,12 +35,15 @@ void llama_model_kimi_k3::load_arch_hparams(llama_model_loader & ml) {
     ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,       hparams.expert_weights_scale, false);
     ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,        hparams.expert_weights_norm, false);
     ml.get_key(LLM_KV_EXPERT_GATING_FUNC,         hparams.expert_gating_func);
-    ml.get_key(LLM_KV_EXPERT_LATENT_LENGTH,       hparams.n_expert_latent);
-
+    // required: a silent default here loads cleanly and produces garbage
+    ml.get_key(LLM_KV_EXPERT_LATENT_LENGTH,         hparams.n_expert_latent);
     ml.get_key(LLM_KV_ATTN_RES_BLOCK_SIZE,          hparams.attn_res_block_size);
     ml.get_key(LLM_KV_ACTIVATION_SITU_BETA,         hparams.situ_beta);
     ml.get_key(LLM_KV_ACTIVATION_SITU_LINEAR_BETA,  hparams.situ_linear_beta);
 
+    GGML_ASSERT(hparams.attn_res_block_size > 0 && "Kimi-K3 requires attn_res.block_size");
+    GGML_ASSERT(hparams.n_expert_latent   > 0 && "Kimi-K3 requires expert_latent_length");
+
     switch (hparams.n_layer()) {
         case 93: type = LLM_TYPE_2_8T_A50B; break; // Kimi-K3
         default: type = LLM_TYPE_UNKNOWN;
@@ -93,7 +96,13 @@ void llama_model_kimi_k3::load_arch_tensors(llama_model_loader &) {
             layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0);
 
             // K3's A_log is a plain 1-D [n_head] parameter (kimi-linear's is padded);
-            layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {n_head}, TENSOR_NOT_REQUIRED);
+            // NOSCAN because this A is consumed by a broadcast ggml_mul in build_kda, not
+            // by ggml_ssm_scan. Both spellings resolve to the same "blk.%d.ssm_a" name, so
+            // no GGUF changes, but LLM_TENSOR_SSM_A declares GGML_OP_SSM_SCAN and
+            // llama_model::create_tensor probes the buffer type with that op. No backend
+            // offers SSM_SCAN for a [n_head] tensor, so the probe fails, ssm_a is placed on
+            // the CPU, and every op that touches it follows.
+            layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {n_head}, TENSOR_NOT_REQUIRED);
             layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}, 0);
 
             // K3 uses a single full-rank gate instead of kimi-linear's g_a/g_b pair

tools/mtmd/CMakeLists.txt

@@ -40,6 +40,7 @@ add_library(mtmd
             models/granite4-vision.cpp
             models/hunyuanvl.cpp
             models/internvl.cpp
+            models/kimik3.cpp
             models/kimivl.cpp
             models/kimik25.cpp
             models/nemotron-v2-vl.cpp

tools/mtmd/clip-impl.h

@@ -443,6 +443,7 @@ enum projector_type {
     PROJECTOR_TYPE_YOUTUVL,
     PROJECTOR_TYPE_YASA2,
     PROJECTOR_TYPE_KIMIK25,
+    PROJECTOR_TYPE_KIMIK3,
     PROJECTOR_TYPE_NEMOTRON_V2_VL,
     PROJECTOR_TYPE_HUNYUANVL,
     PROJECTOR_TYPE_PARAKEET,
@@ -502,6 +503,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_YOUTUVL,           "youtuvl"},
     { PROJECTOR_TYPE_YASA2,             "yasa2"},
     { PROJECTOR_TYPE_KIMIK25,           "kimik25"},
+    { PROJECTOR_TYPE_KIMIK3,            "kimik3"},
     { PROJECTOR_TYPE_NEMOTRON_V2_VL,    "nemotron_v2_vl"},
     { PROJECTOR_TYPE_EXAONE4_5,         "exaone4_5"},
     { PROJECTOR_TYPE_HUNYUANVL,         "hunyuanvl"},

tools/mtmd/clip.cpp

@@ -998,6 +998,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
             {
                 builder = std::make_unique<clip_graph_kimik25>(ctx, img);
             } break;
+        case PROJECTOR_TYPE_KIMIK3:
+            {
+                builder = std::make_unique<clip_graph_kimik3>(ctx, img);
+            } break;
         case PROJECTOR_TYPE_COGVLM:
             {
                 builder = std::make_unique<clip_graph_cogvlm>(ctx, img);
@@ -1495,6 +1499,23 @@ struct clip_model_loader {
                         hparams.rope_theta = 10000.0f;
                         get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
 
+                        int min_pixels = 0, max_pixels = 0;
+                        get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels, false);
+                        get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels, false);
+                        if (min_pixels > 0 && max_pixels > 0) {
+                            hparams.image_min_pixels = min_pixels;
+                            hparams.image_max_pixels = max_pixels;
+                            hparams.warmup_image_size = static_cast<int>(std::sqrt(max_pixels));
+                        } else {
+                            hparams.set_limit_image_tokens(2, 4096);
+                        }
+                    } break;
+                case PROJECTOR_TYPE_KIMIK3:
+                    {
+                        hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
+                        hparams.rope_theta = 10000.0f;
+                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
+
                         int min_pixels = 0, max_pixels = 0;
                         get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels, false);
                         get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels, false);
@@ -2532,6 +2553,13 @@ struct clip_model_loader {
                     model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
                     model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
                 } break;
+            case PROJECTOR_TYPE_KIMIK3:
+                {
+                    // patchmergerv2, bias-free, norm after the projection
+                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
+                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
+                    model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
+                } break;
             case PROJECTOR_TYPE_KIMIVL:
             case PROJECTOR_TYPE_PADDLEOCR:
             case PROJECTOR_TYPE_KIMIK25:
@@ -3870,6 +3898,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
         case PROJECTOR_TYPE_LFM2:
         case PROJECTOR_TYPE_KIMIVL:
         case PROJECTOR_TYPE_KIMIK25:
+        case PROJECTOR_TYPE_KIMIK3:
             {
                 // dynamic size
                 int out_patch_size = params.patch_size * ctx->model.hparams.n_merge;
@@ -4590,6 +4619,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
         case PROJECTOR_TYPE_PIXTRAL:
         case PROJECTOR_TYPE_KIMIVL:
         case PROJECTOR_TYPE_KIMIK25:
+        case PROJECTOR_TYPE_KIMIK3:
         case PROJECTOR_TYPE_LIGHTONOCR:
             {
                 // set the 2D positions
@@ -5401,6 +5431,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
         case PROJECTOR_TYPE_KIMIVL:
         case PROJECTOR_TYPE_PADDLEOCR:
         case PROJECTOR_TYPE_KIMIK25:
+        case PROJECTOR_TYPE_KIMIK3:
         case PROJECTOR_TYPE_YASA2:
             return ctx->model.mm_2_w->ne[1];
         case PROJECTOR_TYPE_HUNYUANVL:

tools/mtmd/models/kimik3.cpp

@@ -0,0 +1,80 @@
+#include "models.h"
+
+#include <cmath>
+#include <cstring>
+
+// Kimi-K3 MoonViT-3d, image path.
+// Follows clip_graph_kimik25, but with RMSNorm, no biases, qkv width != n_embd, and a post-norm patchmergerv2 projector.
+// Images only: at t == 1 the temporal pool and the temporal position term vanish.
+
+ggml_tensor * clip_graph_kimik3::resize_position_embeddings_3d(uint32_t interpolation_mode) {
+    ggml_tensor * pos_embd = model.position_embeddings;
+    const int height       = img.ny() / patch_size;
+    const int width        = img.nx() / patch_size;
+
+    GGML_ASSERT(pos_embd);
+
+    const int64_t stored_c = pos_embd->ne[0];
+    const int64_t orig_w   = pos_embd->ne[1];
+    const int64_t orig_h   = pos_embd->ne[2];
+
+    GGML_ASSERT(stored_c == n_embd);
+
+    if (height == (int) orig_h && width == (int) orig_w) {
+        return ggml_cont_2d(ctx0, pos_embd, n_embd, width * height);
+    }
+
+    pos_embd = ggml_permute(ctx0, pos_embd, 2, 1, 0, 3);
+    pos_embd = ggml_interpolate(ctx0, pos_embd, height, width, n_embd, 1, interpolation_mode);
+    pos_embd = ggml_permute(ctx0, pos_embd, 2, 1, 0, 3);
+    pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, width * height);
+    return pos_embd;
+}
+
+ggml_cgraph * clip_graph_kimik3::build() {
+    ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
+    ggml_set_name(pos_h, "pos_h");
+    ggml_set_input(pos_h);
+
+    ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
+    ggml_set_name(pos_w, "pos_w");
+    ggml_set_input(pos_w);
+
+    ggml_tensor * learned_pos_embd = resize_position_embeddings_3d(GGML_SCALE_MODE_BILINEAR);
+
+    // Q/K are de-interleaved during conversion.
+    auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+        return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false);
+    };
+
+    ggml_tensor * inp = build_inp();
+    inp = ggml_add(ctx0, inp, learned_pos_embd);
+
+    ggml_tensor * cur = build_vit(
+                            inp, n_patches,
+                            NORM_TYPE_RMS,
+                            hparams.ffn_op,
+                            nullptr,
+                            add_pos);
+    cb(cur, "vit_out", -1);
+
+    {
+        const int scale_factor = model.hparams.n_merge;
+        cur = build_patch_merge_permute(cur, scale_factor);
+
+        cur = build_ffn(cur,
+            model.mm_1_w, nullptr,
+            nullptr,      nullptr,
+            model.mm_2_w, nullptr,
+            FFN_GELU,
+            -1);
+        cb(cur, "proj_mlp_out", -1);
+
+        cur = build_norm(cur, model.mm_post_norm_w, nullptr, NORM_TYPE_RMS, hparams.eps, -1);
+        cb(cur, "proj_out", -1);
+    }
+
+    ggml_build_forward_expand(gf, cur);
+
+    return gf;
+}

tools/mtmd/models/models.h

@@ -337,6 +337,13 @@ struct clip_graph_parakeet : clip_graph {
     ggml_cgraph * build() override;
 };
 
+struct clip_graph_kimik3 : clip_graph {
+    clip_graph_kimik3(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+    ggml_cgraph * build() override;
+
+    ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode);
+};
+
 struct clip_graph_exaone4_5 : clip_graph {
     clip_graph_exaone4_5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
     ggml_cgraph * build() override;

tools/mtmd/mtmd.cpp

@@ -578,6 +578,7 @@ struct mtmd_context {
                     image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
                 } break;
             case PROJECTOR_TYPE_KIMIK25:
+            case PROJECTOR_TYPE_KIMIK3:
                 {
                     // GLM-5.2-V reuses the Kimi-K2.5 vision encoder and projector, but marks
                     // images with its own tokens, so decide based on the text model vocab