cuda: copy model into VRAM on single-GPU via --gpu-resident

ds4_agent.c

@@ -668,6 +668,12 @@ static agent_config parse_options(int argc, char **argv) {
             c.gpu_vram_arg = need_arg(&i, argc, argv, arg);
         } else if (!strcmp(arg, "--gpu-devices")) {
             c.gpu_devices_arg = need_arg(&i, argc, argv, arg);
+        } else if (!strcmp(arg, "--gpu-resident")) {
+            /* CUDA: copy the whole model into VRAM at load instead of the
+             * zero-copy host mapping (which streams weights over PCIe). Maps
+             * to the DS4_CUDA_COPY_MODEL_CHUNKED path; requires VRAM >=
+             * model + KV + buffers, otherwise it falls back to host-mapping. */
+            setenv("DS4_CUDA_COPY_MODEL_CHUNKED", "1", 1);
         } else if (!strcmp(arg, "--cuda-tensor-parallel")) {
             c.engine.cuda_tensor_parallel = true;
         } else if (!strcmp(arg, "--cpu")) {

ds4_bench.c

@@ -277,6 +277,12 @@ static bench_config parse_options(int argc, char **argv) {
             c.gpu_vram_arg = need_arg(&i, argc, argv, arg);
         } else if (!strcmp(arg, "--gpu-devices")) {
             c.gpu_devices_arg = need_arg(&i, argc, argv, arg);
+        } else if (!strcmp(arg, "--gpu-resident")) {
+            /* CUDA: copy the whole model into VRAM at load instead of the
+             * zero-copy host mapping (which streams weights over PCIe). Maps
+             * to the DS4_CUDA_COPY_MODEL_CHUNKED path; requires VRAM >=
+             * model + KV + buffers, otherwise it falls back to host-mapping. */
+            setenv("DS4_CUDA_COPY_MODEL_CHUNKED", "1", 1);
         } else if (!strcmp(arg, "--cuda-tensor-parallel")) {
             c.cuda_tensor_parallel = true;
         } else if (!strcmp(arg, "--cpu")) {

ds4_cli.c

@@ -1950,6 +1950,12 @@ static cli_config parse_options(int argc, char **argv) {
             c.gpu_vram_arg = need_arg(&i, argc, argv, arg);
         } else if (!strcmp(arg, "--gpu-devices")) {
             c.gpu_devices_arg = need_arg(&i, argc, argv, arg);
+        } else if (!strcmp(arg, "--gpu-resident")) {
+            /* CUDA: copy the whole model into VRAM at load instead of the
+             * zero-copy host mapping (which streams weights over PCIe). Maps
+             * to the DS4_CUDA_COPY_MODEL_CHUNKED path; requires VRAM >=
+             * model + KV + buffers, otherwise it falls back to host-mapping. */
+            setenv("DS4_CUDA_COPY_MODEL_CHUNKED", "1", 1);
         } else if (!strcmp(arg, "--cuda-tensor-parallel")) {
             c.engine.cuda_tensor_parallel = true;
         } else if (!strcmp(arg, "--dump-tokens")) {

ds4_cuda.cu

@@ -3201,11 +3201,28 @@ extern "C" int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size)
 
 extern "C" int ds4_gpu_set_model_map_range(const void *model_map, uint64_t model_size, uint64_t map_offset, uint64_t map_size, uint64_t max_tensor_bytes) {
     (void)max_tensor_bytes;
-    if (!ds4_gpu_register_model_map_no_copy(model_map, model_size)) return 0;
-    if (getenv("DS4_CUDA_COPY_MODEL_CHUNKED") != NULL &&
-        !cuda_model_copy_chunked(model_map, model_size, map_offset, map_size)) {
-        (void)cuda_model_prefetch_range(model_map, model_size, map_offset, map_size);
+    /* PATCH: full-VRAM residency. Try the chunked device copy BEFORE the
+     * no-copy host registration so cuda_model_copy_chunked()'s guard
+     * (g_model_registered) does not short-circuit it. Active only when
+     * DS4_CUDA_COPY_MODEL_CHUNKED is set; on failure restore state and fall
+     * back to the original no-copy path. */
+    if (getenv("DS4_CUDA_COPY_MODEL_CHUNKED") != NULL) {
+        g_model_host_base               = model_map;
+        g_model_device_base             = (const char *)model_map;
+        g_model_registered_size         = model_size;
+        g_model_range_mapping_supported = 1;
+        g_model_hmm_direct              = 0;
+        g_model_cache_full              = 0;
+        if (g_model_fd >= 0 && g_model_fd_host_base == NULL) {
+            g_model_fd_host_base = model_map;
+        }
+        if (cuda_model_copy_chunked(model_map, model_size, map_offset, map_size)) {
+            return 1;
+        }
+        g_model_host_base       = NULL;
+        g_model_registered_size = 0;
     }
+    if (!ds4_gpu_register_model_map_no_copy(model_map, model_size)) return 0;
     return 1;
 }
 

ds4_help.c

@@ -155,6 +155,7 @@ static void print_model_runtime(FILE *fp, const help_colors *c,
     opt(fp, c, "--backend NAME", "Backend name: metal, cuda, or cpu.");
     opt(fp, c, "--gpu-vram N[,N,...]|auto", "CUDA VRAM budgets per device, in GiB, or auto-detect free VRAM.");
     opt(fp, c, "--gpu-devices N[,N,...]", "CUDA device indices used by multi-GPU placement.");
+    opt(fp, c, "--gpu-resident", "CUDA: copy the whole model into VRAM at load (needs VRAM >= model+KV+buffers); far faster decode when it fits, otherwise falls back to host-mapping.");
     if (tool != DS4_HELP_EVAL) {
         opt(fp, c, "--cuda-tensor-parallel", "Enable the paired DeepSeek tensor/expert path on an even multi-GPU CUDA placement.");
     }

ds4_server.c

@@ -12832,6 +12832,12 @@ static server_config parse_options(int argc, char **argv) {
             c.gpu_vram_arg = need_arg(&i, argc, argv, arg);
         } else if (!strcmp(arg, "--gpu-devices")) {
             c.gpu_devices_arg = need_arg(&i, argc, argv, arg);
+        } else if (!strcmp(arg, "--gpu-resident")) {
+            /* CUDA: copy the whole model into VRAM at load instead of the
+             * zero-copy host mapping (which streams weights over PCIe). Maps
+             * to the DS4_CUDA_COPY_MODEL_CHUNKED path; requires VRAM >=
+             * model + KV + buffers, otherwise it falls back to host-mapping. */
+            setenv("DS4_CUDA_COPY_MODEL_CHUNKED", "1", 1);
         } else if (!strcmp(arg, "--cuda-tensor-parallel")) {
             c.engine.cuda_tensor_parallel = true;
         } else if (!strcmp(arg, "--backend")) {