diff --git a/ds4_agent.c b/ds4_agent.c index d35c125fe..fd38768c8 100644 --- a/ds4_agent.c +++ b/ds4_agent.c @@ -668,6 +668,12 @@ static agent_config parse_options(int argc, char **argv) { c.gpu_vram_arg = need_arg(&i, argc, argv, arg); } else if (!strcmp(arg, "--gpu-devices")) { c.gpu_devices_arg = need_arg(&i, argc, argv, arg); + } else if (!strcmp(arg, "--gpu-resident")) { + /* CUDA: copy the whole model into VRAM at load instead of the + * zero-copy host mapping (which streams weights over PCIe). Maps + * to the DS4_CUDA_COPY_MODEL_CHUNKED path; requires VRAM >= + * model + KV + buffers, otherwise it falls back to host-mapping. */ + setenv("DS4_CUDA_COPY_MODEL_CHUNKED", "1", 1); } else if (!strcmp(arg, "--cuda-tensor-parallel")) { c.engine.cuda_tensor_parallel = true; } else if (!strcmp(arg, "--cpu")) { diff --git a/ds4_bench.c b/ds4_bench.c index 4b2b0d3bd..c723ca0e5 100644 --- a/ds4_bench.c +++ b/ds4_bench.c @@ -277,6 +277,12 @@ static bench_config parse_options(int argc, char **argv) { c.gpu_vram_arg = need_arg(&i, argc, argv, arg); } else if (!strcmp(arg, "--gpu-devices")) { c.gpu_devices_arg = need_arg(&i, argc, argv, arg); + } else if (!strcmp(arg, "--gpu-resident")) { + /* CUDA: copy the whole model into VRAM at load instead of the + * zero-copy host mapping (which streams weights over PCIe). Maps + * to the DS4_CUDA_COPY_MODEL_CHUNKED path; requires VRAM >= + * model + KV + buffers, otherwise it falls back to host-mapping. */ + setenv("DS4_CUDA_COPY_MODEL_CHUNKED", "1", 1); } else if (!strcmp(arg, "--cuda-tensor-parallel")) { c.cuda_tensor_parallel = true; } else if (!strcmp(arg, "--cpu")) { diff --git a/ds4_cli.c b/ds4_cli.c index 811c56e6b..93e97f18e 100644 --- a/ds4_cli.c +++ b/ds4_cli.c @@ -1950,6 +1950,12 @@ static cli_config parse_options(int argc, char **argv) { c.gpu_vram_arg = need_arg(&i, argc, argv, arg); } else if (!strcmp(arg, "--gpu-devices")) { c.gpu_devices_arg = need_arg(&i, argc, argv, arg); + } else if (!strcmp(arg, "--gpu-resident")) { + /* CUDA: copy the whole model into VRAM at load instead of the + * zero-copy host mapping (which streams weights over PCIe). Maps + * to the DS4_CUDA_COPY_MODEL_CHUNKED path; requires VRAM >= + * model + KV + buffers, otherwise it falls back to host-mapping. */ + setenv("DS4_CUDA_COPY_MODEL_CHUNKED", "1", 1); } else if (!strcmp(arg, "--cuda-tensor-parallel")) { c.engine.cuda_tensor_parallel = true; } else if (!strcmp(arg, "--dump-tokens")) { diff --git a/ds4_cuda.cu b/ds4_cuda.cu index aaa4df113..a366b41cb 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -3201,11 +3201,28 @@ extern "C" int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size) extern "C" int ds4_gpu_set_model_map_range(const void *model_map, uint64_t model_size, uint64_t map_offset, uint64_t map_size, uint64_t max_tensor_bytes) { (void)max_tensor_bytes; - if (!ds4_gpu_register_model_map_no_copy(model_map, model_size)) return 0; - if (getenv("DS4_CUDA_COPY_MODEL_CHUNKED") != NULL && - !cuda_model_copy_chunked(model_map, model_size, map_offset, map_size)) { - (void)cuda_model_prefetch_range(model_map, model_size, map_offset, map_size); + /* PATCH: full-VRAM residency. Try the chunked device copy BEFORE the + * no-copy host registration so cuda_model_copy_chunked()'s guard + * (g_model_registered) does not short-circuit it. Active only when + * DS4_CUDA_COPY_MODEL_CHUNKED is set; on failure restore state and fall + * back to the original no-copy path. */ + if (getenv("DS4_CUDA_COPY_MODEL_CHUNKED") != NULL) { + g_model_host_base = model_map; + g_model_device_base = (const char *)model_map; + g_model_registered_size = model_size; + g_model_range_mapping_supported = 1; + g_model_hmm_direct = 0; + g_model_cache_full = 0; + if (g_model_fd >= 0 && g_model_fd_host_base == NULL) { + g_model_fd_host_base = model_map; + } + if (cuda_model_copy_chunked(model_map, model_size, map_offset, map_size)) { + return 1; + } + g_model_host_base = NULL; + g_model_registered_size = 0; } + if (!ds4_gpu_register_model_map_no_copy(model_map, model_size)) return 0; return 1; } diff --git a/ds4_help.c b/ds4_help.c index 5b8fb11cb..27ee2f86b 100644 --- a/ds4_help.c +++ b/ds4_help.c @@ -155,6 +155,7 @@ static void print_model_runtime(FILE *fp, const help_colors *c, opt(fp, c, "--backend NAME", "Backend name: metal, cuda, or cpu."); opt(fp, c, "--gpu-vram N[,N,...]|auto", "CUDA VRAM budgets per device, in GiB, or auto-detect free VRAM."); opt(fp, c, "--gpu-devices N[,N,...]", "CUDA device indices used by multi-GPU placement."); + opt(fp, c, "--gpu-resident", "CUDA: copy the whole model into VRAM at load (needs VRAM >= model+KV+buffers); far faster decode when it fits, otherwise falls back to host-mapping."); if (tool != DS4_HELP_EVAL) { opt(fp, c, "--cuda-tensor-parallel", "Enable the paired DeepSeek tensor/expert path on an even multi-GPU CUDA placement."); } diff --git a/ds4_server.c b/ds4_server.c index f4f563ecb..8b4e37d1c 100644 --- a/ds4_server.c +++ b/ds4_server.c @@ -12832,6 +12832,12 @@ static server_config parse_options(int argc, char **argv) { c.gpu_vram_arg = need_arg(&i, argc, argv, arg); } else if (!strcmp(arg, "--gpu-devices")) { c.gpu_devices_arg = need_arg(&i, argc, argv, arg); + } else if (!strcmp(arg, "--gpu-resident")) { + /* CUDA: copy the whole model into VRAM at load instead of the + * zero-copy host mapping (which streams weights over PCIe). Maps + * to the DS4_CUDA_COPY_MODEL_CHUNKED path; requires VRAM >= + * model + KV + buffers, otherwise it falls back to host-mapping. */ + setenv("DS4_CUDA_COPY_MODEL_CHUNKED", "1", 1); } else if (!strcmp(arg, "--cuda-tensor-parallel")) { c.engine.cuda_tensor_parallel = true; } else if (!strcmp(arg, "--backend")) {