diff --git a/.gitignore b/.gitignore index b37edf00289..6692a8ca1ce 100644 --- a/.gitignore +++ b/.gitignore @@ -120,6 +120,7 @@ playwright/.cache/ # Terraform .terraform/ .terraform.lock.hcl +.terraform.tfstate.lock.info terraform.tfstate terraform.tfstate.backup *.tfvars diff --git a/scripts/terraform/runpod/README.md b/scripts/terraform/runpod/README.md new file mode 100644 index 00000000000..3bbf97409b6 --- /dev/null +++ b/scripts/terraform/runpod/README.md @@ -0,0 +1,166 @@ +# vLLM / SGLang / Ollama on RunPod + +Runs one RunPod GPU pod with `vllm/vllm-openai`, `lmsysorg/sglang` or `ollama/ollama`, as a target for Bifrost's `vllm`, `sgl` or `ollama` provider. Each preset is one of two kinds: + +- **Inference models** (~20–50B open-weight). On vLLM and SGLang the preset sets the tool-call and reasoning parsers; Ollama handles both per model itself. +- **Decision models** (Clef, Liquid d1, PPLX-Decider, Nimble, Tev1), served on the engine's native `POST /v1/systemone` (TypeSafe's System One API). + +## Usage + +```bash +export RUNPOD_API_KEY=... +cp terraform.tfvars.example terraform.tfvars # pick an engine and a model +terraform init +terraform apply # waits until the model is listed on /v1/models + +terraform output bifrost_provider_name # vllm, sgl or ollama +terraform output -json bifrost_provider_config # paste under providers. in config.json +terraform destroy # or let the auto-teardown (60m by default) do it +``` + +Changing any variable and re-applying **replaces** the pod, because the provider's in-place update doesn't work against the RunPod API. Weights are cached on the pod volume, which is destroyed with the pod. + +## Engines + +| | vLLM | SGLang | Ollama | +|---|---|---|---| +| Image | `vllm/vllm-openai:v0.30.0` (stable), `:nightly` | `lmsysorg/sglang:v0.5.21` (stable), `:dev` | `ollama/ollama:0.40.4`, `:latest` | +| Weights | HF checkpoints (BF16 / FP8 / NVFP4 / MXFP4) | HF checkpoints | Ollama library GGUF builds (presets use `q4_K_M`) | +| Host CUDA | 13.0 (12.9 with `cuda_version = "12.9"`) | 13.0 | 12.8+ | +| Multi-GPU | Tensor parallel (`gpu_count`) | Tensor parallel (`gpu_count`) | Splits a model across GPUs only when it doesn't fit on one | +| Auth | `VLLM_API_KEY` | `--api-key` | nginx in front of Ollama checks the bearer token (Ollama has no server-side auth) | +| Config surface | `extra_args` (+ `extra_env`) | `extra_args` (+ `extra_env`) | `extra_env` (`OLLAMA_*`); no server args | + +All three listen on port 8000 behind the same generated API key. `server_args` shows the args without the key. + +**Ollama specifics.** `ollama-serve.sh` runs as the server command: +1. Installs nginx and curl with apt on first container start (curl is for the auto-teardown, which calls the RunPod API). +2. Starts Ollama on `127.0.0.1:11434`. +3. Pulls the preset's tag and copies it to the served name (`ollama cp gpt-oss:20b gpt-oss-20b`). +4. Starts nginx on `:8000`. + +The module also sets `OLLAMA_KEEP_ALIVE=-1` so the model stays loaded, `OLLAMA_CONTEXT_LENGTH` from the preset, and `OLLAMA_MODELS=/workspace/ollama` on the volume. Ollama sizes its default context by VRAM (4K under 24 GiB), which truncates tool-calling prompts. That's why the presets set it explicitly. + +## Inference presets + +vLLM parsers are the registry names in vLLM v0.30.0 (`vllm/tool_parsers`, `vllm/reasoning`), from the [vLLM recipes](https://github.com/vllm-project/recipes) and HF model cards. SGLang parsers are the registry names in SGLang v0.5.21 (`srt/function_call/function_call_parser.py`, `srt/parser/reasoning_parser.py`), from the [SGLang cookbook](https://github.com/sgl-project/sglang/tree/main/docs/cookbook/autoregressive). The two name several parsers differently. Ollama tags are from [ollama.com/library](https://ollama.com/library). + +| `model` | HF repo | Size | vLLM tool / reasoning | SGLang tool / reasoning | Default GPU | ~$/hr | Notes | +|---|---|---|---|---|---|---|---| +| `gpt-oss-20b` | `openai/gpt-oss-20b` | 21B / 3.6B act, MXFP4 14GB | `openai` / auto | `gpt-oss` / `gpt-oss` | 1×24GB: A5000, L4, 3090, PRO 4000, 4090 | 0.27 | 131K context. Tools only support `tool_choice="auto"`. | +| `glm-4.7-flash` | `zai-org/GLM-4.7-Flash` | 31B / 3B act, BF16 62GB | `glm47` / `glm45` | `glm47` / `glm45` | 1×A100 80GB | 1.59 | 65K context. No official FP8. SGLang ≥ 0.5.8. | +| `kimi-linear-48b-a3b` | `moonshotai/Kimi-Linear-48B-A3B-Instruct` | 49B / 3B act, BF16 98GB | `kimi_k2` ⚠️ / — | `kimi_k2` / — | 2×A100 80GB, TP2 | 3.18 | `--trust-remote-code`. Not in the Ollama library. See caveats. | +| `qwen3.8-27b` | `Qwen/Qwen3.8-27B-FP8` | 27B dense + vision, FP8 31GB | `qwen3_coder` / `qwen3` | `qwen3_coder` / `qwen3` | 1×48GB Ada: L40, RTX 6000 Ada, L40S | 0.82 | 65K context. vLLM uses **nightly** (recipe validated on a pre-release image); SGLang ≥ 0.5.18. | +| `qwen3.6-35b-a3b` | `Qwen/Qwen3.6-35B-A3B-FP8` | 36B / 3B act, FP8 38GB | `qwen3_coder` / `qwen3` | `qwen3_coder` / `qwen3` | 1×48GB Ada | 0.82 | 65K context. ~4GB headroom on 48GB; see caveats. | +| `qwen3-coder-30b-a3b` | `Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8` | 30B / 3B act, FP8 31GB | `qwen3_coder` / — | `qwen3_coder` / — | 1×48GB Ada | 0.82 | 65K context. No thinking mode. | +| `nemotron-3.5-lightning-30b-a3b` | `nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4` | 30B / 3B act, NVFP4 18GB | `qwen3_xml` / `nemotron_v3` | `qwen3_coder` / `nemotron_3` | 1×32GB: PRO 4500, RTX 5000 Ada, 5090 | 0.72 | vLLM uses the recipe's NVFP4 flags (`--moe-backend marlin`, FP8 KV), ≥ 0.27.1. SGLang uses `--mamba-ssm-dtype float16`, ≥ 0.5.18. | +| `gemma-4-31b-it` | `google/gemma-4-31B-it` | 31B dense, BF16 63GB | `gemma4` / `gemma4` | `gemma4` / `gemma4` | 1×A100 80GB | 1.59 | 32K context. Gated (`hf_token`). vLLM uses the image's `tool_chat_template_gemma4.jinja`; SGLang uses the HF template. | + +On Ollama the same presets load `q4_K_M` GGUF builds. These are smaller than the checkpoints above, so several use a smaller GPU list: + +| `model` | Ollama tag | Size | Default GPU on Ollama | ~$/hr | +|---|---|---|---|---| +| `gpt-oss-20b` | `gpt-oss:20b` | 13.8GB (MXFP4) | 1×24GB | 0.27 | +| `glm-4.7-flash` | `glm-4.7-flash:q4_K_M` | 19.0GB | 1×32GB | 0.72 (vs 1.59 for BF16 on A100) | +| `qwen3.8-27b` | `qwen3.8:27b-q4_K_M` | 17.7GB + vision projector | 1×32GB | 0.72 | +| `qwen3.6-35b-a3b` | `qwen3.6:35b-a3b-q4_K_M` | 23.9GB | 1×32GB | 0.72 | +| `qwen3-coder-30b-a3b` | `qwen3-coder:30b-a3b-q4_K_M` | 18.6GB | 1×32GB | 0.72 | +| `nemotron-3.5-lightning-30b-a3b` | `nemotron-3.5-lightning:30b-a3b-q4_K_M` | 25.4GB | 1×32GB | 0.72 | +| `gemma-4-31b-it` | `gemma4:31b-it-q4_K_M` | 20.4GB + vision projector + draft model | 1×48GB | 0.82 | + +Prices are RunPod Secure Cloud on-demand for the first GPU in the list; Community Cloud is typically 15–60% lower. With `gpu_type_priority = "custom"` (default), RunPod rents the first type in the list that has stock. + +Kimi K2.x/K3 (~1T+) and GLM-4.5-Air/5.x (100B+) have no variant in this size class. + +## Decision presets + +Decision models answer typed questions (`choice`, `noul`, `score`) with probabilities on `POST /v1/systemone`. A preset without a block for the chosen engine fails at plan time. + +| `model` | Weights | vLLM | SGLang | Ollama | Default GPU | ~$/hr | Notes | +|---|---|---|---|---|---|---|---| +| `clef-flash` | `Cloudflare/clef-flash`, 9.4B BF16 19GB | ✅ nightly | ✅ `dev` | ✅ `clef-flash:9b` (10.6GB) | 1×32GB (Ollama: 1×24GB) | 0.72 / 0.27 | 16,384-token prompt cap. On SGLang it runs at TP=1 and disables radix cache, chunked prefill and CUDA graphs itself. | +| `clef` | `Cloudflare/clef`, 27B BF16 55GB | — | ✅ `dev` | ✅ `clef:27b-q4_K_M` (20.2GB) | 1×A100 80GB (Ollama: 1×32GB) | 1.59 / 0.72 | vLLM support is undocumented. | +| `d1-3b` | `LiquidAI/d1-3B`, 3.1B BF16 7GB | ✅ nightly | — | — | 1×24GB | 0.27 | LFM1.0 license. `--no-enable-prefix-caching` suits one-question traffic over a changing state. | +| `pplx-decider-v1.1-27b` | `perplexity-ai/pplx-decider-v1.1-27b`, 27B BF16 52GB | — | ✅ `dev` | — | 1×A100 80GB | 1.59 | Validated on H200; 80GB is an estimate. vLLM doesn't map its readout head. Trained on prompts up to 8,192 tokens. FP8 moves probabilities by up to 0.2. | +| `nimble` | `nimble:9b` (9.5GB) | — | — | ✅ | 1×24GB | 0.27 | Bespoke Labs, Qwen3.5-9B base. Prompts must fit 8,192 tokens. | +| `tev1` | `tev1:4b` (4.6GB) | — | — | ✅ | 1×24GB | 0.27 | Together AI, experimental, ~2K context. `ollama_model = "tev1:0.8b"` for the 0.8B variant. | + +**Why some combinations are missing:** +- **Nimble and Tev1 on vLLM/SGLang.** Both are trained on their own prompt formats. Nimble's HF weights are LoRA adapters with a calibrated temperature. The vLLM and SGLang native routes use a generic prompt, so their answers wouldn't match; Ollama's builds carry the model's own format. +- **Laya.** Every Ollama build is MLX-format and untested on NVIDIA; it otherwise runs on `laya-serve`, not on any of these engines. + +**Image versions.** vLLM's `/v1/systemone` (all three question types) merged after v0.31.0, so only nightly images have it. SGLang v0.5.21 has the route, but Clef and PPLX-Decider support merged after that release. Ollama serves all five of its decision models from 0.40.0. The vLLM/SGLang decision presets therefore default to `vllm/vllm-openai:nightly` / `lmsysorg/sglang:dev`. Pin a build for repeatable runs, e.g. `image = "vllm/vllm-openai:nightly-77154ca6116ae34cbfe314beef1016bde9b63989"` (2026-10-14, all three question types) or `lmsysorg/sglang:nightly-dev--`. + +**Your own decision model.** Add a `custom_presets` entry with `kind = "decision"` (see `terraform.tfvars.example`): +- **SGLang:** Qwen3.8-27B and Qwen3.5-35B-A3B are validated. +- **vLLM:** covers Qwen3, Qwen3.5 and Qwen3.5-MoE, and returns 501 for other architectures. The preset runs on the nightly image, since stable releases lack `/v1/systemone`, unless you set `vllm.channel`, `channel` or `image`. + +## API Reference + +| Variable | Default | Purpose | +|---|---|---| +| `engine` | `vllm` | `vllm`, `sglang` or `ollama`. | +| `model` | `gpt-oss-20b` | Preset key. | +| `custom_presets` | `{}` | Your own presets (see `terraform.tfvars.example`). They run on every engine; parsers go in the `vllm` / `sglang` blocks, the Ollama tag in the `ollama` block. | +| `hf_model`, `served_model_name`, `max_model_len` | preset | Per-field overrides. `hf_model` applies to vLLM/SGLang. `max_model_len` maps to vLLM `--max-model-len`, SGLang `--context-length` and Ollama `OLLAMA_CONTEXT_LENGTH`. | +| `ollama_model` | preset | Ollama tag override, e.g. `gpt-oss:20b-q8_0` or `hf.co//:`. | +| `tool_call_parser`, `reasoning_parser` | preset | Override; `""` disables. Names are engine-specific; Ollama ignores them. | +| `gpu_memory_utilization` | vLLM `0.92`, SGLang auto | vLLM `--gpu-memory-utilization` or SGLang `--mem-fraction-static` (SGLang's fraction covers weights + KV). Ollama ignores it. | +| `extra_args`, `extra_env` | `[]`, `{}` | Appended to/merged over the preset. Flags are engine-specific; Ollama accepts `extra_env` only. | +| `channel` | preset | `stable` → `vllm/vllm-openai:[-cu129]`, `lmsysorg/sglang:` or `ollama/ollama:`; `nightly` → `vllm/vllm-openai:[cu129-]nightly`, `lmsysorg/sglang:dev` or `ollama/ollama:latest`. | +| `vllm_version` | `v0.30.0` | vLLM stable tag. It's the base of RunPod's `runpod/worker-v1-vllm` image, so hosts may already have its layers; other tags pull ~9 GB. | +| `sglang_version` | `v0.5.21` | SGLang stable tag (CUDA 13). The first release with `/v1/systemone`. | +| `ollama_version` | `0.40.4` | Ollama tag. 0.40.0+ serves all of `nimble`, `tev1`, `clef` and `clef-flash`. | +| `cuda_version` | `13.0` | vLLM/SGLang image CUDA build. `12.9` (vLLM only) uses the `-cu129` tags and accepts CUDA 12.9 or 13.0 hosts, but those images crash on import from v0.30.0 ([vllm#59157](https://github.com/vllm-project/vllm/issues/59157)). SGLang has published no CUDA 12 images since v0.5.19. Ollama always accepts 12.8+ hosts. | +| `image` | — | Full override: a pinned nightly (`vllm/vllm-openai:nightly-`, `lmsysorg/sglang:nightly-dev--`), a model preview tag (`vllm/vllm-openai:qwen38`), or another release. | +| `gpu_type_ids`, `gpu_count`, `tensor_parallel_size` | preset / `gpu_count` | RunPod GPU ids, e.g. `NVIDIA RTX A5000`, `NVIDIA A40`, `NVIDIA L40S`, `NVIDIA A100 80GB PCIe`, `NVIDIA H100 PCIe`. | +| `gpu_type_priority` | `custom` | `custom` tries GPU types in list order; `availability` lets RunPod choose. | +| `cloud_type`, `data_center_ids`, `interruptible` | `SECURE`, any, `false` | Placement and spot pricing. | +| `volume_gb`, `container_disk_gb` | `150`, `50` | Weights go to `/workspace/hf` (vLLM/SGLang) or `/workspace/ollama`. | +| `expose_tcp` | `false` | Also expose `8000/tcp` (see networking). | +| `wait_for_ready`, `ready_timeout_minutes` | `true`, `45` | Block `apply` until the served model is listed. Needs `curl` and `jq` on the machine running Terraform. | +| `teardown_after_minutes` | `60` | Pod tears itself down this long after first container start. `0` disables. | +| `teardown_action` | `terminate` | `terminate` deletes the pod; `stop` releases the GPU but keeps (and bills) the volume. | + +## Auto-teardown + +The pod enforces the timeout itself, so it fires even if the machine that ran `apply` is offline. `entrypoint.sh` becomes the container entrypoint: +1. It records a deadline in `/workspace/.teardown_at`. +2. It `exec`s the server command: `vllm serve …`, `python3 -m sglang.launch_server …` or `ollama-serve.sh`. +3. When the deadline passes, it calls the RunPod REST API with the pod's own `RUNPOD_POD_ID` and pod-scoped `RUNPOD_API_KEY`, both injected by RunPod. + +- **When the clock starts:** at the first container start, after the image pull. The model download that follows counts. Container restarts don't reset it. +- **Extend a running pod** from the RunPod web terminal: `echo $(( $(date +%s) + 3600 )) > /workspace/.teardown_at`. Changing `teardown_after_minutes` and re-applying replaces the pod instead. +- **Terminate falls back to stop.** If the pod-scoped key can't delete, the script stops the pod. That frees the GPU, but the volume keeps billing until `terraform destroy`. +- **Terraform state after a terminate.** The pod is gone but Terraform still tracks it, and this provider errors on the 404 instead of dropping it. Run `terraform state rm runpod_pod.server` before the next `apply` or `destroy`. + +## Caveats + +- **Networking.** The pod is reached through `https://-8000.proxy.runpod.net`, which is behind Cloudflare. Requests that send no bytes within 100s get a 524; streams are unaffected once the first token arrives. + - Long non-streaming requests and long prefills can hit that limit. + - For those, stream the response, or reach port 8000 through a TLS-protected tunnel or endpoint you control (for example an SSH tunnel to the pod). `expose_tcp = true` maps `8000/tcp` to the pod's `public_ip`, but that port is plain HTTP: the API key and prompts cross the internet unencrypted, so don't connect to it directly. +- **SGLang.** + - Current images are CUDA 13 only, so pods land on CUDA 13 hosts. + - RunPod pods can't set `--shm-size`. TP>1 may fail in NCCL's shared-memory transport; `extra_env = { NCCL_SHM_DISABLE = "1" }` is the usual workaround. Single-GPU presets are unaffected. +- **Ollama.** + - The first container start needs apt access to install nginx (a few seconds). + - A model that doesn't fit in VRAM partly runs on the CPU. Run `ollama ps` in the pod's web terminal to see the GPU/CPU split. + - `OLLAMA_NUM_PARALLEL` (default 1) multiplies KV-cache memory. `OLLAMA_KV_CACHE_TYPE = "q8_0"` halves it. +- **Kimi-Linear.** + - The vLLM recipe pins 0.11.2 because of a 0.12.0 regression, and 0.30.0 is untested. To fall back: `image = "vllm/vllm-openai:v0.11.2"`, `allowed_cuda_versions = ["12.8", "12.9", "13.0"]`. + - `kimi_k2` as the tool parser is inferred from the chat template, not documented. + - For a single GPU: `hf_model = "nm-testing/Kimi-Linear-48B-A3B-Instruct-FP8-DYNAMIC"`, `gpu_count = 1` (community quant). +- **GPU sizing is estimated** from model size, not validated on RunPod. The recipes only mark H100/H200/B200-class hardware as verified. + - gpt-oss-20b on Ampere/Ada (A5000, L4, 3090, 4090) uses the Marlin MXFP4 path on vLLM and the Triton MoE kernel on SGLang. + - FP8 presets are kept on Ada or newer (L40/L40S/RTX 6000 Ada). On Ampere (A40/A6000, $0.49/hr vs $0.82/hr for an L40) FP8 falls back to slower W8A16 Marlin kernels. To try it: `gpu_type_ids = ["NVIDIA A40", "NVIDIA RTX A6000"]`. + - On vLLM, `qwen3.6-35b-a3b` leaves ~4GB of headroom on 48GB. If it OOMs at startup, lower `max_model_len`, add `extra_args = ["--language-model-only"]` (vLLM), or use `gpu_type_ids = ["NVIDIA A100 80GB PCIe"]`. + - Several small GPUs (e.g. `gpu_count = 2` on 4090s) run tensor parallel over PCIe, which is slower than one larger card with the same total memory. + - Smaller quantized variants of the 80GB presets fit 1×48GB: `unsloth/GLM-4.7-Flash-FP8-Dynamic`, `RedHatAI/gemma-4-31B-it-FP8-dynamic`, `cyankiwi/Kimi-Linear-48B-A3B-Instruct-AWQ-4bit`. All are community or less-tested quants; set `hf_model` + `gpu_type_ids` to use them. +- **Speculative decoding** is off by default. Qwen3.8/3.6 and GLM-4.7-Flash ship MTP heads; on vLLM enable it with e.g. `extra_args = ["--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":3}"]`. Qwen3.8 needs the token count spelled out. +- **Thinking off** for Qwen on vLLM: `extra_args = ["--default-chat-template-kwargs", "{\"enable_thinking\": false}"]`. On SGLang, send `chat_template_kwargs.enable_thinking` per request. + +## Tests + +```bash +terraform init -backend=false && terraform test +``` \ No newline at end of file diff --git a/scripts/terraform/runpod/entrypoint.sh b/scripts/terraform/runpod/entrypoint.sh new file mode 100644 index 00000000000..907d16ea2b6 --- /dev/null +++ b/scripts/terraform/runpod/entrypoint.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Pod entrypoint: arms a self-teardown timer, then execs the server command passed as "$@" (vllm serve / sglang). +# The deadline lives on the pod volume so container restarts don't reset it; overwrite the file to extend. +deadline_file=/workspace/.teardown_at +[ -f "$deadline_file" ] || echo $(( $(date +%s) + TEARDOWN_AFTER_MINUTES * 60 )) > "$deadline_file" +echo "[teardown] pod will ${TEARDOWN_ACTION} at $(date -d "@$(cat "$deadline_file")")" + +( + while [ "$(date +%s)" -lt "$(cat "$deadline_file")" ]; do sleep 30; done + api="https://rest.runpod.io/v1/pods/${RUNPOD_POD_ID}" + auth="Authorization: Bearer ${RUNPOD_API_KEY}" + echo "[teardown] timeout reached, running ${TEARDOWN_ACTION}" + for attempt in 1 2 3 4 5; do + if [ "$TEARDOWN_ACTION" = "terminate" ] && curl -sf --connect-timeout 10 --max-time 30 -X DELETE -H "$auth" "$api"; then + rm -f "$deadline_file" + exit 0 + fi + # Stop releases the GPU; also the fallback if the pod-scoped key can't delete. + curl -sf --connect-timeout 10 --max-time 30 -X POST -H "$auth" "$api/stop" && rm -f "$deadline_file" && exit 0 + echo "[teardown] attempt ${attempt} failed, retrying in 30s" + sleep 30 + done + echo "[teardown] failed to ${TEARDOWN_ACTION} pod ${RUNPOD_POD_ID} after 5 attempts" +) & + +exec "$@" \ No newline at end of file diff --git a/scripts/terraform/runpod/main.tf b/scripts/terraform/runpod/main.tf new file mode 100644 index 00000000000..cb6ec7fa82b --- /dev/null +++ b/scripts/terraform/runpod/main.tf @@ -0,0 +1,237 @@ +locals { + presets = merge(local.builtin_presets, var.custom_presets) + preset = local.presets[var.model] + kind = try(local.preset.kind, "inference") + + # Per-engine preset block; var.engine validation guarantees it is present. + engine_preset = try(local.preset[var.engine], null) + preset_tool_call_parser = try(local.engine_preset.tool_call_parser, null) + preset_reasoning_parser = try(local.engine_preset.reasoning_parser, null) + preset_extra_args = try(coalesce(local.engine_preset.extra_args, []), []) + preset_gpu_type_ids = try(local.engine_preset.gpu_type_ids, null) != null ? local.engine_preset.gpu_type_ids : local.preset.gpu_type_ids + ollama_model = var.ollama_model != null ? var.ollama_model : try(local.engine_preset.model, null) + + hf_model = coalesce(var.hf_model, local.preset.hf_model) + served_model_name = coalesce(var.served_model_name, local.preset.served_model_name, var.model) + tool_call_parser = var.tool_call_parser != null ? var.tool_call_parser : (local.preset_tool_call_parser != null ? local.preset_tool_call_parser : "") + reasoning_parser = var.reasoning_parser != null ? var.reasoning_parser : (local.preset_reasoning_parser != null ? local.preset_reasoning_parser : "") + max_model_len = var.max_model_len != null ? var.max_model_len : local.preset.max_model_len + gpu_count = coalesce(var.gpu_count, local.preset.gpu_count) + tp_size = coalesce(var.tensor_parallel_size, local.gpu_count) + + channel = coalesce(var.channel, try(local.engine_preset.channel, null), var.engine == "vllm" && local.kind == "decision" ? "nightly" : "stable") + cuda_suffix = var.cuda_version == "12.9" ? "cu129" : "" + default_images = { + vllm = local.channel == "nightly" ? "vllm/vllm-openai:${join("-", compact([local.cuda_suffix, "nightly"]))}" : "vllm/vllm-openai:${join("-", compact([var.vllm_version, local.cuda_suffix]))}" + # SGLang images are CUDA 13 only after v0.5.19 (no -cu129 tags); `dev` tracks main. + sglang = local.channel == "nightly" ? "lmsysorg/sglang:dev" : "lmsysorg/sglang:${var.sglang_version}" + # Ollama has no nightly image; `latest` tracks the newest release. + ollama = local.channel == "nightly" ? "ollama/ollama:latest" : "ollama/ollama:${var.ollama_version}" + } + image = coalesce(var.image, local.default_images[var.engine]) + # The Ollama image bundles CUDA 12.8 and 13.0 runners, so it accepts 12.8+ hosts. + default_cuda_versions = var.engine == "ollama" ? ["12.8", "12.9", "13.0"] : (var.cuda_version == "12.9" ? ["12.9", "13.0"] : ["13.0"]) + allowed_cuda_versions = var.allowed_cuda_versions != null ? var.allowed_cuda_versions : local.default_cuda_versions + + api_key = coalesce(var.api_key, random_password.api_key.result) + + vllm_args = concat( + [local.hf_model], + ["--tensor-parallel-size", tostring(local.tp_size)], + ["--gpu-memory-utilization", tostring(coalesce(var.gpu_memory_utilization, 0.92))], + local.max_model_len != null ? ["--max-model-len", tostring(local.max_model_len)] : [], + local.preset.trust_remote_code ? ["--trust-remote-code"] : [], + local.tool_call_parser != "" ? ["--enable-auto-tool-choice", "--tool-call-parser", local.tool_call_parser] : [], + local.reasoning_parser != "" ? ["--reasoning-parser", local.reasoning_parser] : [], + local.preset_extra_args, + var.extra_args, + ["--served-model-name", local.served_model_name], + ["--host", "0.0.0.0", "--port", "8000"], + ) + + # SGLang sizes its static memory pool itself unless gpu_memory_utilization is set. + sglang_args = concat( + ["--model-path", local.hf_model], + ["--tp-size", tostring(local.tp_size)], + var.gpu_memory_utilization != null ? ["--mem-fraction-static", tostring(var.gpu_memory_utilization)] : [], + local.max_model_len != null ? ["--context-length", tostring(local.max_model_len)] : [], + local.preset.trust_remote_code ? ["--trust-remote-code"] : [], + local.tool_call_parser != "" ? ["--tool-call-parser", local.tool_call_parser] : [], + local.reasoning_parser != "" ? ["--reasoning-parser", local.reasoning_parser] : [], + local.preset_extra_args, + var.extra_args, + ["--served-model-name", local.served_model_name], + ["--host", "0.0.0.0", "--port", "8000"], + ) + + # Ollama is configured through env (below); ollama-serve.sh pulls the tag and aliases it to the served name. + ollama_args = [coalesce(local.ollama_model, "unset"), local.served_model_name] + + server_args = { vllm = local.vllm_args, sglang = local.sglang_args, ollama = local.ollama_args }[var.engine] + server_cmd = { + vllm = ["vllm", "serve"] + sglang = ["python3", "-m", "sglang.launch_server"] + ollama = ["/bin/bash", "-c", file("${path.module}/ollama-serve.sh"), "ollama-serve"] + }[var.engine] + + # vLLM reads its key from VLLM_API_KEY, SGLang only takes the --api-key flag, and Ollama's nginx proxy reads PROXY_API_KEY. + pod_args = var.engine == "sglang" ? concat(local.server_args, ["--api-key", local.api_key]) : local.server_args + + env = merge( + { HF_HOME = "/workspace/hf" }, + var.hf_token != null ? { HF_TOKEN = var.hf_token } : {}, + # Keep the model loaded (first-request loads would blow short decision timeouts) and the store on the volume. + var.engine == "ollama" ? { OLLAMA_MODELS = "/workspace/ollama", OLLAMA_KEEP_ALIVE = "-1" } : {}, + var.engine == "ollama" && local.max_model_len != null ? { OLLAMA_CONTEXT_LENGTH = tostring(local.max_model_len) } : {}, + local.preset.env, + var.extra_env, + local.teardown_enabled ? { TEARDOWN_AFTER_MINUTES = tostring(var.teardown_after_minutes), TEARDOWN_ACTION = var.teardown_action } : {}, + var.engine == "vllm" ? { VLLM_API_KEY = local.api_key } : {}, + var.engine == "ollama" ? { PROXY_API_KEY = local.api_key } : {}, + ) + + # With teardown on, entrypoint.sh arms the timer and then execs the server command passed as its arguments. + teardown_enabled = var.teardown_after_minutes > 0 + entrypoint = local.teardown_enabled ? ["/bin/bash", "-c", file("${path.module}/entrypoint.sh"), "teardown"] : local.server_cmd + start_cmd = local.teardown_enabled ? concat(local.server_cmd, local.pod_args) : local.pod_args + + base_url = "https://${runpod_pod.server.id}-8000.proxy.runpod.net" + + # The key's models list keeps routing to the served model. + bifrost_provider_name = { vllm = "vllm", sglang = "sgl", ollama = "ollama" }[var.engine] + bifrost_key = { + name = local.served_model_name + value = local.api_key + models = [local.served_model_name] + weight = 1 + } + bifrost_provider_configs = { + vllm = { + keys = [merge(local.bifrost_key, { vllm_key_config = { url = local.base_url, model_name = local.served_model_name } })] + } + sglang = { + keys = [merge(local.bifrost_key, { sgl_key_config = { url = local.base_url } })] + } + ollama = { + keys = [merge(local.bifrost_key, { ollama_key_config = { url = local.base_url } })] + } + } + bifrost_provider_config = local.bifrost_provider_configs[var.engine] + + pod_spec = { + name = coalesce(var.pod_name, "${var.engine}-${var.model}") + image = local.image + entrypoint = local.entrypoint + args = local.start_cmd + env = local.env + gpu_type_ids = var.gpu_type_ids != null ? var.gpu_type_ids : local.preset_gpu_type_ids + gpu_type_priority = var.gpu_type_priority + gpu_count = local.gpu_count + allowed_cuda_versions = local.allowed_cuda_versions + cloud_type = var.cloud_type + data_center_ids = var.data_center_ids + interruptible = var.interruptible + container_disk_gb = var.container_disk_gb + volume_gb = var.volume_gb + ports = concat(["8000/http"], var.expose_tcp ? ["8000/tcp"] : []) + } +} + +resource "random_password" "api_key" { + length = 40 + special = false +} + +# Any spec change replaces the pod: the provider's in-place update calls PUT /pods/{id}, which the RunPod API doesn't support. +resource "terraform_data" "pod_spec" { + input = local.pod_spec + + lifecycle { + precondition { + condition = contains(["stable", "nightly"], local.channel) + error_message = "channel must be \"stable\" or \"nightly\"." + } + precondition { + condition = contains(["inference", "decision"], local.kind) + error_message = "preset kind must be \"inference\" or \"decision\"." + } + precondition { + condition = local.tp_size <= local.gpu_count + error_message = "tensor_parallel_size cannot exceed gpu_count." + } + precondition { + condition = var.engine != "ollama" || local.ollama_model != null + error_message = "engine = \"ollama\" needs an Ollama library tag: set ollama_model, or model in the preset's ollama block." + } + precondition { + condition = var.engine != "ollama" || length(var.extra_args) == 0 + error_message = "Ollama takes no server args; configure it with extra_env (OLLAMA_* variables)." + } + } +} + +moved { + from = runpod_pod.vllm + to = runpod_pod.server +} + +resource "runpod_pod" "server" { + name = local.pod_spec.name + image_name = local.pod_spec.image + docker_entrypoint = local.pod_spec.entrypoint + docker_start_cmd = local.pod_spec.args + env = local.pod_spec.env + compute_type = "GPU" + cloud_type = local.pod_spec.cloud_type + gpu_type_ids = local.pod_spec.gpu_type_ids + gpu_type_priority = local.pod_spec.gpu_type_priority + gpu_count = local.pod_spec.gpu_count + allowed_cuda_versions = local.pod_spec.allowed_cuda_versions + data_center_ids = length(local.pod_spec.data_center_ids) > 0 ? local.pod_spec.data_center_ids : null + interruptible = local.pod_spec.interruptible + container_disk_in_gb = local.pod_spec.container_disk_gb + volume_in_gb = local.pod_spec.volume_gb + volume_mount_path = "/workspace" + ports = local.pod_spec.ports + + lifecycle { + ignore_changes = all + replace_triggered_by = [terraform_data.pod_spec] + } +} + +resource "terraform_data" "wait_for_ready" { + count = var.wait_for_ready ? 1 : 0 + triggers_replace = [runpod_pod.server.id] + + # Decision models also get one System One probe, so an image without /v1/systemone shows up at apply time. + provisioner "local-exec" { + command = <<-EOT + deadline=$(( $(date +%s) + ${var.ready_timeout_minutes * 60} )) + echo "Waiting for $MODEL on $BASE_URL/v1/models (image pull + weight download can take a while)..." + until curl -sf --connect-timeout 10 --max-time 30 -H "Authorization: Bearer $API_KEY" "$BASE_URL/v1/models" | jq -e --arg m "$MODEL" 'any(.data[]?.id; . == $m or . == $m + ":latest")' >/dev/null 2>&1; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "Timed out after ${var.ready_timeout_minutes}m; check the pod logs in the RunPod console." + exit 1 + fi + sleep 20 + done + echo "$ENGINE is ready." + if [ "$KIND" = "decision" ]; then + body='{"model":"'"$MODEL"'","state":"The deploy finished and every health check passed.","questions":{"healthy":{"type":"noul","instructions":"Is the system healthy?"}}}' + if ! curl -sS --fail-with-body --max-time 60 -H "Authorization: Bearer $API_KEY" -H "Content-Type: application/json" -d "$body" "$BASE_URL/v1/systemone"; then + echo + echo "WARNING: POST /v1/systemone failed; this image may not serve decisions for $MODEL." + fi + echo + fi + EOT + environment = { + BASE_URL = local.base_url + API_KEY = local.api_key + ENGINE = var.engine + KIND = local.kind + MODEL = local.served_model_name + } + } +} \ No newline at end of file diff --git a/scripts/terraform/runpod/ollama-serve.sh b/scripts/terraform/runpod/ollama-serve.sh new file mode 100644 index 00000000000..225f0e9bce1 --- /dev/null +++ b/scripts/terraform/runpod/ollama-serve.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Ollama server command: ollama-serve . +# Ollama has no server-side auth, so it listens on loopback and nginx on :8000 checks PROXY_API_KEY as a bearer token. +# nginx starts only after the model is pulled and loaded, so /v1/models answering on :8000 means the model is there. +set -euo pipefail +tag=$1 +name=$2 +export OLLAMA_HOST=127.0.0.1:11434 + +if ! command -v nginx >/dev/null || ! command -v curl >/dev/null; then + printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d && chmod +x /usr/sbin/policy-rc.d + apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends nginx curl >/dev/null + rm -f /usr/sbin/policy-rc.d +fi +rm -f /etc/nginx/sites-enabled/default +cat > /etc/nginx/conf.d/ollama.conf </dev/null 2>&1; do + kill -0 "$server" 2>/dev/null || exit 1 + sleep 1 +done +echo "[ollama] pulling ${tag}" +ollama pull "$tag" +[ "$tag" = "$name" ] || ollama cp "$tag" "$name" +echo "[ollama] loading ${name}" +ollama run "$name" "" /dev/null 2>&1 || echo "[ollama] WARNING: failed to load ${name}; the first request will load it" +nginx +echo "[ollama] serving ${tag} as ${name} on :8000" +wait "$server" \ No newline at end of file diff --git a/scripts/terraform/runpod/outputs.tf b/scripts/terraform/runpod/outputs.tf new file mode 100644 index 00000000000..7048feb7e34 --- /dev/null +++ b/scripts/terraform/runpod/outputs.tf @@ -0,0 +1,59 @@ +output "pod_id" { + value = runpod_pod.server.id +} + +output "engine" { + value = var.engine +} + +output "kind" { + description = "\"inference\" (OpenAI-compatible) or \"decision\" (POST /v1/systemone)." + value = local.kind +} + +output "base_url" { + description = "Server root (Bifrost key/network URL). OpenAI clients use /v1; decision models answer on /v1/systemone." + value = local.base_url +} + +output "served_model_name" { + value = local.served_model_name +} + +output "api_key" { + value = local.api_key + sensitive = true +} + +output "image" { + value = local.image +} + +output "server_args" { + description = "Server args without the API key." + value = local.server_args +} + +output "teardown" { + value = local.teardown_enabled ? "${var.teardown_action} ${var.teardown_after_minutes}m after first container start" : "disabled" +} + +output "cost_per_hr" { + value = runpod_pod.server.cost_per_hr +} + +output "public_ip" { + description = "Populated after the pod starts; run `terraform apply -refresh-only` if empty. TCP port mapping is in the RunPod console." + value = runpod_pod.server.public_ip +} + +output "bifrost_provider_name" { + description = "Key to paste bifrost_provider_config under in config.json providers: vllm or sgl or ollama." + value = local.bifrost_provider_name +} + +output "bifrost_provider_config" { + description = "Drop-in providers. entry for Bifrost config.json." + sensitive = true + value = local.bifrost_provider_config +} \ No newline at end of file diff --git a/scripts/terraform/runpod/presets.tf b/scripts/terraform/runpod/presets.tf new file mode 100644 index 00000000000..06a258722b1 --- /dev/null +++ b/scripts/terraform/runpod/presets.tf @@ -0,0 +1,249 @@ +# Model presets. Each preset carries one block per engine it is known to run on; a null (absent) block means unsupported. +# vllm / sglang block fields (all optional): tool_call_parser, reasoning_parser, channel ("stable" | "nightly"), extra_args, gpu_type_ids. +# ollama block fields: model (Ollama library tag, required), gpu_type_ids. Ollama picks tool/reasoning handling per model itself, +# and its default q4_K_M GGUF builds need less VRAM than the BF16/FP8 checkpoints the other engines load. +# vLLM parser names are the registry keys in vllm/tool_parsers and vllm/reasoning (v0.30.0); SGLang's are the keys in +# srt/function_call/function_call_parser.py and srt/parser/reasoning_parser.py (v0.5.21). They differ per engine. +# Sources: github.com/vllm-project/recipes, the SGLang cookbook (docs/cookbook/autoregressive), the HF model cards, and ollama.com/library. +locals { + # GPU lists ordered by RunPod secure $/hr (as of 2026-09), lowest first; pods try them in order. + # MXFP4/NVFP4 run on Ampere+ via Marlin; FP8 checkpoints are kept on Ada or newer for native FP8 kernels. + gpus_24gb = ["NVIDIA RTX A5000", "NVIDIA L4", "NVIDIA GeForce RTX 3090", "NVIDIA RTX PRO 4000 Blackwell", "NVIDIA GeForce RTX 4090"] # $0.27-0.74 + gpus_32gb = ["NVIDIA RTX PRO 4500 Blackwell", "NVIDIA RTX 5000 Ada Generation", "NVIDIA GeForce RTX 5090"] # $0.72-0.99 + gpus_48gb_ada = ["NVIDIA L40", "NVIDIA RTX 6000 Ada Generation", "NVIDIA L40S"] # $0.82-1.09 + gpus_80gb = ["NVIDIA A100 80GB PCIe", "NVIDIA A100-SXM4-80GB", "NVIDIA H100 PCIe"] # $1.59-2.89 + + builtin_presets = { + # --------------------------------------------------------------------------------------------------------------- + # Inference models (kind = "inference"): OpenAI-compatible chat/responses, routed through Bifrost's vllm or sgl provider. + # --------------------------------------------------------------------------------------------------------------- + + # 21B / 3.6B active, native MXFP4 (~14GB); fits a 24GB card at full context. vLLM sets its reasoning parser + # (openai_gptoss) automatically; SGLang needs it passed. Function calling only supports tool_choice="auto". + "gpt-oss-20b" = { + kind = "inference" + hf_model = "openai/gpt-oss-20b" + served_model_name = "gpt-oss-20b" + trust_remote_code = false + max_model_len = 131072 + gpu_count = 1 + gpu_type_ids = local.gpus_24gb + env = {} + vllm = { tool_call_parser = "openai" } + sglang = { tool_call_parser = "gpt-oss", reasoning_parser = "gpt-oss" } + ollama = { model = "gpt-oss:20b" } + } + + # 31B / ~3B active MoE with MLA, BF16 ~62GB so A100 80GB. No official FP8; unsloth/GLM-4.7-Flash-FP8-Dynamic fits 48GB (unverified). + "glm-4.7-flash" = { + kind = "inference" + hf_model = "zai-org/GLM-4.7-Flash" + served_model_name = "glm-4.7-flash" + trust_remote_code = false + max_model_len = 65536 + gpu_count = 1 + gpu_type_ids = local.gpus_80gb + env = {} + vllm = { tool_call_parser = "glm47", reasoning_parser = "glm45" } + sglang = { tool_call_parser = "glm47", reasoning_parser = "glm45" } + ollama = { model = "glm-4.7-flash:q4_K_M", gpu_type_ids = local.gpus_32gb } + } + + # 49B / 3B active, linear attention, BF16 ~98GB so 2x A100 80GB. vLLM recipe pins 0.11.2; kimi_k2 parser is + # inferred from the chat template's tool-call tokens. For 1 GPU: nm-testing/Kimi-Linear-48B-A3B-Instruct-FP8-DYNAMIC. + "kimi-linear-48b-a3b" = { + kind = "inference" + hf_model = "moonshotai/Kimi-Linear-48B-A3B-Instruct" + served_model_name = "kimi-linear-48b-a3b" + trust_remote_code = true + max_model_len = 131072 + gpu_count = 2 + gpu_type_ids = local.gpus_80gb + env = {} + vllm = { tool_call_parser = "kimi_k2" } + sglang = { tool_call_parser = "kimi_k2" } + } + + # 27B dense hybrid-attention + vision, FP8 ~31GB on a 48GB card. The vLLM recipe was validated on a pre-release image + # (vllm/vllm-openai:qwen38), so vLLM defaults to nightly; SGLang supports it from v0.5.18. + "qwen3.8-27b" = { + kind = "inference" + hf_model = "Qwen/Qwen3.8-27B-FP8" + served_model_name = "qwen3.8-27b" + trust_remote_code = false + max_model_len = 65536 + gpu_count = 1 + gpu_type_ids = local.gpus_48gb_ada + env = {} + vllm = { tool_call_parser = "qwen3_coder", reasoning_parser = "qwen3", channel = "nightly" } + sglang = { tool_call_parser = "qwen3_coder", reasoning_parser = "qwen3" } + ollama = { model = "qwen3.8:27b-q4_K_M", gpu_type_ids = local.gpus_32gb } + } + + # 36B / 3B active MoE, FP8 ~38GB; tight on 48GB (hybrid attention keeps KV small). + "qwen3.6-35b-a3b" = { + kind = "inference" + hf_model = "Qwen/Qwen3.6-35B-A3B-FP8" + served_model_name = "qwen3.6-35b-a3b" + trust_remote_code = false + max_model_len = 65536 + gpu_count = 1 + gpu_type_ids = local.gpus_48gb_ada + env = {} + vllm = { tool_call_parser = "qwen3_coder", reasoning_parser = "qwen3" } + sglang = { tool_call_parser = "qwen3_coder", reasoning_parser = "qwen3" } + ollama = { model = "qwen3.6:35b-a3b-q4_K_M", gpu_type_ids = local.gpus_32gb } + } + + # 30B / 3B active coder MoE, FP8 ~31GB on a 48GB card. No reasoning mode. + "qwen3-coder-30b-a3b" = { + kind = "inference" + hf_model = "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8" + served_model_name = "qwen3-coder-30b-a3b" + trust_remote_code = false + max_model_len = 65536 + gpu_count = 1 + gpu_type_ids = local.gpus_48gb_ada + env = {} + vllm = { tool_call_parser = "qwen3_coder" } + sglang = { tool_call_parser = "qwen3_coder" } + ollama = { model = "qwen3-coder:30b-a3b-q4_K_M", gpu_type_ids = local.gpus_32gb } + } + + # 30B / 3B active hybrid Mamba-MoE, recipe-default NVFP4 (~18GB) via Marlin. BF16 repo needs 80GB. Needs vLLM >= 0.27.1 + # or SGLang >= 0.5.18. The engines name the parsers differently (vLLM qwen3_xml/nemotron_v3, SGLang qwen3_coder/nemotron_3). + "nemotron-3.5-lightning-30b-a3b" = { + kind = "inference" + hf_model = "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4" + served_model_name = "nemotron-3.5-lightning-30b-a3b" + trust_remote_code = false + max_model_len = 65536 + gpu_count = 1 + gpu_type_ids = local.gpus_32gb + env = {} + vllm = { + tool_call_parser = "qwen3_xml" + reasoning_parser = "nemotron_v3" + extra_args = ["--mamba-backend", "flashinfer", "--mamba-cache-mode", "align", "--enable-prefix-caching", "--max-num-batched-tokens", "16384", "--kv-cache-dtype", "fp8", "--moe-backend", "marlin"] + } + sglang = { + tool_call_parser = "qwen3_coder" + reasoning_parser = "nemotron_3" + extra_args = ["--mamba-ssm-dtype", "float16"] + } + ollama = { model = "nemotron-3.5-lightning:30b-a3b-q4_K_M" } + } + + # 31B dense multimodal, BF16 ~63GB so A100 80GB (recipe lists FP8 for Hopper/Blackwell only). vLLM's tool calling needs + # the chat template shipped in its image; SGLang uses the HF template. + "gemma-4-31b-it" = { + kind = "inference" + hf_model = "google/gemma-4-31B-it" + served_model_name = "gemma-4-31b-it" + trust_remote_code = false + max_model_len = 32768 + gpu_count = 1 + gpu_type_ids = local.gpus_80gb + env = {} + vllm = { + tool_call_parser = "gemma4" + reasoning_parser = "gemma4" + extra_args = ["--chat-template", "/vllm-workspace/examples/tool_chat_template_gemma4.jinja"] + } + sglang = { tool_call_parser = "gemma4", reasoning_parser = "gemma4" } + ollama = { model = "gemma4:31b-it-q4_K_M", gpu_type_ids = local.gpus_48gb_ada } + } + + # --------------------------------------------------------------------------------------------------------------- + # Decision models (kind = "decision"): served on the engine's native POST /v1/systemone (TypeSafe System One API). + # No tool or reasoning parsers apply. + # vLLM ships /v1/systemone only in nightlies (merged after v0.31.0). SGLang has it from v0.5.21, but Clef and + # PPLX-Decider support landed after that release, so these presets use the dev image there too. + # --------------------------------------------------------------------------------------------------------------- + + # 9.4B joint-schema classifier, BF16 ~19GB. Prompts cap at 16,384 tokens. On SGLang it must run at TP=1 and + # disables the radix cache, chunked prefill and CUDA graphs on its own. + "clef-flash" = { + kind = "decision" + hf_model = "Cloudflare/clef-flash" + served_model_name = "clef-flash" + trust_remote_code = false + max_model_len = 16384 + gpu_count = 1 + gpu_type_ids = local.gpus_32gb + env = {} + vllm = { channel = "nightly" } + sglang = { channel = "nightly" } + ollama = { model = "clef-flash:9b", gpu_type_ids = local.gpus_24gb } + } + + # 27B Clef, BF16 ~55GB so A100/H100 80GB on SGLang (vLLM support undocumented); Ollama's q4_K_M build is 20GB. + "clef" = { + kind = "decision" + hf_model = "Cloudflare/clef" + served_model_name = "clef" + trust_remote_code = false + max_model_len = 16384 + gpu_count = 1 + gpu_type_ids = local.gpus_80gb + env = {} + sglang = { channel = "nightly" } + ollama = { model = "clef:27b-q4_K_M", gpu_type_ids = local.gpus_32gb } + } + + # 3.1B LFM2-VL decision model (~7GB). vLLM only. Prefix caching only helps when several questions share a state. + "d1-3b" = { + kind = "decision" + hf_model = "LiquidAI/d1-3B" + served_model_name = "d1-3b" + trust_remote_code = false + max_model_len = null + gpu_count = 1 + gpu_type_ids = local.gpus_24gb + env = {} + vllm = { channel = "nightly" } + } + + # 27B Qwen3.5-based decider, BF16 ~52GB (validated on H200; A100/H100 80GB is an estimate). SGLang only: vLLM does not + # map its Qwen3_5Model readout head. v1.1 turns off the radix cache and chunked prefill itself. Trained on prompts + # up to 8,192 tokens. FP8 moves probabilities by up to 0.2, so this stays BF16. + "pplx-decider-v1.1-27b" = { + kind = "decision" + hf_model = "perplexity-ai/pplx-decider-v1.1-27b" + served_model_name = "pplx-decider-v1.1-27b" + trust_remote_code = false + max_model_len = null + gpu_count = 1 + gpu_type_ids = local.gpus_80gb + env = {} + sglang = { channel = "nightly" } + } + + # Bespoke Labs' 9B decision model (Qwen3.5-9B base), Ollama only: the published weights are LoRA adapters trained on + # their own prompt format, which Ollama's build applies and the vLLM/SGLang native routes don't. 9.5GB; prompts cap at 8,192 tokens. + "nimble" = { + kind = "decision" + hf_model = "bespokelabs/Bespoke-Nimble-9B-v2" + served_model_name = "nimble" + trust_remote_code = false + max_model_len = 8192 + gpu_count = 1 + gpu_type_ids = local.gpus_24gb + env = {} + ollama = { model = "nimble:9b" } + } + + # Together AI's experimental 4B decision model (~4.5GB), Ollama only for the same prompt-format reason. ~2K context. + "tev1" = { + kind = "decision" + hf_model = "togethercomputer/Tev1-4B-experimental" + served_model_name = "tev1" + trust_remote_code = false + max_model_len = null + gpu_count = 1 + gpu_type_ids = local.gpus_24gb + env = {} + ollama = { model = "tev1:4b" } + } + } +} diff --git a/scripts/terraform/runpod/terraform.tfvars.example b/scripts/terraform/runpod/terraform.tfvars.example new file mode 100644 index 00000000000..5b815539b15 --- /dev/null +++ b/scripts/terraform/runpod/terraform.tfvars.example @@ -0,0 +1,67 @@ +# export RUNPOD_API_KEY=... + +# vllm | sglang | ollama +engine = "vllm" + +# Inference: gpt-oss-20b | glm-4.7-flash | kimi-linear-48b-a3b | qwen3.8-27b | qwen3.6-35b-a3b +# qwen3-coder-30b-a3b | nemotron-3.5-lightning-30b-a3b | gemma-4-31b-it +# Decision: clef-flash (all) | clef (sglang, ollama) | d1-3b (vllm) | pplx-decider-v1.1-27b (sglang) +# nimble, tev1 (ollama) +model = "gpt-oss-20b" + +# hf_token = "hf_..." # gated models (gemma) +# api_key = "my-key" # generated when unset + +# --- Image --- +# channel = "nightly" # default comes from the preset's engine block +# vllm_version = "v0.30.0" +# sglang_version = "v0.5.21" +# ollama_version = "0.40.4" +# cuda_version = "13.0" # or "12.9", vLLM only (cu129 images are broken from v0.30.0) +# image = "vllm/vllm-openai:nightly-" # or lmsysorg/sglang:nightly-dev-- + +# --- Per-model tweaks --- +# hf_model = "nm-testing/Kimi-Linear-48B-A3B-Instruct-FP8-DYNAMIC" +# ollama_model = "gpt-oss:20b-q8_0" # Ollama tag override +# max_model_len = 32768 +# gpu_memory_utilization = 0.9 # vLLM --gpu-memory-utilization / SGLang --mem-fraction-static +# reasoning_parser = "" # "" disables +# extra_args = ["--kv-cache-dtype", "fp8"] +# extra_env = { VLLM_LOGGING_LEVEL = "DEBUG" } # Ollama: { OLLAMA_KV_CACHE_TYPE = "q8_0" } + +# --- Pod --- +# gpu_type_ids = ["NVIDIA A40", "NVIDIA RTX A6000"] +# gpu_type_priority = "availability" +# gpu_count = 1 +# cloud_type = "COMMUNITY" +# data_center_ids = ["US-TX-3", "US-KS-2"] +# interruptible = true +# volume_gb = 150 + +# --- Teardown --- +# teardown_after_minutes = 60 # 0 disables +# teardown_action = "terminate" # or "stop" + +# --- Custom model (runs on every engine; parsers per engine, Ollama tag in the ollama block) --- +# model = "seed-oss-36b" +# custom_presets = { +# "seed-oss-36b" = { +# hf_model = "ByteDance-Seed/Seed-OSS-36B-Instruct" +# max_model_len = 32768 +# gpu_type_ids = ["NVIDIA H200"] +# vllm = { tool_call_parser = "seed_oss", reasoning_parser = "seed_oss" } +# sglang = { tool_call_parser = "seed_oss", reasoning_parser = "seed_oss" } +# ollama = { model = "hf.co//Seed-OSS-36B-Instruct-GGUF:Q4_K_M" } +# } +# } + +# --- Custom decision model (any generation model with a supported read strategy) --- +# model = "qwen3.5-35b-a3b-decider" +# custom_presets = { +# "qwen3.5-35b-a3b-decider" = { +# kind = "decision" +# hf_model = "Qwen/Qwen3.5-35B-A3B" +# gpu_type_ids = ["NVIDIA H200"] # SGLang validated it BF16 on 1x H200 +# vllm = { channel = "nightly" } # /v1/systemone is nightly-only on vLLM +# } +# } diff --git a/scripts/terraform/runpod/tests/presets.tftest.hcl b/scripts/terraform/runpod/tests/presets.tftest.hcl new file mode 100644 index 00000000000..b8da836f91a --- /dev/null +++ b/scripts/terraform/runpod/tests/presets.tftest.hcl @@ -0,0 +1,609 @@ +mock_provider "runpod" { + mock_resource "runpod_pod" { + defaults = { id = "abc123" } + override_during = plan + } +} + +variables { + model = "gpt-oss-20b" + wait_for_ready = false +} + +run "gpt_oss_defaults" { + command = plan + + assert { + condition = local.image == "vllm/vllm-openai:v0.30.0" + error_message = "unexpected stable image: ${local.image}" + } + assert { + condition = join(" ", local.server_args) == "openai/gpt-oss-20b --tensor-parallel-size 1 --gpu-memory-utilization 0.92 --max-model-len 131072 --enable-auto-tool-choice --tool-call-parser openai --served-model-name gpt-oss-20b --host 0.0.0.0 --port 8000" + error_message = "unexpected args: ${join(" ", local.server_args)}" + } + assert { + condition = local.allowed_cuda_versions == tolist(["13.0"]) + error_message = "unexpected cuda versions" + } + assert { + condition = runpod_pod.server.gpu_type_ids[0] == "NVIDIA RTX A5000" && runpod_pod.server.gpu_type_priority == "custom" + error_message = "expected the first 24GB card in the list with custom priority" + } + assert { + condition = runpod_pod.server.name == "vllm-gpt-oss-20b" && local.bifrost_provider_name == "vllm" + error_message = "unexpected pod/provider name" + } + assert { + condition = local.bifrost_provider_config.keys[0].vllm_key_config.model_name == "gpt-oss-20b" && local.bifrost_provider_config.keys[0].models == ["gpt-oss-20b"] + error_message = "expected a vllm key pinned to the served model" + } +} + +run "teardown_default_wraps_entrypoint" { + command = plan + + assert { + condition = runpod_pod.server.docker_entrypoint[0] == "/bin/bash" && strcontains(runpod_pod.server.docker_entrypoint[2], "exec \"$@\"") + error_message = "expected teardown wrapper entrypoint" + } + assert { + condition = slice(runpod_pod.server.docker_start_cmd, 0, 3) == tolist(["vllm", "serve", "openai/gpt-oss-20b"]) + error_message = "expected the wrapper to exec vllm serve" + } + assert { + condition = local.env["TEARDOWN_AFTER_MINUTES"] == "60" && local.env["TEARDOWN_ACTION"] == "terminate" + error_message = "expected 60m terminate defaults" + } +} + +run "teardown_disabled_runs_server_directly" { + command = plan + variables { teardown_after_minutes = 0 } + + assert { + condition = runpod_pod.server.docker_entrypoint == tolist(["vllm", "serve"]) && !contains(keys(local.env), "TEARDOWN_ACTION") + error_message = "expected no teardown wrapper" + } + assert { + condition = runpod_pod.server.docker_start_cmd[0] == "openai/gpt-oss-20b" + error_message = "expected serve args only" + } +} + +run "glm_parsers" { + command = plan + variables { model = "glm-4.7-flash" } + + assert { + condition = strcontains(join(" ", local.server_args), "--tool-call-parser glm47 --reasoning-parser glm45") + error_message = "unexpected args: ${join(" ", local.server_args)}" + } +} + +run "kimi_tp2_trust_remote_code" { + command = plan + variables { model = "kimi-linear-48b-a3b" } + + assert { + condition = strcontains(join(" ", local.server_args), "--tensor-parallel-size 2") && contains(local.server_args, "--trust-remote-code") + error_message = "unexpected args: ${join(" ", local.server_args)}" + } + assert { + condition = runpod_pod.server.gpu_count == 2 + error_message = "expected 2 GPUs" + } +} + +run "qwen38_preset_nightly" { + command = plan + variables { model = "qwen3.8-27b" } + + assert { + condition = local.image == "vllm/vllm-openai:nightly" + error_message = "unexpected nightly image: ${local.image}" + } +} + +run "nightly_cuda129_override" { + command = plan + variables { + model = "qwen3.6-35b-a3b" + channel = "nightly" + cuda_version = "12.9" + } + + assert { + condition = local.image == "vllm/vllm-openai:cu129-nightly" && local.allowed_cuda_versions == tolist(["12.9", "13.0"]) + error_message = "unexpected image: ${local.image}" + } +} + +run "overrides_disable_parsers_and_append_args" { + command = plan + variables { + model = "qwen3.6-35b-a3b" + hf_model = "Qwen/Qwen3.6-35B-A3B" + reasoning_parser = "" + max_model_len = 32768 + gpu_count = 2 + extra_args = ["--kv-cache-dtype", "fp8"] + image = "vllm/vllm-openai:nightly-deadbeef" + } + + assert { + condition = join(" ", local.server_args) == "Qwen/Qwen3.6-35B-A3B --tensor-parallel-size 2 --gpu-memory-utilization 0.92 --max-model-len 32768 --enable-auto-tool-choice --tool-call-parser qwen3_coder --kv-cache-dtype fp8 --served-model-name qwen3.6-35b-a3b --host 0.0.0.0 --port 8000" + error_message = "unexpected args: ${join(" ", local.server_args)}" + } + assert { + condition = local.image == "vllm/vllm-openai:nightly-deadbeef" + error_message = "image override ignored" + } +} + +run "custom_preset" { + command = plan + variables { + model = "seed-oss-36b" + custom_presets = { + "seed-oss-36b" = { + hf_model = "ByteDance-Seed/Seed-OSS-36B-Instruct" + max_model_len = 32768 + gpu_type_ids = ["NVIDIA H200"] + vllm = { tool_call_parser = "seed_oss" } + } + } + } + + assert { + condition = join(" ", local.server_args) == "ByteDance-Seed/Seed-OSS-36B-Instruct --tensor-parallel-size 1 --gpu-memory-utilization 0.92 --max-model-len 32768 --enable-auto-tool-choice --tool-call-parser seed_oss --served-model-name seed-oss-36b --host 0.0.0.0 --port 8000" + error_message = "unexpected args: ${join(" ", local.server_args)}" + } + assert { + condition = runpod_pod.server.gpu_type_ids == tolist(["NVIDIA H200"]) + error_message = "gpu override ignored" + } +} + +run "custom_preset_runs_on_sglang_without_engine_block" { + command = plan + variables { + engine = "sglang" + model = "seed-oss-36b" + custom_presets = { + "seed-oss-36b" = { + hf_model = "ByteDance-Seed/Seed-OSS-36B-Instruct" + vllm = { tool_call_parser = "seed_oss" } + } + } + } + + assert { + condition = join(" ", local.server_args) == "--model-path ByteDance-Seed/Seed-OSS-36B-Instruct --tp-size 1 --served-model-name seed-oss-36b --host 0.0.0.0 --port 8000" && local.image == "lmsysorg/sglang:v0.5.21" + error_message = "unexpected args: ${join(" ", local.server_args)}" + } +} + +run "custom_preset_engine_gpu_override" { + command = plan + variables { + engine = "sglang" + model = "seed-oss-36b" + custom_presets = { + "seed-oss-36b" = { + hf_model = "ByteDance-Seed/Seed-OSS-36B-Instruct" + gpu_type_ids = ["NVIDIA L40S"] + vllm = { gpu_type_ids = ["NVIDIA A100 80GB PCIe"] } + sglang = { gpu_type_ids = ["NVIDIA H200"] } + } + } + } + + assert { + condition = runpod_pod.server.gpu_type_ids == tolist(["NVIDIA H200"]) + error_message = "expected the sglang block's GPU list, got ${jsonencode(runpod_pod.server.gpu_type_ids)}" + } +} + +run "custom_preset_engine_gpu_override_vllm" { + command = plan + variables { + model = "seed-oss-36b" + custom_presets = { + "seed-oss-36b" = { + hf_model = "ByteDance-Seed/Seed-OSS-36B-Instruct" + gpu_type_ids = ["NVIDIA L40S"] + vllm = { gpu_type_ids = ["NVIDIA A100 80GB PCIe"] } + sglang = { gpu_type_ids = ["NVIDIA H200"] } + } + } + } + + assert { + condition = runpod_pod.server.gpu_type_ids == tolist(["NVIDIA A100 80GB PCIe"]) + error_message = "expected the vllm block's GPU list, got ${jsonencode(runpod_pod.server.gpu_type_ids)}" + } +} + +run "gemma_plan" { + command = plan + variables { model = "gemma-4-31b-it" } + + assert { + condition = strcontains(join(" ", local.server_args), "--tool-call-parser gemma4 --reasoning-parser gemma4 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja") + error_message = "unexpected args: ${join(" ", local.server_args)}" + } +} + +run "nemotron_plan" { + command = plan + variables { model = "nemotron-3.5-lightning-30b-a3b" } + + assert { + condition = local.hf_model == "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4" && strcontains(join(" ", local.server_args), "--tool-call-parser qwen3_xml --reasoning-parser nemotron_v3 --mamba-backend flashinfer") && strcontains(join(" ", local.server_args), "--moe-backend marlin") + error_message = "unexpected args: ${join(" ", local.server_args)}" + } +} + +run "unknown_model_fails" { + command = plan + variables { model = "nope" } + expect_failures = [var.model] +} + +# --- SGLang --- + +run "sglang_gpt_oss" { + command = plan + variables { engine = "sglang" } + + assert { + condition = local.image == "lmsysorg/sglang:v0.5.21" + error_message = "unexpected image: ${local.image}" + } + assert { + condition = join(" ", local.server_args) == "--model-path openai/gpt-oss-20b --tp-size 1 --context-length 131072 --tool-call-parser gpt-oss --reasoning-parser gpt-oss --served-model-name gpt-oss-20b --host 0.0.0.0 --port 8000" + error_message = "unexpected args: ${join(" ", local.server_args)}" + } + assert { + condition = slice(nonsensitive(runpod_pod.server.docker_start_cmd), 0, 3) == tolist(["python3", "-m", "sglang.launch_server"]) && contains(nonsensitive(runpod_pod.server.docker_start_cmd), "--api-key") + error_message = "expected the wrapper to exec sglang.launch_server with --api-key" + } + assert { + condition = !contains(keys(nonsensitive(local.env)), "VLLM_API_KEY") + error_message = "VLLM_API_KEY should not be set for sglang" + } + assert { + condition = runpod_pod.server.name == "sglang-gpt-oss-20b" && local.bifrost_provider_name == "sgl" && nonsensitive(local.bifrost_provider_config.keys[0].sgl_key_config.url) == "https://abc123-8000.proxy.runpod.net" + error_message = "expected an sgl provider entry" + } +} + +run "sglang_teardown_disabled" { + command = plan + variables { + engine = "sglang" + teardown_after_minutes = 0 + } + + assert { + condition = runpod_pod.server.docker_entrypoint == tolist(["python3", "-m", "sglang.launch_server"]) && nonsensitive(runpod_pod.server.docker_start_cmd)[0] == "--model-path" + error_message = "expected sglang.launch_server as the entrypoint" + } +} + +run "sglang_parsers_differ_from_vllm" { + command = plan + variables { + engine = "sglang" + model = "nemotron-3.5-lightning-30b-a3b" + gpu_memory_utilization = 0.85 + } + + assert { + condition = join(" ", local.server_args) == "--model-path nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 --tp-size 1 --mem-fraction-static 0.85 --context-length 65536 --tool-call-parser qwen3_coder --reasoning-parser nemotron_3 --mamba-ssm-dtype float16 --served-model-name nemotron-3.5-lightning-30b-a3b --host 0.0.0.0 --port 8000" + error_message = "unexpected args: ${join(" ", local.server_args)}" + } +} + +run "sglang_nightly" { + command = plan + variables { + engine = "sglang" + model = "qwen3.6-35b-a3b" + channel = "nightly" + } + + assert { + condition = local.image == "lmsysorg/sglang:dev" + error_message = "unexpected image: ${local.image}" + } +} + +run "sglang_rejects_cuda129" { + command = plan + variables { + engine = "sglang" + cuda_version = "12.9" + } + expect_failures = [var.cuda_version] +} + +# --- Decision models --- + +run "decision_clef_flash_vllm" { + command = plan + variables { model = "clef-flash" } + + assert { + condition = local.kind == "decision" && local.image == "vllm/vllm-openai:nightly" + error_message = "unexpected kind/image: ${local.kind} ${local.image}" + } + assert { + condition = join(" ", local.server_args) == "Cloudflare/clef-flash --tensor-parallel-size 1 --gpu-memory-utilization 0.92 --max-model-len 16384 --served-model-name clef-flash --host 0.0.0.0 --port 8000" + error_message = "unexpected args: ${join(" ", local.server_args)}" + } + assert { + condition = local.bifrost_provider_name == "vllm" && local.bifrost_provider_config.keys[0].models == ["clef-flash"] && local.bifrost_provider_config.keys[0].vllm_key_config.model_name == "clef-flash" + error_message = "expected a vllm key pinned to the served model" + } +} + +run "decision_clef_flash_sglang" { + command = plan + variables { + engine = "sglang" + model = "clef-flash" + } + + assert { + condition = local.image == "lmsysorg/sglang:dev" && join(" ", local.server_args) == "--model-path Cloudflare/clef-flash --tp-size 1 --context-length 16384 --served-model-name clef-flash --host 0.0.0.0 --port 8000" + error_message = "unexpected image/args: ${local.image} ${join(" ", local.server_args)}" + } + assert { + condition = local.bifrost_provider_name == "sgl" && local.bifrost_provider_config.keys[0].models == ["clef-flash"] && nonsensitive(local.bifrost_provider_config.keys[0].sgl_key_config.url) == "https://abc123-8000.proxy.runpod.net" + error_message = "expected an sgl key pinned to the served model" + } +} + +run "decision_pplx_sglang" { + command = plan + variables { + engine = "sglang" + model = "pplx-decider-v1.1-27b" + } + + assert { + condition = join(" ", local.server_args) == "--model-path perplexity-ai/pplx-decider-v1.1-27b --tp-size 1 --served-model-name pplx-decider-v1.1-27b --host 0.0.0.0 --port 8000" + error_message = "unexpected args: ${join(" ", local.server_args)}" + } +} + +run "decision_d1_vllm" { + command = plan + variables { model = "d1-3b" } + + assert { + condition = join(" ", local.server_args) == "LiquidAI/d1-3B --tensor-parallel-size 1 --gpu-memory-utilization 0.92 --served-model-name d1-3b --host 0.0.0.0 --port 8000" + error_message = "unexpected args: ${join(" ", local.server_args)}" + } +} + +run "decision_pplx_rejects_vllm" { + command = plan + variables { model = "pplx-decider-v1.1-27b" } + expect_failures = [var.engine] +} + +run "decision_d1_rejects_sglang" { + command = plan + variables { + engine = "sglang" + model = "d1-3b" + } + expect_failures = [var.engine] +} + +run "custom_decision_preset" { + command = plan + variables { + engine = "sglang" + model = "my-decider" + custom_presets = { + "my-decider" = { + kind = "decision" + hf_model = "Qwen/Qwen3.5-35B-A3B" + } + } + } + + assert { + condition = local.kind == "decision" && join(" ", local.server_args) == "--model-path Qwen/Qwen3.5-35B-A3B --tp-size 1 --served-model-name my-decider --host 0.0.0.0 --port 8000" + error_message = "unexpected kind/args: ${local.kind} ${join(" ", local.server_args)}" + } +} +# --- Ollama --- + +run "custom_decision_preset_vllm_defaults_to_nightly" { + command = plan + variables { + model = "my-decider" + custom_presets = { + "my-decider" = { + kind = "decision" + hf_model = "Qwen/Qwen3.5-35B-A3B" + } + } + } + + assert { + condition = local.image == "vllm/vllm-openai:nightly" + error_message = "a custom vLLM decision preset needs a build with /v1/systemone, got ${local.image}" + } +} + +run "custom_decision_preset_vllm_keeps_explicit_channel" { + command = plan + variables { + model = "my-decider" + custom_presets = { + "my-decider" = { + kind = "decision" + hf_model = "Qwen/Qwen3.5-35B-A3B" + vllm = { channel = "stable" } + } + } + } + + assert { + condition = local.image == "vllm/vllm-openai:v0.30.0" + error_message = "an explicit channel must win, got ${local.image}" + } +} + +run "custom_inference_preset_vllm_stays_stable" { + command = plan + variables { + model = "my-chat" + custom_presets = { + "my-chat" = { hf_model = "Qwen/Qwen3.5-35B-A3B" } + } + } + + assert { + condition = local.image == "vllm/vllm-openai:v0.30.0" + error_message = "inference presets keep the stable channel, got ${local.image}" + } +} + +run "ollama_gpt_oss" { + command = plan + variables { engine = "ollama" } + + assert { + condition = local.image == "ollama/ollama:0.40.4" && join(" ", local.server_args) == "gpt-oss:20b gpt-oss-20b" + error_message = "unexpected image/args: ${local.image} ${join(" ", local.server_args)}" + } + assert { + condition = runpod_pod.server.docker_entrypoint[0] == "/bin/bash" && slice(nonsensitive(runpod_pod.server.docker_start_cmd), 0, 2) == tolist(["/bin/bash", "-c"]) && strcontains(nonsensitive(runpod_pod.server.docker_start_cmd)[2], "ollama pull") && slice(nonsensitive(runpod_pod.server.docker_start_cmd), 3, 6) == tolist(["ollama-serve", "gpt-oss:20b", "gpt-oss-20b"]) + error_message = "expected the teardown wrapper to exec ollama-serve.sh with tag and served name" + } + assert { + condition = nonsensitive(local.env)["OLLAMA_KEEP_ALIVE"] == "-1" && nonsensitive(local.env)["OLLAMA_CONTEXT_LENGTH"] == "131072" && nonsensitive(local.env)["OLLAMA_MODELS"] == "/workspace/ollama" && contains(keys(nonsensitive(local.env)), "PROXY_API_KEY") && !contains(keys(nonsensitive(local.env)), "VLLM_API_KEY") + error_message = "unexpected ollama env" + } + assert { + condition = local.allowed_cuda_versions == tolist(["12.8", "12.9", "13.0"]) && runpod_pod.server.gpu_type_ids[0] == "NVIDIA RTX A5000" + error_message = "unexpected cuda versions or GPUs" + } + assert { + condition = local.bifrost_provider_name == "ollama" && local.bifrost_provider_config.keys[0].models == ["gpt-oss-20b"] && nonsensitive(local.bifrost_provider_config.keys[0].ollama_key_config.url) == "https://abc123-8000.proxy.runpod.net" + error_message = "expected an ollama key pinned to the served model" + } +} + +run "ollama_teardown_disabled" { + command = plan + variables { + engine = "ollama" + teardown_after_minutes = 0 + } + + assert { + condition = runpod_pod.server.docker_entrypoint[3] == "ollama-serve" && nonsensitive(runpod_pod.server.docker_start_cmd) == tolist(["gpt-oss:20b", "gpt-oss-20b"]) + error_message = "expected ollama-serve.sh as the entrypoint" + } +} + +run "ollama_engine_gpu_override" { + command = plan + variables { + engine = "ollama" + model = "glm-4.7-flash" + } + + assert { + condition = runpod_pod.server.gpu_type_ids[0] == "NVIDIA RTX PRO 4500 Blackwell" && local.server_args[0] == "glm-4.7-flash:q4_K_M" + error_message = "expected the ollama block's 32GB GPU list" + } +} + +run "ollama_model_override" { + command = plan + variables { + engine = "ollama" + ollama_model = "gpt-oss:20b-q8_0" + } + + assert { + condition = join(" ", local.server_args) == "gpt-oss:20b-q8_0 gpt-oss-20b" + error_message = "ollama_model override ignored" + } +} + +run "ollama_decision_nimble" { + command = plan + variables { + engine = "ollama" + model = "nimble" + } + + assert { + condition = local.kind == "decision" && join(" ", local.server_args) == "nimble:9b nimble" && nonsensitive(local.env)["OLLAMA_CONTEXT_LENGTH"] == "8192" + error_message = "unexpected nimble config" + } +} + +run "ollama_decision_clef_flash" { + command = plan + variables { + engine = "ollama" + model = "clef-flash" + } + + assert { + condition = join(" ", local.server_args) == "clef-flash:9b clef-flash" && runpod_pod.server.gpu_type_ids[0] == "NVIDIA RTX A5000" + error_message = "unexpected clef-flash config" + } +} + +run "nimble_rejects_vllm" { + command = plan + variables { model = "nimble" } + expect_failures = [var.engine] +} + +run "kimi_rejects_ollama" { + command = plan + variables { + engine = "ollama" + model = "kimi-linear-48b-a3b" + } + expect_failures = [var.engine] +} + +run "ollama_custom_preset_needs_tag" { + command = plan + variables { + engine = "ollama" + model = "seed-oss-36b" + custom_presets = { + "seed-oss-36b" = { hf_model = "ByteDance-Seed/Seed-OSS-36B-Instruct" } + } + } + expect_failures = [terraform_data.pod_spec] +} + +run "ollama_rejects_extra_args" { + command = plan + variables { + engine = "ollama" + extra_args = ["--foo"] + } + expect_failures = [terraform_data.pod_spec] +} + +run "api_key_charset" { + command = plan + variables { api_key = "has space\"quote" } + expect_failures = [var.api_key] +} diff --git a/scripts/terraform/runpod/variables.tf b/scripts/terraform/runpod/variables.tf new file mode 100644 index 00000000000..eeec9c928a8 --- /dev/null +++ b/scripts/terraform/runpod/variables.tf @@ -0,0 +1,321 @@ +# --- Engine & model --- + +variable "engine" { + description = "Inference server: \"vllm\" (vllm/vllm-openai), \"sglang\" (lmsysorg/sglang), or \"ollama\" (ollama/ollama)." + type = string + default = "vllm" + + validation { + condition = contains(["vllm", "sglang", "ollama"], var.engine) + error_message = "engine must be \"vllm\", \"sglang\", or \"ollama\"." + } + validation { + condition = !contains(keys(local.presets), var.model) || try(local.presets[var.model][var.engine], null) != null + error_message = "Preset ${var.model} has no ${var.engine} config; pick another engine or add a custom preset." + } +} + +variable "model" { + description = "Preset key from presets.tf or custom_presets (e.g. gpt-oss-20b, qwen3.6-35b-a3b, or a decision model like clef-flash)." + type = string + default = "gpt-oss-20b" + + validation { + condition = contains(concat(keys(local.builtin_presets), keys(var.custom_presets)), var.model) + error_message = "Unknown model preset. Built-in: ${join(", ", keys(local.builtin_presets))}; or add it to custom_presets." + } +} + +variable "custom_presets" { + description = "Extra presets merged over the built-in ones; select with var.model. Custom presets run on every engine; parsers go in the vllm/sglang blocks, the Ollama tag in the ollama block." + type = map(object({ + hf_model = string + kind = optional(string, "inference") + served_model_name = optional(string) + trust_remote_code = optional(bool, false) + max_model_len = optional(number) + gpu_count = optional(number, 1) + gpu_type_ids = optional(list(string), ["NVIDIA L40", "NVIDIA RTX 6000 Ada Generation", "NVIDIA L40S", "NVIDIA A100 80GB PCIe"]) + env = optional(map(string), {}) + vllm = optional(object({ + tool_call_parser = optional(string) + reasoning_parser = optional(string) + channel = optional(string) + extra_args = optional(list(string), []) + gpu_type_ids = optional(list(string)) + }), {}) + sglang = optional(object({ + tool_call_parser = optional(string) + reasoning_parser = optional(string) + channel = optional(string, "stable") + extra_args = optional(list(string), []) + gpu_type_ids = optional(list(string)) + }), {}) + ollama = optional(object({ + model = optional(string) + gpu_type_ids = optional(list(string)) + }), {}) + })) + default = {} + + validation { + condition = alltrue([for p in values(var.custom_presets) : p.gpu_count > 0 && floor(p.gpu_count) == p.gpu_count && (p.max_model_len == null ? true : p.max_model_len > 0 && floor(p.max_model_len) == p.max_model_len)]) + error_message = "custom_presets gpu_count and max_model_len must be positive whole numbers (max_model_len may be omitted)." + } + validation { + condition = alltrue([for p in values(var.custom_presets) : contains(["inference", "decision"], p.kind)]) + error_message = "custom_presets kind must be \"inference\" or \"decision\"." + } +} + +variable "hf_model" { + description = "Override the preset's HF repo (e.g. swap to an FP8/AWQ variant). vLLM and SGLang only." + type = string + default = null +} + +variable "ollama_model" { + description = "Override the preset's Ollama library tag (e.g. gpt-oss:20b, or hf.co//:). Ollama only." + type = string + default = null +} + +variable "served_model_name" { + description = "Override the model name exposed on /v1/models." + type = string + default = null +} + +variable "tool_call_parser" { + description = "Override the preset's --tool-call-parser. Empty string disables auto tool choice. Ignored by Ollama." + type = string + default = null +} + +variable "reasoning_parser" { + description = "Override the preset's --reasoning-parser. Empty string disables it. Ignored by Ollama." + type = string + default = null +} + +variable "max_model_len" { + description = "Override the preset's context length (vLLM --max-model-len, SGLang --context-length, Ollama OLLAMA_CONTEXT_LENGTH)." + type = number + default = null + + validation { + condition = var.max_model_len == null ? true : var.max_model_len > 0 && floor(var.max_model_len) == var.max_model_len + error_message = "max_model_len must be a positive whole number or null." + } +} + +variable "gpu_memory_utilization" { + description = "vLLM --gpu-memory-utilization (null means 0.92) or SGLang --mem-fraction-static (null lets SGLang size it)." + type = number + default = null + + validation { + condition = var.gpu_memory_utilization == null ? true : var.gpu_memory_utilization > 0 && var.gpu_memory_utilization <= 1 + error_message = "gpu_memory_utilization must be in (0, 1]." + } +} + +variable "extra_args" { + description = "Extra server args appended after the preset's (e.g. [\"--kv-cache-dtype\", \"fp8\"]). Flag names differ per engine." + type = list(string) + default = [] +} + +variable "extra_env" { + description = "Extra container env vars merged over the preset's." + type = map(string) + default = {} +} + +variable "hf_token" { + description = "HuggingFace token for gated models. Stored in state in plain text." + type = string + default = null + sensitive = true +} + +variable "api_key" { + description = "Server API key (vLLM VLLM_API_KEY, SGLang --api-key, Ollama's nginx proxy). Generated when null." + type = string + default = null + sensitive = true + + validation { + condition = var.api_key == null ? true : can(regex("^[A-Za-z0-9._~+/=-]+$", var.api_key)) + error_message = "api_key may only contain letters, digits and ._~+/=- (it is embedded in a bearer-token check)." + } +} + +# --- Image --- + +variable "channel" { + description = "\"stable\" or \"nightly\". Null uses the preset's channel for the engine." + type = string + default = null +} + +variable "vllm_version" { + description = "Stable release tag used when channel is stable. v0.30.0 is the base of RunPod's own vLLM worker image, so its layers are the likeliest to be cached on hosts." + type = string + default = "v0.30.0" +} + +variable "sglang_version" { + description = "Stable lmsysorg/sglang tag used when channel is stable. v0.5.21 is the first with /v1/systemone (CUDA 13 only)." + type = string + default = "v0.5.21" +} + +variable "ollama_version" { + description = "ollama/ollama tag used when channel is stable. 0.40.0+ serves all of nimble, tev1, clef and clef-flash on /v1/systemone." + type = string + default = "0.40.4" +} + +variable "cuda_version" { + description = "Image CUDA build: \"13.0\" or \"12.9\" (12.9 is vLLM only: it runs on more RunPod hosts, but the v0.30.0+ cu129 images ship a mismatched torch, vllm#59157)." + type = string + default = "13.0" + + validation { + condition = contains(["12.9", "13.0"], var.cuda_version) + error_message = "cuda_version must be \"12.9\" or \"13.0\"." + } + validation { + condition = var.cuda_version == "13.0" || var.engine == "vllm" + error_message = "SGLang publishes no CUDA 12.9 images after v0.5.19; use cuda_version = \"13.0\"." + } +} + +variable "image" { + description = "Full image override, e.g. vllm/vllm-openai:nightly-, vllm/vllm-openai:qwen38, or lmsysorg/sglang:nightly-dev--." + type = string + default = null +} + +variable "allowed_cuda_versions" { + description = "Host CUDA versions to accept. Null derives from cuda_version." + type = list(string) + default = null +} + +# --- Pod --- + +variable "pod_name" { + description = "Pod name. Defaults to -." + type = string + default = null +} + +variable "gpu_type_ids" { + description = "RunPod GPU type ids in preference order. Null uses the preset's list." + type = list(string) + default = null +} + +variable "gpu_type_priority" { + description = "\"custom\" rents GPU types in list order (presets order them by $/hr, lowest first); \"availability\" lets RunPod pick." + type = string + default = "custom" +} + +variable "gpu_count" { + description = "GPUs on the pod. Null uses the preset's count." + type = number + default = null + + validation { + condition = var.gpu_count == null ? true : var.gpu_count > 0 && floor(var.gpu_count) == var.gpu_count + error_message = "gpu_count must be a positive whole number or null." + } +} + +variable "tensor_parallel_size" { + description = "--tensor-parallel-size. Null uses gpu_count." + type = number + default = null + + validation { + condition = var.tensor_parallel_size == null ? true : var.tensor_parallel_size > 0 && floor(var.tensor_parallel_size) == var.tensor_parallel_size + error_message = "tensor_parallel_size must be a positive whole number or null." + } +} + +variable "cloud_type" { + description = "SECURE or COMMUNITY." + type = string + default = "SECURE" +} + +variable "data_center_ids" { + description = "Restrict to these data centers (e.g. [\"US-TX-3\", \"US-KS-2\"]). Empty means any." + type = list(string) + default = [] +} + +variable "interruptible" { + description = "Rent as a spot pod." + type = bool + default = false +} + +variable "container_disk_gb" { + description = "Container disk size (wiped on restart)." + type = number + default = 50 +} + +variable "volume_gb" { + description = "Pod volume size mounted at /workspace; HF weights are cached here." + type = number + default = 150 +} + +variable "expose_tcp" { + description = "Also expose 8000/tcp for direct access, bypassing the proxy's 100s time-to-first-byte limit." + type = bool + default = false +} + +# --- Readiness --- + +variable "wait_for_ready" { + description = "Block apply until /v1/models answers." + type = bool + default = true +} + +variable "ready_timeout_minutes" { + description = "How long to wait for the server to come up (image pull + weight download + load)." + type = number + default = 45 +} + +# --- Teardown --- + +variable "teardown_after_minutes" { + description = "Pod tears itself down this many minutes after its first container start (0 disables)." + type = number + default = 60 + + validation { + condition = var.teardown_after_minutes >= 0 && floor(var.teardown_after_minutes) == var.teardown_after_minutes + error_message = "teardown_after_minutes must be a whole number >= 0." + } +} + +variable "teardown_action" { + description = "\"terminate\" deletes the pod (falls back to stop if the pod-scoped key can't delete); \"stop\" releases the GPU but keeps the volume." + type = string + default = "terminate" + + validation { + condition = contains(["terminate", "stop"], var.teardown_action) + error_message = "teardown_action must be \"terminate\" or \"stop\"." + } +} diff --git a/scripts/terraform/runpod/versions.tf b/scripts/terraform/runpod/versions.tf new file mode 100644 index 00000000000..92e1d0bb63c --- /dev/null +++ b/scripts/terraform/runpod/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.9" + + required_providers { + # runpod/runpod (official) fails to load its own schema as of 1.0.9, so use the community provider. + runpod = { + source = "decentralized-infrastructure/runpod" + version = "= 1.0.1" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + } +} + +# Reads RUNPOD_API_KEY from the environment. +provider "runpod" {}