diff --git a/site/content/library/clef.md b/site/content/library/clef.md index 011bb41..b0d0a39 100644 --- a/site/content/library/clef.md +++ b/site/content/library/clef.md @@ -22,10 +22,10 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `clef ## How it works -- **One sequence per request.** Ollaya builds the prompt exactly as Cloudflare's `encode_record` does: the state, then every question with its options as a schema. The head reads the hidden states over each question's instructions and each option, routes evidence from the state to every option, and scores all options of all questions at once. +- **One sequence per request.** The prompt is built exactly as Cloudflare's `encode_record` builds it: the state, then every question with its options as a schema. The head scores all options of all questions at once, from the hidden states over each question and option. - **Weights.** Cloudflare's four BF16 shards and the head's `joint_head.safetensors` download from Hugging Face, pinned to a commit and verified by sha256. Ollaya hosts only the ONNX graph (11 MB). - **Parity.** Ollaya's Rust runtime matches Cloudflare's own code in fp32 on CUDA: identical token ids and spans, and the same decision on all 571 test questions, probabilities within 6.3e-6. -- **fp16 tag.** `clef:flash-fp16` computes the trunk in fp16 on NVIDIA GPUs, with the norms, the recurrent state, the attention softmax and the heads in fp32 ([how](https://github.com/ollaya-dev/ollaya/blob/main/docs/decisions/0008-fp16-decoder-tags.md)): {{result clef:flash-fp16 gpu.p50_ms 0}} ms for five questions on the RTX 5090 instead of {{result clef:flash gpu.p50_ms 0}} ms, {{result clef:flash-fp16 parity.cuda.decisions_same 0}} of {{result clef:flash-fp16 parity.cuda.questions 0}} decisions the same as `clef:flash` on the parity set, and {{result clef:flash-fp16 typed.accuracy 3}} on typed decisions. On a CPU use `clef:flash`. +- **fp16 tag.** `clef:flash-fp16` runs the trunk in fp16 on NVIDIA GPUs ([how](https://github.com/ollaya-dev/ollaya/blob/main/docs/decisions/0008-fp16-decoder-tags.md)): {{result clef:flash-fp16 gpu.p50_ms 0}} ms for five questions on the RTX 5090 instead of {{result clef:flash gpu.p50_ms 0}} ms, {{result clef:flash-fp16 parity.cuda.decisions_same 0}} of {{result clef:flash-fp16 parity.cuda.questions 0}} decisions the same as `clef:flash` on the parity set, and {{result clef:flash-fp16 typed.accuracy 3}} on typed decisions. On a CPU use `clef:flash`. ## Limits diff --git a/site/content/library/clm.md b/site/content/library/clm.md index ec9164d..b936454 100644 --- a/site/content/library/clm.md +++ b/site/content/library/clm.md @@ -20,8 +20,8 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `clm` ## Speed -- **RTX 5090, through the HTTP API (Ollaya Bench):** the triage preset (five questions) on a new short message takes {{result clm:8b gpu.p50_ms 0}} ms at the median once the questions are cached, and under a millisecond for a repeated request. With nothing cached, five questions take about 0.5 s. -- **Memory:** the encoder's weights stay BF16, about 16 GB; use a GPU with 20 GB or more. On a 24-core CPU a request takes about 13 s. +- **RTX 5090, through the HTTP API (Ollaya Bench):** five questions on a new short message take {{result clm:8b gpu.p50_ms 0}} ms at the median once the questions are cached, under a millisecond for a repeated request and about 0.5 s with nothing cached. +- **Memory:** the encoder's weights stay BF16, about 16 GB; use a GPU with 20 GB or more. On the Threadripper 3960X (24 cores) a request takes {{result clm:8b cpu.p50_ms 0}} ms. ## How it works diff --git a/site/content/library/decider.md b/site/content/library/decider.md index 42115d3..20c1a55 100644 --- a/site/content/library/decider.md +++ b/site/content/library/decider.md @@ -28,7 +28,7 @@ ollaya run decider:4b --preset triage "My order never arrived and support ignore ollaya run decider:2b-vision --image shelf.png --questions '{"blocked":{"type":"noul","instructions":"Is the aisle blocked?"}}' "A photo from the warehouse camera." ``` -- **One PNG per request**, up to about one megapixel (1024x1024). The model's own preprocessing (Qwen's resize to multiples of 32 pixels, PIL's bicubic filter, normalization) is reproduced value for value, so a JPEG has to be converted to PNG first: Rust's JPEG decoders differ from the one the model's authors use by a few levels on some pixels. +- **One PNG per request**, up to about one megapixel (1024x1024). The model's preprocessing is reproduced value for value, so convert a JPEG to PNG first: Rust's JPEG decoders differ from the authors' by a few levels on some pixels. - **Up to 10 options** per question (v5's lettered prompt, A to J). Text-only requests work too. - **Parity.** On 404 test questions, 12 of them with images: identical resized pixels, token ids and positions, the same decision on every question, and option logits within 1.6e-4 of the Python reference, on CPU and CUDA. - **Speed.** On an RTX 4090, five questions about a 256x240 image take about 200 ms, and about 0.9 s about a 1000x750 one. On the CPU, 3 to 16 seconds. @@ -38,19 +38,19 @@ ollaya run decider:2b-vision --image shelf.png --questions '{"blocked":{"type":" ## Speed -- **RTX 5090, through the HTTP API (Ollaya Bench):** a five-question request with a short state takes {{result decider:0.8b gpu.p50_ms 0}} ms on `0.8b`, {{result decider:2b gpu.p50_ms 0}} ms on `2b` and {{result decider:4b gpu.p50_ms 0}} ms on `4b` at the median. That is slower than the encoder models (`laya:en`: {{result laya:en gpu.p50_ms 0}} ms). `0.8b` and `2b` are still faster than hosted Jev (236–276 ms). +- **RTX 5090, through the HTTP API (Ollaya Bench):** five questions with a short state take {{result decider:0.8b gpu.p50_ms 0}} ms on `0.8b`, {{result decider:2b gpu.p50_ms 0}} ms on `2b` and {{result decider:4b gpu.p50_ms 0}} ms on `4b` at the median: slower than the encoders (`laya:en`: {{result laya:en gpu.p50_ms 0}} ms), and for `0.8b` and `2b` still faster than hosted Jev (236–276 ms). - **State length:** cost grows with the number of options times the context length. - **Memory:** on a 24 GB GPU, `2b` handles states up to about 8k tokens. Longer states spill out of GPU memory and become very slow. ## How it works - **Weights.** They come from Mapika's own `model.safetensors`, downloaded from Hugging Face, pinned to a commit and verified by sha256. Ollaya hosts only the ONNX graph (8 to 11 MB). -- **Precision.** Every model computes in fp32 on the BF16 weights. `0.8b` and `2b` widen the weights to fp32 when they load. `4b` keeps them BF16, half the memory, and widens each layer just before it runs; its answers are the same, and a forward pass takes about 12% longer. +- **Precision.** fp32 compute on the BF16 weights. `0.8b` and `2b` widen the weights when they load; `4b` keeps them BF16 (half the memory) and widens each layer just before it runs, with the same answers and about 12% more time per pass. - **Calibration.** `4b` (v2.1) has one fitted temperature per answer type: choice 1.110, yes/no 1.560, score 1.287. `2b` and `0.8b` have one each. - **Parity.** Ollaya's Rust runtime matches the Python reference exactly: identical token ids and answer-slot positions, and the same decision on every test question, on CPU and CUDA. ## Limits -- **Memory.** On the GPU, with requests of about 1,400 tokens, `2b` took about 13 GB and `4b` about 14 GB (`4b` keeps its 8.4 GB of weights BF16). All three are slow on the CPU, where `4b` peaked at 21 GB on the same requests; `4b` belongs on a GPU. +- **Memory.** With requests of about 1,400 tokens, `2b` took about 13 GB of GPU memory and `4b` about 14 GB. All three are slow on the CPU, where `4b` peaked at 21 GB; `4b` belongs on a GPU. - **Score criteria.** Criteria given as an object instead of a list are rejected, as TypeSafe's API does. - **Instructions.** Every question needs `instructions`. Mapika's newer package also accepts a yes/no question without them; Ollaya rejects it. diff --git a/site/content/library/decima.md b/site/content/library/decima.md index bb3a273..cdd9e07 100644 --- a/site/content/library/decima.md +++ b/site/content/library/decima.md @@ -32,5 +32,5 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `deci ## Limits - **State.** 512 tokens per question (2,048 for `decima:agent`), the question included. `/v1/systemone` answers a longer state with `STATE_TRUNCATED`, as the author's server does; `/api/decide` answers from its first tokens. -- **Options.** Each option row holds the question and the option in 64 tokens. When they don't fit, `decima:agent` keeps the option whole and shortens the question's copy beside it (the full question is still read with the state), as the author's 2.2 runtime does. `decima:base` and `decima:small` keep cutting the row on the right, which their training saw, except when that would make two options identical. 2 to 255 choices, 2 to 10 score levels. +- **Options.** 2 to 255 choices, 2 to 10 score levels. Each option row fits the question and the option in 64 tokens: `decima:agent` shortens the question's copy and keeps the option whole, as the author's 2.2 runtime does; `base` and `small` cut the row on the right, as in training, unless two options would become identical. - **Languages.** Multilingual. The author evaluated small in 20 languages (the weakest are Swahili and Hindi). For base, the author reports Persian about 5 points below English on the same tasks, and other languages less tested. diff --git a/site/content/library/decision.md b/site/content/library/decision.md index 51332ad..0078694 100644 --- a/site/content/library/decision.md +++ b/site/content/library/decision.md @@ -21,8 +21,8 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `deci ## Speed - **RTX 5090, through the HTTP API (Ollaya Bench):** a five-question request with a short message takes {{result decision:eos gpu.p50_ms 0}} ms at the median (`kev:0.8b` {{result kev:0.8b gpu.p50_ms 0}} ms, `decider:0.8b` {{result decider:0.8b gpu.p50_ms 0}} ms). -- **CPU:** the same request takes about 0.85 s on a Mac mini (M4 Pro), through the daemon. -- **State length:** every question reads the whole context again, so the cost grows with the number of questions times the context length. Five typed-decisions questions about a JSON state (about 1,050 tokens in all) take about 0.2 s on the RTX 4090, 0.84 s on the Mac mini and 2.1 s on a 24-core x86 CPU, measured in the runtime without HTTP. +- **CPU:** the same request takes {{result decision:eos cpu.p50_ms 0}} ms on the Threadripper 3960X (24 cores). +- **State length:** every question reads the whole context again, so the cost grows with the number of questions times the context length. ## How it works diff --git a/site/content/library/jevk5.md b/site/content/library/jevk5.md index 416a0ae..00eb8ab 100644 --- a/site/content/library/jevk5.md +++ b/site/content/library/jevk5.md @@ -21,8 +21,8 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `jevk ## Speed - **RTX 5090, through the HTTP API (Ollaya Bench):** the triage preset (five questions) on a short customer message takes {{result jevk5:4b gpu.p50_ms 0}} ms at the median. -- **RTX 4090, in the runner:** on the parity fixtures, whose states are longer, five questions take about 288 ms at the median. Every question is its own pass over the whole prompt (see below), so the time grows with the state times the number of questions. -- **CPU:** not measured by Ollaya. The author reports about 0.6 s per short decision on a 32- or 48-thread CPU server. +- **State length:** every question is its own pass over the whole prompt (see below), so the time grows with the state times the number of questions. +- **CPU:** {{result jevk5:4b cpu.p50_ms 0}} ms for the same request on the Threadripper 3960X (24 cores). The author reports about 0.6 s per short decision on a 32- or 48-thread CPU server. ## How it works @@ -31,7 +31,7 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `jevk - **Options.** Up to 16 per question, labelled `A` to `P`. A yes/no question reads `true` as `A` and `false` as `B`. - **Calibration.** Temperature 1.22 over the letters, the value the author gives for the v0.3 4B files. - **Engine.** llama.cpp v0.5.0, ggml-org's own release build, runs inside Ollaya's runner process. It uses an NVIDIA GPU (CUDA) or an Apple silicon GPU (Metal) when the model fits, and the CPU otherwise. -- **Parity.** Ollaya's runner matches stock llama.cpp (`llama-server` of the same build, on the same file) on CUDA (RTX 4090): the same decision on all 593 test questions, probabilities within 1.6e-6. The prompts are byte-identical to the author's `jevk5.prompt` on all 450 fixture prompts. Not checked yet: the CPU, Metal and Windows. +- **Parity.** On CUDA, Ollaya's runner matches stock `llama-server` of the same build on the same file: the same decision on all {{result jevk5:4b parity.cuda.questions 0}} test questions, probabilities within 1.6e-6, with prompts byte-identical to the author's `jevk5.prompt`. Not checked yet: the CPU, Metal and Windows. ## Limits diff --git a/site/content/library/kev.md b/site/content/library/kev.md index 3d15010..da5c354 100644 --- a/site/content/library/kev.md +++ b/site/content/library/kev.md @@ -29,10 +29,10 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `kev` - **One row per question.** The state comes first, then the question, then one span per option, then a decide token. The pointer head compares each option's closing token with the decide token. - **Calibrated.** The scores are divided by the temperature each checkpoint ships, fitted upstream on development data: 2.35 (`0.8b`), 2.41 (`4b`), 2.30 (`9b`). -- **Weights.** The files download from Hugging Face, pinned to a commit and verified by sha256: the Qwen3.5 base weights from Qwen's repository, and the adapter and `head.pt` from Jared Palmer's. The adapter is applied at run time, not merged, so every file stays byte for byte the author's. Ollaya hosts only the ONNX graph (10 to 14 MB). +- **Weights.** From Hugging Face, pinned to a commit and verified by sha256: the Qwen3.5 base from Qwen, the adapter and `head.pt` from Jared Palmer. The adapter is applied at run time, not merged; Ollaya hosts only the ONNX graph (10 to 14 MB). - **Precision.** The fp32 tags compute in fp32 on the BF16 base weights. `0.8b` widens them when it loads; `4b` and `9b` keep them BF16, half the memory, and widen each layer just before it runs. - **Parity.** Ollaya's Rust runtime matches upstream Kev exactly: identical token rows and option positions, and the same decision on every test question, on CPU and CUDA. -- **fp16 tags.** `kev:4b-fp16` and `kev:9b-fp16` compute the trunk in fp16 on NVIDIA GPUs, with the norms, the recurrent state, the attention softmax and the heads in fp32 ([how](https://github.com/ollaya-dev/ollaya/blob/main/docs/decisions/0008-fp16-decoder-tags.md)). On the RTX 5090 five questions take {{result kev:4b-fp16 gpu.p50_ms 0}} and {{result kev:9b-fp16 gpu.p50_ms 0}} ms (fp32: {{result kev:4b gpu.p50_ms 0}} and {{result kev:9b gpu.p50_ms 0}} ms); on the parity set {{result kev:4b-fp16 parity.cuda.decisions_same 0}} and {{result kev:9b-fp16 parity.cuda.decisions_same 0}} of {{result kev:4b-fp16 parity.cuda.questions 0}} decisions are the same as the fp32 tag's, and on typed decisions they score {{result kev:4b-fp16 typed.accuracy 3}} and {{result kev:9b-fp16 typed.accuracy 3}}. `kev:0.8b` has no fp16 tag: its fp32 graph is as fast. On a CPU use the fp32 tags. +- **fp16 tags.** `kev:4b-fp16` and `kev:9b-fp16` run the trunk in fp16 on NVIDIA GPUs ([how](https://github.com/ollaya-dev/ollaya/blob/main/docs/decisions/0008-fp16-decoder-tags.md)). On the RTX 5090: {{result kev:4b-fp16 gpu.p50_ms 0}} and {{result kev:9b-fp16 gpu.p50_ms 0}} ms for five questions (fp32: {{result kev:4b gpu.p50_ms 0}} and {{result kev:9b gpu.p50_ms 0}} ms), {{result kev:4b-fp16 parity.cuda.decisions_same 0}} and {{result kev:9b-fp16 parity.cuda.decisions_same 0}} of {{result kev:4b-fp16 parity.cuda.questions 0}} parity decisions the same as fp32, {{result kev:4b-fp16 typed.accuracy 3}} and {{result kev:9b-fp16 typed.accuracy 3}} on typed decisions. `kev:0.8b` has no fp16 tag (its fp32 graph is as fast). On a CPU use the fp32 tags. ## Limits diff --git a/site/content/library/nimble.md b/site/content/library/nimble.md index c1bc7a6..797f3c6 100644 --- a/site/content/library/nimble.md +++ b/site/content/library/nimble.md @@ -27,12 +27,12 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `nimb ## How it works -- **Prompt.** Ollaya builds Nimble's prompt exactly as the author's code does (`serving_schema.prepare_prompts` in the model repository), and maps TypeSafe questions to Nimble's schema the way the author's own `/v1/systemone` server does: noul becomes a true/false field, choice an enum of its labels, score an enum of its levels. +- **Prompt.** Built exactly as the author's `serving_schema.prepare_prompts` builds it, with TypeSafe questions mapped as the author's `/v1/systemone` server maps them: noul to a true/false field, choice to an enum of its labels, score to an enum of its levels. - **Calibrated.** Probabilities use the author's recommended temperature, 2.179 (`temperature_config.json`). A plain softmax of the code logits, what Ollama returns, is overconfident. - **Up to 255 options.** Questions with more than 26 options use the author's serving extension: two-letter codes and the "short code" prompt. -- **Weights.** Qwen's Qwen3.5-9B shards and Bespoke Labs' adapter download from Hugging Face, pinned to a commit and verified by sha256. The adapter is applied at run time, not merged, so every file stays byte for byte the authors'. Ollaya hosts only the ONNX graph (14 MB). +- **Weights.** Qwen3.5-9B from Qwen and Bespoke Labs' adapter, from Hugging Face, pinned to a commit and verified by sha256. The adapter is applied at run time, not merged; Ollaya hosts only the ONNX graph (14 MB). - **Parity.** Ollaya's Rust runtime matches the author's code in fp32 on CUDA: identical token rows, the same rejections and the same decision on all 492 test questions, probabilities within 6.5e-6. `nimble:9b-v3` matches the author's Decision Index engine the same way on 498 questions, probabilities within 1.1e-5. -- **fp16 tag.** `nimble:9b-fp16` computes the trunk in fp16 on NVIDIA GPUs, with the norms, the recurrent state, the attention softmax and the heads in fp32 ([how](https://github.com/ollaya-dev/ollaya/blob/main/docs/decisions/0008-fp16-decoder-tags.md)): {{result nimble:9b-fp16 gpu.p50_ms 0}} ms for five questions on the RTX 5090 instead of {{result nimble:9b gpu.p50_ms 0}} ms, {{result nimble:9b-fp16 parity.cuda.decisions_same 0}} of {{result nimble:9b-fp16 parity.cuda.questions 0}} decisions the same as `nimble:9b` on the parity set, and {{result nimble:9b-fp16 typed.accuracy 3}} on typed decisions. On a CPU use `nimble:9b`. +- **fp16 tag.** `nimble:9b-fp16` runs the trunk in fp16 on NVIDIA GPUs ([how](https://github.com/ollaya-dev/ollaya/blob/main/docs/decisions/0008-fp16-decoder-tags.md)): {{result nimble:9b-fp16 gpu.p50_ms 0}} ms for five questions on the RTX 5090 instead of {{result nimble:9b gpu.p50_ms 0}} ms, {{result nimble:9b-fp16 parity.cuda.decisions_same 0}} of {{result nimble:9b-fp16 parity.cuda.questions 0}} decisions the same as `nimble:9b` on the parity set, and {{result nimble:9b-fp16 typed.accuracy 3}} on typed decisions. On a CPU use `nimble:9b`. ## Limits diff --git a/site/content/library/pplx-decider.md b/site/content/library/pplx-decider.md index 4f1c1b6..bd0d89f 100644 --- a/site/content/library/pplx-decider.md +++ b/site/content/library/pplx-decider.md @@ -18,7 +18,7 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `pplx ## Speed -- **RTX 5090, over HTTP (Ollaya Bench):** five-question triage requests take {{result pplx-decider:flash gpu.p50_ms 0}} ms at the median, all questions in one forward pass; {{result pplx-decider:flash public.median_ms 0}} ms per question at the median on Bespoke Labs' public benchmark, {{result pplx-decider:flash typed.median_ms 0}} ms per five-question typed-decisions request. +- **RTX 5090, over HTTP (Ollaya Bench):** {{result pplx-decider:flash gpu.p50_ms 0}} ms at the median for five triage questions (one forward pass), {{result pplx-decider:flash public.median_ms 0}} ms per question on Bespoke Labs' public benchmark and {{result pplx-decider:flash typed.median_ms 0}} ms per five-question typed-decisions request. ## Accuracy @@ -26,11 +26,11 @@ Measured by Ollaya on an RTX 5090: {{result pplx-decider:flash public.macro_accu ## How it works -- **One sequence per request.** Every question reads the state and the other questions: the model's full-attention layers see the whole request, not only what comes before. The model card finds the flash model more accurate with one question per request on most multi-question tasks, so send related questions together and independent ones apart. +- **One sequence per request.** Every question sees the state and the other questions. The model card finds the flash model more accurate with one question per request on most multi-question tasks: send related questions together and independent ones apart. - **Calibrated.** The readout logits are divided by the temperature Perplexity fitted on held-out requests (1.55). - **Weights.** The files download from Hugging Face, pinned to a commit and verified by sha256, byte for byte Perplexity's. Ollaya hosts only the ONNX graph (11 MB). - **Parity.** Ollaya's Rust runtime matches upstream `pplx_decider` in fp32: identical token sequences, the same rejections and the same decision on all 696 test questions, probabilities within 6.8e-6. -- **fp16 tag.** `pplx-decider:flash-fp16` computes the trunk in fp16 on NVIDIA GPUs, with the norms, the recurrent state, the attention softmax and the heads in fp32 ([how](https://github.com/ollaya-dev/ollaya/blob/main/docs/decisions/0008-fp16-decoder-tags.md)): {{result pplx-decider:flash-fp16 gpu.p50_ms 0}} ms for five questions on the RTX 5090 instead of {{result pplx-decider:flash gpu.p50_ms 0}} ms, {{result pplx-decider:flash-fp16 parity.cuda.decisions_same 0}} of {{result pplx-decider:flash-fp16 parity.cuda.questions 0}} decisions the same as `pplx-decider:flash` on the parity set, and {{result pplx-decider:flash-fp16 typed.accuracy 3}} on typed decisions. On a CPU use `pplx-decider:flash`. +- **fp16 tag.** `pplx-decider:flash-fp16` runs the trunk in fp16 on NVIDIA GPUs ([how](https://github.com/ollaya-dev/ollaya/blob/main/docs/decisions/0008-fp16-decoder-tags.md)): {{result pplx-decider:flash-fp16 gpu.p50_ms 0}} ms for five questions on the RTX 5090 instead of {{result pplx-decider:flash gpu.p50_ms 0}} ms, {{result pplx-decider:flash-fp16 parity.cuda.decisions_same 0}} of {{result pplx-decider:flash-fp16 parity.cuda.questions 0}} decisions the same as `pplx-decider:flash` on the parity set, and {{result pplx-decider:flash-fp16 typed.accuracy 3}} on typed decisions. On a CPU use `pplx-decider:flash`. ## Limits diff --git a/site/content/library/qwen3guard.md b/site/content/library/qwen3guard.md index ef69ad4..3cc84a1 100644 --- a/site/content/library/qwen3guard.md +++ b/site/content/library/qwen3guard.md @@ -35,7 +35,7 @@ curl http://localhost:11435/api/decide -d '{"model": "qwen3guard", "state": "How - **RTX 5090, through the HTTP API (Ollaya Bench):** {{result qwen3guard:0.6b gpu.p50_ms 0}} ms at the median for a short message, with all four questions. - **Text length:** attention covers the whole text, so the cost grows faster than its length: about 70 ms for a 500-token text, 0.36 s for 2,000 tokens and 1 s for 4,000 tokens. -- **CPU:** about 1.7 s at the median for texts of a few hundred tokens, on a 24-core x86 CPU. +- **CPU:** {{result qwen3guard:0.6b cpu.p50_ms 0}} ms at the median for a short message, on the Threadripper 3960X (24 cores). ## How it works diff --git a/site/content/library/snap.md b/site/content/library/snap.md index de24c9c..3de0259 100644 --- a/site/content/library/snap.md +++ b/site/content/library/snap.md @@ -21,7 +21,7 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `snap ## How it works -- **Prompt.** Ollaya builds snap's prompt exactly as snap 0.5.0 does: identical on all 573 test prompts, token for token. A structured state is written in TOON, snap's compact rendering of JSON; the question goes before the state or after it as snap decides for the whole request. +- **Prompt.** Built exactly as snap 0.5.0 builds it, token for token on all 573 test prompts. A structured state is written in TOON, snap's compact JSON; the question goes before or after the state, as snap decides for the whole request. - **Options.** Each option is a letter, up to 26 per question, read in one pass. A yes/no question reads `A) Yes`, `B) No`. - **Calibration.** None: snap1-2b ships raw probabilities, and so does Ollaya. - **Engine.** llama.cpp v0.5.0, ggml-org's own build, on an NVIDIA GPU (CUDA), an Apple silicon GPU (Metal) or the CPU. About 3 GB of memory. diff --git a/site/content/library/strands-decider.md b/site/content/library/strands-decider.md index 4e4721a..b73cbe2 100644 --- a/site/content/library/strands-decider.md +++ b/site/content/library/strands-decider.md @@ -25,9 +25,9 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `stra - **One row per question.** The state comes first, then the question with a header for its type and one numbered line per option, then ``. The pointer head compares the last token of each option's line with the last token of the row. - **Calibrated.** The scores are divided by the temperature the authors fitted for each question type: 0.86 for choice, 1.83 for score and 1.07 for noul. -- **Weights.** The files download from Hugging Face, pinned to a commit and verified by sha256: the base weights from Qwen's repository, the adapter and `head.safetensors` from Strands Agents'. The adapter is applied at run time, not merged, so every file stays byte for byte the authors'. Ollaya hosts only the ONNX graph (10 MB). +- **Weights.** From Hugging Face, pinned to a commit and verified by sha256: the base from Qwen, the adapter and `head.safetensors` from Strands Agents. The adapter is applied at run time, not merged; Ollaya hosts only the ONNX graph (10 MB). - **Parity.** Ollaya's Rust runtime matches upstream `strands_decider` in fp32: identical token rows and option positions, the same rejections and the same decision on all 650 test questions, probabilities within 2.8e-6. -- **fp16 tag.** `strands-decider:2b-fp16` computes the trunk in fp16 on NVIDIA GPUs, with the norms, the recurrent state, the attention softmax and the heads in fp32 ([how](https://github.com/ollaya-dev/ollaya/blob/main/docs/decisions/0008-fp16-decoder-tags.md)): {{result strands-decider:2b-fp16 gpu.p50_ms 0}} ms for five questions on the RTX 5090 instead of {{result strands-decider:2b gpu.p50_ms 0}} ms, {{result strands-decider:2b-fp16 parity.cuda.decisions_same 0}} of {{result strands-decider:2b-fp16 parity.cuda.questions 0}} decisions the same as `strands-decider:2b` on the parity set, and {{result strands-decider:2b-fp16 typed.accuracy 3}} on typed decisions. On a CPU use `strands-decider:2b`. +- **fp16 tag.** `strands-decider:2b-fp16` runs the trunk in fp16 on NVIDIA GPUs ([how](https://github.com/ollaya-dev/ollaya/blob/main/docs/decisions/0008-fp16-decoder-tags.md)): {{result strands-decider:2b-fp16 gpu.p50_ms 0}} ms for five questions on the RTX 5090 instead of {{result strands-decider:2b gpu.p50_ms 0}} ms, {{result strands-decider:2b-fp16 parity.cuda.decisions_same 0}} of {{result strands-decider:2b-fp16 parity.cuda.questions 0}} decisions the same as `strands-decider:2b` on the parity set, and {{result strands-decider:2b-fp16 typed.accuracy 3}} on typed decisions. On a CPU use `strands-decider:2b`. ## Limits diff --git a/site/content/library/von.md b/site/content/library/von.md index 7c277a5..64abc91 100644 --- a/site/content/library/von.md +++ b/site/content/library/von.md @@ -19,8 +19,8 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `von` ## Speed - **RTX 5090, through the HTTP API (Ollaya Bench):** a five-question request with a short state takes {{result von:1.1 gpu.p50_ms 0}} ms at the median. -- **CPU:** the same request takes about 0.76 s on a 24-core x86 CPU. -- **State length:** every question reads the whole state, and attention cost grows faster than the sequence. Five questions about a 600-token state take about 0.17 s on the GPU and 5.6 s on the CPU; about a 2,200-token state, 0.78 s and 25 s. A state near 7,500 tokens takes about 7 s on the GPU. On the CPU a full 8,192-token sequence needs about 25 GB of memory. +- **CPU:** the same request takes {{result von:1.1 cpu.p50_ms 0}} ms on the Threadripper 3960X (24 cores). +- **State length:** every question reads the whole state, and attention cost grows faster than the sequence: five questions about a 2,200-token state take about 0.78 s on the GPU and 25 s on the CPU. On the CPU a full 8,192-token sequence needs about 25 GB of memory. ## How it works @@ -28,7 +28,7 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `von` - **State as text.** A JSON state is written the way Von was trained on it: one `key: value` line per top-level key, with Python's rendering of nested values. - **Yes/no without criteria.** A `noul` question without `true`/`false` descriptions adds a second sequence without the state. It measures the question's own lean towards yes or no, which is taken off the answer, as Von does. - **Calibration.** The temperature is computed per question from the answer's entropy, the state's length and the number of options, with Von's own fitted coefficients. -- **Weights.** Von's trained weights exist only in the author's `option_marker.pt`, a PyTorch archive whose tensors are stored uncompressed. Ollaya downloads it from Hugging Face, pinned to a commit and verified by sha256, and the ONNX graph reads the tensors in place by byte offset. Nothing in it is unpickled or executed, and Ollaya hosts only the graph (3 MB). +- **Weights.** Von's weights exist only in the author's `option_marker.pt`, a PyTorch archive with uncompressed tensors, downloaded from Hugging Face, pinned to a commit and verified by sha256. The ONNX graph reads the tensors in place by byte offset, so nothing is unpickled or executed; Ollaya hosts only the graph (3 MB). - **Parity.** Ollaya's Rust runtime reproduces upstream Von computed in float64: the same token ids and option markers, the same decision on every test question, and logits within 4.4e-4 on x86-64 CPU and CUDA. On Apple silicon's CPU one row is 1.1e-3 off, and every decision is still the same. ## Limits diff --git a/site/content/library/winnow.md b/site/content/library/winnow.md index 43991d8..dd88a2e 100644 --- a/site/content/library/winnow.md +++ b/site/content/library/winnow.md @@ -24,8 +24,8 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `winn ## Speed -- **RTX 5090, through the HTTP API (Ollaya Bench):** the triage preset (five questions) on a short customer message takes {{result winnow:e4b gpu.p50_ms 0}} ms on `e4b` and {{result winnow:12b gpu.p50_ms 0}} ms on `12b` at the median. On the parity fixtures, measured in the runner, the medians are 96 ms and 154 ms. -- **CPU:** `e4b` takes about 5.1 s for the same request on a 24-core x86 CPU. Use a GPU. +- **RTX 5090, through the HTTP API (Ollaya Bench):** the triage preset (five questions) on a short customer message takes {{result winnow:e4b gpu.p50_ms 0}} ms on `e4b` and {{result winnow:12b gpu.p50_ms 0}} ms on `12b` at the median. +- **CPU:** `e4b` takes {{result winnow:e4b cpu.p50_ms 0}} ms for the same request on the Threadripper 3960X (24 cores). Use a GPU. ## How it works @@ -34,11 +34,11 @@ Point any TypeSafe client at `http://localhost:11435` and set the model to `winn - **Labels.** Up to 64 options per question: `A` to `Z`, then two-letter labels, each a single token. - **Calibration.** `winnow:12b` uses temperature 1, the author's default: the author fitted no calibration for it. `winnow:e4b` uses 1.2574, the temperature the author fitted for its Q8_0 file on 778 held-out questions. - **Engine.** llama.cpp v0.5.0, ggml-org's own release build, runs inside Ollaya's runner process. It uses an NVIDIA GPU (CUDA) or an Apple silicon GPU (Metal) when the model fits, and the CPU otherwise. -- **Parity.** Ollaya's runner matches stock llama.cpp (`llama-server` of the same build, on the same file): the same decision on all 505 test questions, probabilities within 3.0e-6, on CUDA (RTX 4090, both models) and an x86-64 CPU (`e4b`). On Windows the CPU check covered 15 questions. Against the author's own server, `e4b` agrees on all 503 decisions. Not checked yet: Metal and the Apple CPU, Linux on ARM, and the full `12b` text suite on the CPU. +- **Parity.** Ollaya's runner matches stock `llama-server` of the same build on the same file: the same decision on all 505 test questions, probabilities within 3.0e-6, on CUDA (both models) and an x86-64 CPU (`e4b`). Against the author's own server, `e4b` agrees on all 503 decisions. Not checked yet: Metal, the Apple CPU, Linux on ARM and the full `12b` text suite on the CPU. ## Limits -- **Size.** These are large language models. `winnow:12b` needs about 14 GB of GPU memory and `winnow:e4b` about 9 GB at the 8,192-token context; on the CPU they are much slower than the encoder models. `winnow:12b-vision` needs about 0.4 GB more: on an RTX 5090, CUDA context included, it peaked at 16,263 MiB against 15,877 MiB for `winnow:12b`, so a 16 GB GPU is marginal for it. +- **Size.** `winnow:12b` needs about 14 GB of GPU memory and `winnow:e4b` about 9 GB at the 8,192-token context; on the CPU both are much slower than the encoders. `winnow:12b-vision` needs about 0.4 GB more than `winnow:12b`, so a 16 GB GPU is marginal for it. - **Context.** State, question and options share 8,192 tokens. A longer state is cut to 6,144 tokens. - **Options.** 2 to 64 options per question and up to 256 questions per request. Probabilities are conditional on the options offered. - **Images.** The vision tags read images: up to 16 PNGs per request, sharing the 8,192-token context with the state. `/v1/*` and the other tags are text only. diff --git a/site/docs/faq.md b/site/docs/faq.md index 7d666c8..43b4c97 100644 --- a/site/docs/faq.md +++ b/site/docs/faq.md @@ -24,7 +24,7 @@ Ollama 0.35 (September 2026) added `/v1/systemone` for two decoder models, Bespo | | Ollaya | Ollama 0.40 | |---|---|---| -| Models | 23 families: encoders (`laya`, `nli`, `gliclass`, `von`, `decima`) and decoders (`winnow`, `clef`, `kev`, `decider`, `nimble`, `pplx-decider`, `strands-decider`, `jeb`, `jeeves`, `cygnet`, `snap`, `arbiter` and more) | Nine decision models, all decoders except Laya | +| Models | 23 families: encoders (`laya`, `nli`, `gliclass`, `von`, `decima`) and decoders, listed below | Nine decision models, all decoders except Laya | | Encoders | Read every question in one forward pass. `laya:en` answers five questions in {{result laya:en gpu.p50_ms 0}} ms on an RTX 5090 | Laya only | | Probabilities | Calibrated with each model's fitted temperatures, which you can refit on your own data in a [Modelfile](/docs/modelfile#calibration) | Mostly the softmax of the raw label scores: Nimble's and Strands Decider's fitted temperatures are not applied. `confidence` is one minus the normalized entropy | | Limits | TypeSafe's: 1 to 256 questions, 2 to 255 options, 2 to 10 score levels | 1 to 64 questions; 2 to 26 options for Nimble, Tev1 and Clef (255 for Kev and Strands Decider); 2 to 10 score levels; 64 KiB of text | diff --git a/site/src/bench/pages.tsx b/site/src/bench/pages.tsx index e11f43f..b210246 100644 --- a/site/src/bench/pages.tsx +++ b/site/src/bench/pages.tsx @@ -41,7 +41,7 @@ function dateLabel(iso: string | undefined): string { } const shortLicense = (l: string) => - l.startsWith('Apache-2.0 (') ? 'Apache-2.0 + Gemma' : l.replace('LFM Open License v1.0', 'LFM Open 1.0') + l.startsWith('Apache-2.0 (') ? 'Apache-2.0 + Gemma' : l.replace('LFM Open License v1.0', 'LFM Open 1.0').replace(/ \([^)]*\)/g, '').replace(' and ', ' + ') // --------------------------------------------------------------------------------------------- // Leaderboard @@ -111,23 +111,23 @@ function LeaderTable({ d }: { d: Device }) { {pct1(r.public?.macro_accuracy)} - + {pct1(r.typed?.accuracy)} {r.typed?.trained_on_split ? * : null} {num3(r.public?.pooled_ece)} - + {s?.error ? failed : fmtMs(s?.p50_ms)} - + {fmtMb(mem)} {r.params ?? '–'} - + {shortLicense(r.license)} @@ -517,7 +517,7 @@ export function ModelPage({ r, origin, main }: { r: Row; origin: string; main: s {p.device_detail ?? p.device} {p.decisions ?? (p.decisions_same !== undefined ? `${p.decisions_same}/${p.questions}` : '–')} - {p.max_logit_diff == null ? '–' : p.max_logit_diff.toExponential(1)} + {p.max_logit_diff == null ? '–' : p.max_logit_diff.toExponential(1)} {p.pass === true ? ( pass{p.gate === 'fp16' ? ' (fp16 gate)' : ''} @@ -627,7 +627,7 @@ export function MethodologyPage() {
-
    +
      {[ ['Measured, not modeled', 'Every latency and memory figure is measured on a named machine. No adjustment factors, no costs modeled from price lists.'], ['The authors’ model', 'Each model runs from its author’s unmodified weights and passes a parity check against the author’s own code before it is benchmarked. The number belongs to their model, not to a broken port.'], @@ -637,14 +637,13 @@ export function MethodologyPage() { ['No private pre-testing', 'We publish every version we measure. If Ollaya ever ships a model of its own, it runs under the same rules and is labeled as ours.'], ['Authors can respond', 'Each model runs with its author’s recommended temperature and precision. Corrections go through public GitHub issues.'], ['Contamination flagged', 'A model trained on a task’s train split carries a flag, and its score leaves that task out, so a result on seen data never counts as a held-out one.'], - ].map(([t, d], i) => ( + ].map(([t, d]) => (
    • -

      {String(i + 1).padStart(2, '0')}

      -

      {t}

      +

      {t}

      {d}

      • ))} -
+
@@ -827,8 +826,8 @@ export function MethodologyPage() { machine: 

-            {`python -m ollaya_convert.bench_run --machine community- --out runs/ \
-    --models kev:4b d1:3b --suites latency-cuda typed \
+            {`python -m ollaya_convert.bench_run --machine community- --out runs/ \\
+    --models kev:4b d1:3b --suites latency-cuda typed \\
     --typed-data typed-decisions-test.jsonl`}

diff --git a/site/src/components/charts.tsx b/site/src/components/charts.tsx index bf9b45e..043d2a3 100644 --- a/site/src/components/charts.tsx +++ b/site/src/components/charts.tsx @@ -161,7 +161,8 @@ function segHitsBox([x1, y1, x2, y2]: Seg, b: Box): boolean { * Points with their labels. Each label goes in the nearest free spot around its point (right, * left, above, below, then the diagonals, further out each round); a label away from its point * gets a short leader line, and no leader crosses another leader, a label or a point. Crowded - * points are labelled first, while the space around them is still free. + * points are labelled first, while the space around them is still free. A label with no free + * spot is left out rather than drawn over another; every point names itself on hover. */ export function Scatter({ points, @@ -188,12 +189,12 @@ export function Scatter({ }) { const W = compact ? 400 : 760 const top = standalone && legend ? 40 : 16 - const H = (compact ? 470 : 540) + top - 16 - const m = { l: compact ? 40 : 52, r: compact ? 8 : 16, t: top, b: compact ? 44 : 52 } + const H = (compact ? 520 : 540) + top - 16 + const m = { l: compact ? 40 : 52, r: compact ? 12 : 16, t: top, b: compact ? 44 : 52 } const fs = compact ? 10.5 : 11.5 const x = logScale(xTicks[0]!, xTicks.at(-1)!, m.l, W - m.r) const y = linScale(yTicks[0]!, yTicks.at(-1)!, H - m.b, m.t) - const charW = compact ? 6.1 : 6.7 + const charW = fs * 0.61 // the advance of a monospace glyph, with a little to spare const lh = compact ? 13 : 14 const pts = points.map((p) => ({ p, px: x(p.x), py: y(p.y) })) const pointBoxes: Box[] = pts.map(({ px, py }) => ({ x: px - 7, y: py - 7, w: 14, h: 14 })) @@ -212,16 +213,21 @@ export function Scatter({ ] const labels = [...pts] .sort((a, b) => crowd(b) - crowd(a) || a.py - b.py) - .map(({ p, px, py }, _i) => { + .map(({ p, px, py }) => { const w = p.label.length * charW const self = pointBoxes[pts.findIndex((q) => q.p === p)] - for (const d of [9, 20, 32, 46, 62, 80, 100]) { + for (const d of [9, 14, 20, 26, 32, 40, 48, 58]) { for (const [dx, dy] of dirs) { const bx = dx > 0 ? px + d : dx < 0 ? px - d - w : px - w / 2 const by = dy > 0 ? py + d - 4 : dy < 0 ? py - d - lh + 4 : py - lh / 2 const box = { x: bx, y: by, w, h: lh } if (box.x < m.l + 2 || box.x + w > W - 2 || box.y < m.t - 12 || box.y + lh > H - m.b) continue - if (boxes.some((b) => overlaps(box, b))) continue + // A few pixels of air, so two labels never touch end to end, and no label sits on another's leader. + const air = { x: bx - 4, y: by - 1, w: w + 8, h: lh + 2 } + if (boxes.some((b) => overlaps(b === self ? box : air, b)) || leaders.some((s) => segHitsBox(s, air))) continue + // A label without a leader is closer to its own point than to any other, so a reader pairs them right. + const near = (q: { px: number; py: number }) => Math.hypot(q.px - (bx + w / 2), q.py - (by + lh / 2)) + if (d === 9 && pts.some((q) => q.p !== p && near(q) < near({ px, py }))) continue const ex = Math.max(bx, Math.min(px, bx + w)) const ey = Math.max(by, Math.min(py, by + lh)) const lead = d > 9 @@ -235,9 +241,9 @@ export function Scatter({ return { p, x: bx, y: by + 11, lead, ex, ey, px, py } } } - boxes.push({ x: px + 9, y: py - 7, w, h: lh }) - return { p, x: px + 9, y: py + 4, lead: false, ex: 0, ey: 0, px, py } + return null }) + .filter((l) => l !== null) const sa = standaloneParts(W, H, standalone) return ( @@ -268,7 +274,10 @@ export function Scatter({ {labels.map((l) => (l.lead ? : null))} {pts.map(({ p, px, py }) => ( - + + {`${p.label}: ${fmt(p.y)}, ${Math.round(p.x)} ms`} + + ))} {labels.map((l) => ( (

- {r.label} + {r.label} {r.note ? {r.note} : null} @@ -357,7 +366,7 @@ export function Heatmap({ {r.tone && r.tone !== 'us' ? r.label : {r.label}} @@ -409,7 +418,9 @@ export function DotPlot({ }) { const W = compact ? 400 : 760 const rowH = compact ? 22 : 24 - const m = { l: compact ? 122 : 150, r: compact ? 10 : 16, t: standalone ? 34 : 8, b: 34 } + // Wide enough for the longest model name (a monospace glyph is about 0.61 em). + const labelW = Math.max(0, ...rows.map((r) => r.label.length)) * (compact ? 10 : 11) * 0.61 + const m = { l: Math.max(compact ? 122 : 150, Math.ceil(labelW) + 12), r: compact ? 14 : 16, t: standalone ? 34 : 8, b: 34 } const shown = compact ? ticks.filter((t) => [10, 100, 1000, 10000, 100000].includes(t)) : ticks const H = m.t + rows.length * rowH + m.b const x = logScale(ticks[0]!, ticks.at(-1)!, m.l, W - m.r) diff --git a/site/src/data/catalog.ts b/site/src/data/catalog.ts index 48f589d..be3b93a 100644 --- a/site/src/data/catalog.ts +++ b/site/src/data/catalog.ts @@ -101,7 +101,7 @@ const overlays: Record = { stats: stats('decider:4b'), title: 'decider', description: - `Decoder decision models by Mapika on Qwen3.5: the answer is read from option-letter logits in one forward pass. decider:4b scores ${result('decider:4b', 'typed.accuracy', 3)} on typed decisions, decider:2b ${result('decider:2b', 'typed.accuracy', 3)}.`, + `Decoder decision models by Mapika on Qwen3.5: the answer is read from option-letter logits in one forward pass.`, publisher: { name: 'Mapika', url: 'https://huggingface.co/Mapika' }, capabilities: ['decision', 'long-context'], keywords: ['decider', 'decision', 'qwen', 'llm', 'classification', 'typesafe', 'jev', 'system one'], diff --git a/site/src/pages/download.tsx b/site/src/pages/download.tsx index beea9f9..ca96324 100644 --- a/site/src/pages/download.tsx +++ b/site/src/pages/download.tsx @@ -1,6 +1,7 @@ import type { Child } from 'hono/jsx' import { Icon } from '../components/Icon' import { btnPrimary, CodeBlock, textLink } from '../components/ui' +import { benchGpu, result } from '../data/numbers' import { DOCKER_IMAGE, LATEST_DOWNLOAD, LOCAL_PORT, RELEASES_URL, WINDOWS_APP_SIGNED } from '../site' type Os = 'macos' | 'windows' | 'linux' | 'docker' @@ -139,8 +140,8 @@ export function DownloadPage({ origin }: { origin: string }) { - The recommended model: close to Jev's accuracy, in about 90 ms on an NVIDIA GPU with 10 GB or more (an 8 GB - download). Otherwise, ollaya run laya answers in a fraction of a second, on the + The recommended model: close to Jev's accuracy, in {result('winnow:e4b', 'gpu.p50_ms', 0)} ms on an {benchGpu}. It needs an + NVIDIA GPU with 10 GB or more (an 8 GB download). Otherwise, ollaya run laya answers in a fraction of a second, on the CPU too. @@ -185,8 +186,8 @@ export function DownloadPage({ origin }: { origin: string }) { - The recommended model: close to Jev's accuracy, in about 90 ms on an NVIDIA GPU with 10 GB or more (an 8 GB - download). Otherwise, ollaya run laya answers in a fraction of a second, on the + The recommended model: close to Jev's accuracy, in {result('winnow:e4b', 'gpu.p50_ms', 0)} ms on an {benchGpu}. It needs an + NVIDIA GPU with 10 GB or more (an 8 GB download). Otherwise, ollaya run laya answers in a fraction of a second, on the CPU too. diff --git a/site/src/pages/home.tsx b/site/src/pages/home.tsx index ebbb93b..8de4972 100644 --- a/site/src/pages/home.tsx +++ b/site/src/pages/home.tsx @@ -5,7 +5,7 @@ import { LogoMark } from '../components/Logo' import { Legend } from '../components/charts' import { btnPrimary, Code, CodeBlock, textLink } from '../components/ui' import { catalog, comingNext, featuredTags, fullName } from '../data/catalog' -import { result } from '../data/numbers' +import { benchEdition, benchGpu, measured, result, rowsOf, trainedOnTyped } from '../data/numbers' import { publicBench, shortName } from '../data/results' import { hero, type HeroOption, type HeroQuestion } from '../data/hero' import { GITHUB_URL, LOCAL_API } from '../site' @@ -282,36 +282,23 @@ function Section({ ) } -// Accuracy: the typed-decisions test split (400 states, 2,000 questions, argmax against the -// majority label), as published on each model's page; Jev's is from Winnow's benchmark report on -// the same 2,000 questions. Latency: median of 15 warm five-question requests (the triage preset on -// the hero's message) through the HTTP API on an RTX 4090, each model in its shipped precision; -// Jev's is the hosted API's median request in third-party benchmarks, network included. -// laya:typed-decisions (0.766) and jeb (0.79-0.80) are left out: both were trained on this dataset's train split. clm's latency is a new -// message whose five questions are already cached (a repeated request takes under a millisecond). -const scoreboard: { tag: string; acc: number; ms: number; pick?: boolean; note?: string }[] = [ - { tag: 'winnow:e4b', acc: 0.722, ms: 89, pick: true }, - { tag: 'kev:9b', acc: 0.722, ms: 498 }, - { tag: 'clef:flash', acc: 0.703, ms: 532 }, - { tag: 'winnow:12b', acc: 0.702, ms: 131 }, - { tag: 'cygnet:12b', acc: 0.683, ms: 202 }, - { tag: 'decider:4b', acc: 0.68, ms: 520 }, - { tag: 'jeeves:9b', acc: 0.68, ms: 838 }, - { tag: 'kev:4b', acc: 0.669, ms: 354 }, - { tag: 'nimble:9b', acc: 0.665, ms: 2297 }, - { tag: 'snap:2b', acc: 0.648, ms: 68 }, - { tag: 'jevk5:4b', acc: 0.625, ms: 105 }, - { tag: 'decider:2b', acc: 0.591, ms: 190 }, - { tag: 'nli', acc: 0.548, ms: 20 }, - { tag: 'decider:0.8b', acc: 0.506, ms: 155 }, - { tag: 'decima:base', acc: 0.495, ms: 15 }, - { tag: 'gliclass', acc: 0.477, ms: 15 }, - { tag: 'kev:0.8b', acc: 0.46, ms: 128 }, - { tag: 'von', acc: 0.447, ms: 23 }, - { tag: 'decima:small', acc: 0.432, ms: 7 }, - { tag: 'laya:en', acc: 0.361, ms: 10 }, - { tag: 'clm:8b', acc: 0.357, ms: 149, note: 'questions cached' }, -] +// Accuracy: typed-decisions (400 states, 2,000 questions, argmax against the majority label); latency: the median +// five-question request on the edition's GPU. Both from results/ (data/numbers.ts). One row per family, its most accurate +// tag; -fp16 tags sit under their base tag, and models trained on the dataset's train split are left out. +// Jev's accuracy is TypeSafe's report; its latency is the hosted API in third-party benchmarks, network included. +const scoreboard: { tag: string; acc: number; ms: number; pick?: boolean; note?: string }[] = (() => { + const best = new Map() + for (const [tag, row] of rowsOf('typed')) { + const acc = row.accuracy as number + const ms = measured(tag, 'gpu.p50_ms') + if (tag.endsWith('-fp16') || trainedOnTyped(tag) || !acc || ms === undefined) continue + const family = tag.split(':')[0]! + if (!best.has(family) || best.get(family)!.acc < acc) best.set(family, { tag, acc, ms: Math.round(ms) }) + } + return [...best.values()] + .sort((a, b) => b.acc - a.acc) + .map((r) => ({ ...r, pick: r.tag === 'winnow:e4b', note: r.tag === 'clm:8b' ? 'questions cached' : undefined })) +})() const JEV = { acc: 0.738, text: '236–276 ms' } const ACC_SCALE = 0.8 const ACC_AXIS = [0, 0.2, 0.4, 0.6, 0.8] @@ -360,10 +347,10 @@ function Fast() { id="fast" title="Fast and accurate" lead={`Close to Jev's accuracy, in ${result('winnow:e4b', 'gpu.p50_ms', 0)} ms.`} - body={`A decision model answers in a single forward pass, with no token-by-token generation. On an RTX 5090, winnow:e4b answers a five-question request in ${result('winnow:e4b', 'gpu.p50_ms', 0)} ms end to end, and scores ${result('winnow:e4b', 'typed.accuracy', 3)} on typed decisions; TypeSafe reports ${JEV_TYPED} for its hosted Jev. Smaller models such as laya:en answer in ${result('laya:en', 'gpu.p50_ms', 0)} ms, and run well on a CPU.`} + body={`One forward pass per request, no token-by-token generation. Small models such as laya:en answer in ${result('laya:en', 'gpu.p50_ms', 0)} ms and run well on a CPU.`} >
- +
@@ -371,8 +358,8 @@ function Fast() {
- Accuracy and speed of every model{' '} - · typed-decisions accuracy, higher is better; latency, lower is better + Accuracy and speed, best tag of each model{' '} + · typed-decisions; five questions on an {benchGpu}

- Accuracy: the typed-decisions test split (400 states, 2,000 questions) against the majority label; Jev's from Winnow's - report on the same questions. laya:typed-decisions and jeb{' '} - were trained on this dataset, so they are left out. Latency: the median five-question request through the HTTP API on an - RTX 4090 (clm with its questions cached); Jev's is the hosted API in third-party benchmarks ( + 2,000 questions. laya:typed-decisions and jeb were trained on + them and are left out. Jev: TypeSafe's accuracy, hosted latency from{' '} - AbdelStark/jev-benchmarks - - ,{' '} - - nibzard/decision-model-benchmark + third-party benchmarks - ), network included, so compare orders of magnitude. + , network included.

@@ -446,6 +427,7 @@ const vsRuns: VsRun[] = VS_TAGS.map(([tag, server]) => { return { tag, server, acc: r.macro_accuracy, ece: r.pooled_ece, ms: Math.round(r.median_ms) } }).sort((a, b) => b.acc - a.acc) const vs = (tag: string, server: VsRun['server']) => vsRuns.find((r) => r.tag === tag && r.server === server)! +const OLLAMA = `Ollama ${publicBench?.servers.ollama ?? ''}`.trim() /** How many times faster Ollama is on the same weights, as a range over the pairs both serve. */ function speedup() { const r = [['nimble:9b', 'nimble'], ['clef:flash', 'clef-flash'], ['kev:4b', 'kev:4b-q8_0']].map(([a, b]) => vs(a!, 'ollaya').ms / vs(b!, 'ollama').ms) @@ -463,14 +445,13 @@ const VS_TICKS = [0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85] const vsPos = (a: number) => `${(((Math.min(Math.max(a, VS_MIN), VS_MAX) - VS_MIN) / (VS_MAX - VS_MIN)) * 100).toFixed(2)}%` const vsFeatures: { label: string; ollaya: string; ollama: string }[] = [ - { label: 'Decision models', ollaya: '23 families: encoders (laya, nli, gliclass, von, decima) and decoders (winnow, clef, kev, decider, nimble, pplx-decider, strands-decider, jeb, jeeves, cygnet, snap, arbiter and more)', ollama: 'Nine, all decoders except Laya' }, - { label: 'Small encoders (milliseconds, CPU-friendly)', ollaya: 'laya, nli, gliclass, von, decima', ollama: 'Laya' }, - { label: 'Probabilities', ollaya: "Calibrated with each author's fitted temperature, refittable in a Modelfile", ollama: "Mostly raw softmax; Nimble's and Strands Decider's temperatures not applied" }, - { label: 'Options per question', ollaya: 'Up to 255, as TypeSafe', ollama: 'Up to 26 (255 for Kev and Strands Decider)' }, - { label: 'Questions per request', ollaya: 'Up to 256, as TypeSafe', ollama: 'Up to 64' }, - { label: 'TypeSafe endpoints', ollaya: '/v1/systemone, /v1/decisions, /v1/models', ollama: '/v1/systemone' }, - { label: 'Language routing', ollaya: 'laya picks English or multilingual per request', ollama: 'None' }, - { label: 'Weights', ollaya: "The author's files, pinned by commit and sha256", ollama: 'Converted to GGUF and re-hosted' }, + { label: 'Decision models', ollaya: `${catalog.length} families`, ollama: '9' }, + { label: 'CPU-fast encoders', ollaya: 'laya, nli, gliclass, von, decima', ollama: 'laya' }, + { label: 'Probabilities', ollaya: "Author's temperature", ollama: 'Raw softmax' }, + { label: 'Options per question', ollaya: '255', ollama: '26 (Kev, Strands: 255)' }, + { label: 'Questions per request', ollaya: '256', ollama: '64' }, + { label: 'TypeSafe endpoints', ollaya: 'systemone, decisions, models', ollama: 'systemone' }, + { label: 'Weights', ollaya: "Author's files, sha256-pinned", ollama: 'Re-hosted GGUF' }, ] /** @@ -485,7 +466,7 @@ function VsBar({ run }: { run: VsRun }) {
  • {run.tag} - {ollama ? 'on Ollama 0.40.6' : 'on Ollaya'} + {ollama ? `on ${OLLAMA}` : 'on Ollaya'}