diff --git a/embeddings/CHUNKING.md b/embeddings/CHUNKING.md new file mode 100644 index 00000000..9e48858a --- /dev/null +++ b/embeddings/CHUNKING.md @@ -0,0 +1,91 @@ +# Embedding strategy FFI (embeddings lib → Manticore daemon) + +Embeddings lib **v8**. The one embedding call, `make_vect_embeddings`, takes an +optional `ChunkSettings*` that selects how a document becomes one or many +vectors. Cardinality is carried as **data** in the return (a per-document +offsets sidecar), so a single method covers both 1-vector and N-vector +strategies — no second method. + +## FFI + +```c +constexpr uint32_t STRATEGY_TRUNCATE = 0; // 1 vector/doc +constexpr uint32_t STRATEGY_MEAN = 1; // 1 vector/doc +constexpr uint32_t STRATEGY_FIXED = 2; // N vectors/doc +constexpr uint32_t STRATEGY_RECURSIVE = 3; // N vectors/doc +constexpr uint32_t STRATEGY_SENTENCE = 4; // N vectors/doc + +struct ChunkSettings { + uint32_t strategy; // one of STRATEGY_* + uint32_t max_tokens; // chunk size in tokens; 0 = model max + uint32_t overlap_tokens; // token overlap between chunks; 0 = none + uint32_t max_chunks; // cap on chunks/doc; 0 = unlimited (overflow merges into last) +}; + +// settings == nullptr ⇒ truncate (today's behavior) +FloatVecResult make_vect_embeddings( + const TextModelWrapper*, const StringItem* texts, uintptr_t count, + const ChunkSettings*, int32_t threads); +``` + +### Return: flat vectors + per-row offsets (cardinality is data) + +```c +struct FloatVecResult { + char *m_szError; + const FloatVec *m_tEmbedding; // FLAT: every document's vectors concatenated, `len` total + uintptr_t len; + uintptr_t cap; + const uintptr_t *m_pRowOffsets; // length rows+1; doc i = m_tEmbedding[off[i] .. off[i+1]] + uintptr_t rows; // number of input documents + uintptr_t offsets_cap; +}; +``` + +Read document `i`'s vectors as `m_tEmbedding[m_pRowOffsets[i] .. m_pRowOffsets[i+1]]`. +- `truncate`/`mean` → one vector/doc, so `len == rows == count` and offsets are + `[0, 1, …, rows]` (you may just index `m_tEmbedding[i]`). +- `fixed`/`recursive`/`sentence` → N vectors/doc; `len` = total chunks, and the + offsets group them per document. + +Free with `free_vec_result` (it frees the offsets too). Load-time check: +`EmbedLib.version == 8`. + +## Strategies + +| strategy | val | output | what it does | +|---|---|---|---| +| **truncate** | 0 | 1 vec/doc | embed the first `max_tokens` tokens (rest dropped). `max_tokens`/`overlap`/`max_chunks` ignored. | +| **mean** | 1 | 1 vec/doc | split (recursive, token-aware) → embed every chunk → **average** into one L2-normalized vector. Whole document, no tail loss. | +| **fixed** | 2 | N vecs/doc | split into fixed `max_tokens`-token windows; one vector per chunk. | +| **recursive** | 3 | N vecs/doc | split on a separator hierarchy (paragraph → line → space) ≤ `max_tokens`; one vector per chunk. | +| **sentence** | 4 | N vecs/doc | UAX-29 sentence segmentation (ES-style), grouped to `max_tokens`; one vector per chunk. | + +- `max_tokens = 0` → the model's own max input length. +- `overlap_tokens` → token overlap between chunks (multi-vector + mean). +- `max_chunks` → cap chunks/doc; overflow merges the tail into the last chunk. +- Local models chunk on the model's real subword tokens; remote API models + (OpenAI/Voyage/Jina) chunk by a char/byte heuristic (no local tokenizer). + +## Calling it + +```cpp +// non-chunked field & queries: pass nullptr → truncate +pFuncs->make_vect_embeddings( &model, items.data(), items.size(), nullptr, iThreads ); + +// any strategy, taken from a table option: +ChunkSettings tCfg { STRATEGY_SENTENCE, /*max_tokens*/ 256, /*overlap*/ 0, /*max_chunks*/ 0 }; +FloatVecResult tRes = pFuncs->make_vect_embeddings( &model, items.data(), items.size(), &tCfg, iThreads ); +// doc i owns vectors tRes.m_tEmbedding[ tRes.m_pRowOffsets[i] .. tRes.m_pRowOffsets[i+1] ] +``` + +The daemon owns the SQL/DDL surface (e.g. a per-`float_vector`-field option), +parses it into a `ChunkSettings`, and passes it on every embed call. Queries are +short: pass `nullptr` so they are never chunked. + +## Daemon side (next phase, not this lib) + +The lib already returns N vectors/doc for `fixed`/`recursive`/`sentence` via the +offsets sidecar. To *use* them the daemon needs N-vectors-per-row storage + +max-over-chunks search; `truncate`/`mean` (1 vector/doc) work with today's +storage unchanged. diff --git a/embeddings/Cargo.lock b/embeddings/Cargo.lock index e94a79b6..53719fe9 100644 --- a/embeddings/Cargo.lock +++ b/embeddings/Cargo.lock @@ -16,9 +16,9 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -31,18 +31,18 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "approx" @@ -72,9 +72,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "base16ct" @@ -94,6 +94,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bindgen_cuda" version = "0.1.6" @@ -128,9 +134,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "block" @@ -158,28 +164,28 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.10.2" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.4", ] [[package]] @@ -190,9 +196,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "candle-core" @@ -204,7 +210,7 @@ dependencies = [ "candle-kernels", "candle-metal-kernels", "candle-ug", - "cudarc 0.19.6", + "cudarc 0.19.9", "float8 0.6.1", "gemm 0.19.0", "half", @@ -216,12 +222,12 @@ dependencies = [ "num_cpus", "objc2-foundation", "objc2-metal", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "rayon", "safetensors 0.7.0", - "thiserror 2.0.18", - "yoke 0.8.2", + "thiserror 2.0.20", + "yoke 0.8.3", "zip", ] @@ -243,7 +249,7 @@ dependencies = [ "objc2-foundation", "objc2-metal", "once_cell", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", ] @@ -263,7 +269,7 @@ dependencies = [ "rayon", "safetensors 0.7.0", "serde", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -278,7 +284,7 @@ dependencies = [ "fancy-regex", "intel-mkl-src", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "rayon", "serde", "serde_json", @@ -317,9 +323,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.62" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" dependencies = [ "find-msvc-tools", "shlex", @@ -333,15 +339,26 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -423,20 +440,29 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-deque" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -444,18 +470,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -485,9 +511,9 @@ dependencies = [ [[package]] name = "cudarc" -version = "0.19.6" +version = "0.19.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83be2fd94791d2580692ced87ea19e9728102ce75582543b82765dbe3e884557" +checksum = "804764d10e844da09765a7b2ca9641a0851523d1702efb0d7299d73e31b86e80" dependencies = [ "float8 0.7.0", "half", @@ -539,7 +565,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -561,7 +587,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -603,7 +629,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -623,7 +649,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core 0.20.2", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -672,19 +698,19 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "objc2", ] [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.4", ] [[package]] @@ -705,9 +731,9 @@ checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" [[package]] name = "either" -version = "1.15.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "encode_unicode" @@ -724,7 +750,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -762,9 +788,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "filetime" @@ -778,18 +804,19 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -798,10 +825,10 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719a903cc23e4a89e87962c2a80fdb45cdaad0983a89bd150bb57b4c8571a7d5" dependencies = [ - "cudarc 0.19.6", + "cudarc 0.19.9", "half", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", ] @@ -820,12 +847,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "foldhash" version = "0.2.0" @@ -844,13 +865,13 @@ dependencies = [ [[package]] name = "foreign-types-macros" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +checksum = "ea5190182e6915eb873ddbc16e23b711b6eb1f9c00a0d0a3a91b5f6228475225" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.4", ] [[package]] @@ -870,9 +891,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -880,33 +901,33 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-io", @@ -1052,7 +1073,7 @@ dependencies = [ "num-traits", "once_cell", "paste", - "pulp 0.22.2", + "pulp 0.22.3", "raw-cpuid", "rayon", "seq-macro", @@ -1185,43 +1206,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", - "wasip2", - "wasip3", + "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] name = "getset" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf0fc11e47561d47397154977bc219f4cf809b2974facc3ccb3b89e2436f912" +checksum = "6cf442baaabe4213ce7d1239afc26c039180b6456da2cededa316ae2c8a77a77" dependencies = [ - "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "glob" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "half" @@ -1233,7 +1252,7 @@ dependencies = [ "cfg-if", "crunchy", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "zerocopy", ] @@ -1244,15 +1263,6 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash 0.1.5", -] - [[package]] name = "hashbrown" version = "0.16.1" @@ -1261,7 +1271,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash 0.2.0", + "foldhash", "serde", "serde_core", ] @@ -1295,9 +1305,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" [[package]] name = "hf-hub" @@ -1308,7 +1318,7 @@ dependencies = [ "http", "indicatif", "log", - "rand 0.8.6", + "rand 0.8.8", "serde", "serde_json", "thiserror 1.0.69", @@ -1323,9 +1333,9 @@ checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" [[package]] name = "http" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -1333,9 +1343,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -1343,9 +1353,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -1362,9 +1372,9 @@ checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "hyper" -version = "1.9.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -1393,7 +1403,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.7", + "webpki-roots 1.0.9", ] [[package]] @@ -1445,23 +1455,23 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", "utf8_iter", - "yoke 0.8.2", + "yoke 0.8.3", "zerofrom", "zerovec", ] [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -1472,9 +1482,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1486,16 +1496,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -1506,31 +1517,25 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", "writeable", - "yoke 0.8.2", + "yoke 0.8.3", "zerofrom", "zerotrie", "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -1570,14 +1575,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" dependencies = [ "equivalent", "hashbrown 0.17.1", - "serde", - "serde_core", ] [[package]] @@ -1617,9 +1620,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" [[package]] name = "itertools" @@ -1647,13 +1650,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "js-sys" -version = "0.3.98" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -1663,17 +1665,11 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libloading" @@ -1703,9 +1699,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.16" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" dependencies = [ "libc", ] @@ -1718,15 +1714,15 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "log" -version = "0.4.29" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru-slab" @@ -1736,25 +1732,25 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lzma-rust2" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1670343e58806300d87950e3401e820b519b9384281bbabfb15e3636689ffd69" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" [[package]] name = "macro_rules_attribute" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" dependencies = [ "macro_rules_attribute-proc_macro", - "paste", + "pastey", ] [[package]] name = "macro_rules_attribute-proc_macro" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" [[package]] name = "malloc_buf" @@ -1778,19 +1774,20 @@ dependencies = [ "hf-hub", "ndarray 0.16.1", "ort", - "rand 0.8.6", + "rand 0.8.8", "rayon", "reqwest", "serde", "serde_json", "tokenizers", + "unicode-segmentation", ] [[package]] name = "matrixmultiply" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" dependencies = [ "autocfg", "rawpointer", @@ -1798,15 +1795,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memmap2" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" dependencies = [ "libc", "stable_deref_trait", @@ -1818,7 +1815,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ecfd3296f8c56b7c1f6fbac3c71cefa9d78ce009850c45000015f206dc7fa21" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "block", "core-graphics-types", "foreign-types", @@ -1835,9 +1832,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.9" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -1845,9 +1842,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -1873,7 +1870,7 @@ checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -1932,9 +1929,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -1952,20 +1949,19 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] [[package]] name = "num-iter" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -1997,7 +1993,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "hermit-abi 0.5.2", + "hermit-abi 0.5.3", "libc", ] @@ -2031,7 +2027,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "dispatch2", "objc2", ] @@ -2048,7 +2044,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "block2", "libc", "objc2", @@ -2061,7 +2057,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "block2", "dispatch2", "objc2", @@ -2123,7 +2119,7 @@ version = "6.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "libc", "once_cell", "onig_sys", @@ -2147,26 +2143,26 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "ort" -version = "2.0.0-rc.12" +version = "2.0.0-rc.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +checksum = "4336a1e2b38848325241c72889086886004e589b7c74f335e60a8e8db5138a0b" dependencies = [ "ndarray 0.17.2", "ort-sys", "smallvec", "tracing", - "ureq 3.3.0", + "ureq 3.4.0", ] [[package]] name = "ort-sys" -version = "2.0.0-rc.12" +version = "2.0.0-rc.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" +checksum = "cf211e3776eea6aec988552fa118dd746d70e1b1e5e244058d1c98015f3e5872" dependencies = [ "hmac-sha256", "lzma-rust2", - "ureq 3.3.0", + "ureq 3.4.0", ] [[package]] @@ -2181,6 +2177,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -2195,15 +2197,15 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" @@ -2216,9 +2218,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -2232,43 +2234,11 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.117", -] - -[[package]] -name = "proc-macro-error-attr2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" -dependencies = [ - "proc-macro2", - "quote", -] - -[[package]] -name = "proc-macro-error2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" -dependencies = [ - "proc-macro-error-attr2", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -2289,9 +2259,9 @@ dependencies = [ [[package]] name = "pulp" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e205bb30d5b916c55e584c22201771bcf2bad9aabd5d4127f38387140c38632" +checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" dependencies = [ "bytemuck", "cfg-if", @@ -2306,15 +2276,15 @@ dependencies = [ [[package]] name = "pulp-wasm-simd-flag" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40e24eee682d89fb193496edf918a7f407d30175b2e785fe057e4392dfd182e0" +checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -2324,7 +2294,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -2332,20 +2302,21 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.4", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -2353,23 +2324,23 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ "cfg_aliases", "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -2388,9 +2359,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -2399,14 +2370,25 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -2445,6 +2427,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_distr" version = "0.5.1" @@ -2452,7 +2440,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.4", + "rand 0.9.5", +] + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", ] [[package]] @@ -2461,7 +2458,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", ] [[package]] @@ -2520,9 +2517,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -2532,9 +2529,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -2543,9 +2540,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" @@ -2584,7 +2581,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 1.0.7", + "webpki-roots 1.0.9", ] [[package]] @@ -2603,9 +2600,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustix" @@ -2613,7 +2610,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", @@ -2622,9 +2619,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "log", "once_cell", @@ -2637,9 +2634,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -2647,9 +2644,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", @@ -2658,9 +2655,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ryu" @@ -2698,12 +2695,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - [[package]] name = "seq-macro" version = "0.3.6" @@ -2712,9 +2703,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -2722,29 +2713,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.4", ] [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -2790,21 +2781,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] name = "slab" @@ -2814,15 +2805,15 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -2885,7 +2876,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -2907,9 +2898,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" dependencies = [ "proc-macro2", "quote", @@ -2933,7 +2935,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -2942,7 +2944,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "byteorder", "enum-as-inner", "libc", @@ -2952,9 +2954,9 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.45" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" dependencies = [ "filetime", "libc", @@ -2968,7 +2970,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -3000,11 +3002,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.20", ] [[package]] @@ -3015,25 +3017,25 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.4", ] [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -3041,9 +3043,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -3071,7 +3073,7 @@ dependencies = [ "monostate", "onig", "paste", - "rand 0.8.6", + "rand 0.8.8", "rayon", "rayon-cond", "regex", @@ -3087,9 +3089,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -3145,7 +3147,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "serde", "serde_spanned", "toml_datetime", @@ -3176,11 +3178,11 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "bytes", "futures-util", "http", @@ -3223,7 +3225,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -3249,9 +3251,9 @@ checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ug" @@ -3318,9 +3320,9 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-width" @@ -3328,12 +3330,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "unicode_categories" version = "0.1.1" @@ -3367,11 +3363,11 @@ dependencies = [ [[package]] name = "ureq" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "log", "percent-encoding", "rustls", @@ -3379,16 +3375,16 @@ dependencies = [ "socks", "ureq-proto", "utf8-zero", - "webpki-roots 1.0.7", + "webpki-roots 1.0.9", ] [[package]] name = "ureq-proto" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "http", "httparse", "log", @@ -3420,11 +3416,11 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.23.1" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "js-sys", "wasm-bindgen", ] @@ -3462,27 +3458,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" -dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.121" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -3493,9 +3480,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.71" +version = "0.4.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" dependencies = [ "js-sys", "wasm-bindgen", @@ -3503,9 +3490,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.121" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3513,65 +3500,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.121" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.121" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap 2.14.0", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags 2.11.1", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "semver", -] - [[package]] name = "web-sys" -version = "0.3.98" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" dependencies = [ "js-sys", "wasm-bindgen", @@ -3593,14 +3546,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.7", + "webpki-roots 1.0.9", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" dependencies = [ "rustls-pki-types", ] @@ -3657,7 +3610,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -3668,7 +3621,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -3722,15 +3675,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -3764,30 +3708,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3800,12 +3727,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -3818,12 +3739,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -3836,24 +3751,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -3866,12 +3769,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -3884,12 +3781,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3902,12 +3793,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -3920,12 +3805,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "0.7.15" @@ -3935,105 +3814,17 @@ dependencies = [ "memchr", ] -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck 0.5.0", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck 0.5.0", - "indexmap 2.14.0", - "prettyplease", - "syn 2.0.117", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.117", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags 2.11.1", - "indexmap 2.14.0", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap 2.14.0", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "xattr" @@ -4059,9 +3850,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive 0.8.2", @@ -4076,7 +3867,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "synstructure", ] @@ -4088,28 +3879,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -4129,47 +3920,47 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", - "yoke 0.8.2", + "yoke 0.8.3", "zerofrom", ] [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ - "yoke 0.8.2", + "yoke 0.8.3", "zerofrom", "zerovec-derive", ] [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.4", ] [[package]] @@ -4179,13 +3970,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" dependencies = [ "crc32fast", - "indexmap 2.14.0", + "indexmap 2.14.1", "memchr", "typed-path", ] +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/embeddings/Cargo.toml b/embeddings/Cargo.toml index 8758130d..e6a98749 100644 --- a/embeddings/Cargo.toml +++ b/embeddings/Cargo.toml @@ -13,6 +13,7 @@ edition = "2021" # LNK2038. esaxx is used exclusively by the unigram trainer, which we never # run. onig stays: tokenizers' SysRegex is onig-backed on non-wasm targets. tokenizers = { version = "0.15.2", default-features = false, features = ["onig"] } +unicode-segmentation = "1" hf-hub = { git = "https://github.com/huggingface/hf-hub.git", rev = "ac22200ea0b5af4d8c362f699be0340647b19060", default-features = false,features = ["ureq"] } anyhow = "1.0.81" serde_json = "1.0.114" diff --git a/embeddings/INSTRUCTIONS.md b/embeddings/INSTRUCTIONS.md index 069aafd2..b22c797c 100644 --- a/embeddings/INSTRUCTIONS.md +++ b/embeddings/INSTRUCTIONS.md @@ -53,7 +53,7 @@ src/ 1. **`src/model/text_model_wrapper.rs`** - The heart of FFI - `load_model()` - Creates model instances - - `make_vect_embeddings()` - Generates embeddings (FIXED: no empty vectors on error) + - `make_vect_embeddings()` - Generates embeddings; takes an optional `ChunkSettings*` selecting the strategy (truncate / mean / fixed / recursive / sentence). Returns a flat `FloatVecResult` grouped per document by `m_pRowOffsets`. **Contract + strategy reference: [`CHUNKING.md`](CHUNKING.md).** - `free_model_result()` / `free_vec_result()` - Memory cleanup 2. **`manticoresearch_text_embeddings.h`** - Auto-generated C header diff --git a/embeddings/manticoresearch_text_embeddings.h b/embeddings/manticoresearch_text_embeddings.h index 3a8d5ef3..ac494995 100644 --- a/embeddings/manticoresearch_text_embeddings.h +++ b/embeddings/manticoresearch_text_embeddings.h @@ -9,6 +9,17 @@ #include #include +/// Strategy, mirrored as a `u32` across the FFI in [`ChunkSettings`]. +constexpr static const uint32_t STRATEGY_TRUNCATE = 0; + +constexpr static const uint32_t STRATEGY_MEAN = 1; + +constexpr static const uint32_t STRATEGY_FIXED = 2; + +constexpr static const uint32_t STRATEGY_RECURSIVE = 3; + +constexpr static const uint32_t STRATEGY_SENTENCE = 4; + struct TextModelResult { void *m_pModel; char *m_szError; @@ -33,11 +44,25 @@ struct FloatVec { uintptr_t cap; }; +/// Embedding result for one `make_vect_embeddings` call. +/// +/// `m_tEmbedding` is a FLAT array of `len` vectors — every input document's +/// vectors concatenated. `m_pRowOffsets` (length `rows + 1`) groups them per +/// input document, Arrow-style: document `i` owns +/// `m_tEmbedding[m_pRowOffsets[i] .. m_pRowOffsets[i + 1]]`. For the v1 +/// strategies (truncate / mean) every document yields exactly one vector, so +/// `len == rows` and the offsets are `[0, 1, ..., rows]`. The sidecar lets a +/// future multi-vector strategy return N vectors per document through this same +/// struct — no second method, cardinality carried as data. +/// struct FloatVecResult { char *m_szError; const FloatVec *m_tEmbedding; uintptr_t len; uintptr_t cap; + const uintptr_t *m_pRowOffsets; + uintptr_t rows; + uintptr_t offsets_cap; }; using TextModelWrapper = void*; @@ -47,9 +72,25 @@ struct StringItem { uintptr_t len; }; +/// Chunking parameters. `#[repr(C)]` — passed straight across the FFI by the +/// daemon, which owns the DDL surface and validates against the model. +struct ChunkSettings { + /// One of the `STRATEGY_*` constants. + uint32_t strategy; + /// Target chunk size in tokens. `0` ⇒ use the model's max. Always clamped to + /// the model's real input limit. + uint32_t max_tokens; + /// Token overlap between consecutive chunks. `0` ⇒ none. + uint32_t overlap_tokens; + /// Hard cap on chunks per document. `0` ⇒ unlimited. Overflow merges the + /// tail into the last chunk (matches OpenSearch's `max_chunk_limit`). + uint32_t max_chunks; +}; + using MakeVectEmbeddingsFn = FloatVecResult(*)(const TextModelWrapper*, const StringItem*, uintptr_t, + const ChunkSettings*, int32_t); using FreeVecResultFn = void(*)(FloatVecResult); diff --git a/embeddings/src/chunk.rs b/embeddings/src/chunk.rs new file mode 100644 index 00000000..640021ff --- /dev/null +++ b/embeddings/src/chunk.rs @@ -0,0 +1,500 @@ +//! Document chunking + embedding strategies. +//! +//! One document maps to one or many vectors depending on the strategy: +//! - `truncate` / `mean` collapse to **one** vector per document. +//! - `fixed` / `recursive` / `sentence` keep **N** vectors per document (one per +//! chunk) — the daemon groups them via the row-offsets sidecar. +//! +//! Local models chunk token-accurately via their loaded tokenizer; remote API +//! models (no local tokenizer) fall back to a conservative char/byte heuristic. +//! `ChunkSettings` arrives from the daemon's DDL surface and selects everything. + +use crate::utils::normalize; +use tokenizers::Tokenizer; + +/// Strategy, mirrored as a `u32` across the FFI in [`ChunkSettings`]. +pub const STRATEGY_TRUNCATE: u32 = 0; // 1 vector/doc: first `max_tokens` tokens +pub const STRATEGY_MEAN: u32 = 1; // 1 vector/doc: chunk → embed → average +pub const STRATEGY_FIXED: u32 = 2; // N vectors/doc: fixed token windows +pub const STRATEGY_RECURSIVE: u32 = 3; // N vectors/doc: recursive token-aware split +pub const STRATEGY_SENTENCE: u32 = 4; // N vectors/doc: UAX-29 sentence segmentation + +/// Special tokens (`[CLS]`/`[SEP]`) that `predict()` re-adds when it re-tokenizes +/// a chunk string. We size chunks to leave room so a chunk is never truncated. +const SPECIAL_TOKENS_MARGIN: usize = 2; + +/// Conservative bytes-per-token for the remote heuristic. Deliberately low so a +/// heuristic chunk lands *under* the provider's token cap rather than over it. +const REMOTE_BYTES_PER_TOKEN: usize = 3; + +/// Chunking parameters. `#[repr(C)]` — passed straight across the FFI by the +/// daemon, which owns the DDL surface and validates against the model. +#[repr(C)] +pub struct ChunkSettings { + /// One of the `STRATEGY_*` constants. + pub strategy: u32, + /// Target chunk size in tokens. `0` ⇒ use the model's max. Always clamped to + /// the model's real input limit. + pub max_tokens: u32, + /// Token overlap between consecutive chunks. `0` ⇒ none. + pub overlap_tokens: u32, + /// Hard cap on chunks per document. `0` ⇒ unlimited. Overflow merges the + /// tail into the last chunk (matches OpenSearch's `max_chunk_limit`). + pub max_chunks: u32, +} + +impl ChunkSettings { + /// True when the document must be split (every strategy except `truncate`). + pub fn needs_chunking(&self) -> bool { + self.strategy != STRATEGY_TRUNCATE + } + + /// True when the strategy collapses to one vector per document + /// (`truncate`/`mean`); false for the multi-vector strategies. + pub fn is_single_vector(&self) -> bool { + self.strategy == STRATEGY_TRUNCATE || self.strategy == STRATEGY_MEAN + } +} + +/// Resolve the chunk size in *content* tokens, clamped to what the model can +/// actually take (minus the special-token budget). `0` ⇒ model max. +pub fn effective_max(settings: &ChunkSettings, model_max: usize) -> usize { + let ceiling = model_max.saturating_sub(SPECIAL_TOKENS_MARGIN).max(1); + let target = settings.max_tokens as usize; + if target == 0 { + ceiling + } else { + target.min(ceiling) + } +} + +/// Average a document's chunk embeddings into one L2-normalized vector. Chunk +/// vectors are already normalized by `predict`; the mean of unit vectors is +/// re-normalized so the result is a unit vector too. +pub fn mean_pool(chunks: &[Vec]) -> Vec { + let dim = chunks.first().map(Vec::len).unwrap_or(0); + let mut acc = vec![0.0f32; dim]; + for v in chunks { + for (a, x) in acc.iter_mut().zip(v.iter()) { + *a += *x; + } + } + let n = chunks.len().max(1) as f32; + for a in acc.iter_mut() { + *a /= n; + } + normalize(&mut acc); + acc +} + +/// Arrow-style per-row offsets (length `counts.len() + 1`) from per-document +/// vector counts: document `i` owns vectors `[offsets[i] .. offsets[i + 1]]`. +pub fn row_offsets_from_counts(counts: &[usize]) -> Vec { + let mut offsets = Vec::with_capacity(counts.len() + 1); + let mut acc = 0usize; + offsets.push(0); + for &c in counts { + acc += c; + offsets.push(acc); + } + offsets +} + +/// Split one document into chunk byte spans `(start, end)` according to the +/// strategy's split method. `tokenizer` is `Some` for local models (token-exact) +/// and `None` for remote API models (char heuristic). `STRATEGY_TRUNCATE` never +/// reaches here (it does not chunk); `STRATEGY_MEAN` splits recursively. +pub fn chunk_text( + text: &str, + max_tokens: usize, + overlap: usize, + strategy: u32, + tokenizer: Option<&Tokenizer>, +) -> Vec<(usize, usize)> { + match strategy { + STRATEGY_SENTENCE => chunk_sentence(text, max_tokens, overlap, tokenizer), + STRATEGY_FIXED => window(text, max_tokens, overlap, false, tokenizer), + // recursive (3), mean's internal split (1), and any other → recursive. + _ => window(text, max_tokens, overlap, true, tokenizer), + } +} + +/// Token-window split, picking the token-accurate or heuristic backend. +fn window( + text: &str, + max_tokens: usize, + overlap: usize, + snap: bool, + tokenizer: Option<&Tokenizer>, +) -> Vec<(usize, usize)> { + match tokenizer { + Some(t) => chunk_local(text, t, max_tokens, overlap, snap), + None => chunk_chars(text, max_tokens, overlap), + } +} + +/// Token-accurate chunking for local models. One tokenization pass; windows by +/// token count; `recursive` snaps each cut back to a natural boundary; overlap +/// is realigned to the (possibly snapped) cut so no tokens are dropped. +/// +/// NOTE: assumes `tokenizer.encode(..).get_offsets()` returns byte offsets into +/// the input (true for the WordPiece/BPE paths Manticore loads). +pub fn chunk_local( + text: &str, + tokenizer: &Tokenizer, + max_tokens: usize, + overlap: usize, + recursive: bool, +) -> Vec<(usize, usize)> { + if text.is_empty() { + return vec![(0, 0)]; + } + // `false` = no special tokens, so offsets map cleanly onto the input text. + let encoding = match tokenizer.encode(text, false) { + Ok(e) => e, + // Tokenizer failure (rare): fall back to the whole doc as one chunk. + // predict() truncates it if needed — degrades to truncate, but is safe. + Err(_) => return vec![(0, text.len())], + }; + let offsets = encoding.get_offsets(); + let n = offsets.len(); + if max_tokens == 0 || n <= max_tokens { + return vec![(0, text.len())]; + } + + let overlap = overlap.min(max_tokens / 2); + let mut chunks = Vec::new(); + let mut start = 0usize; // start token index + loop { + let win_end = (start + max_tokens).min(n); // exclusive token index + let byte_start = offsets[start].0; + let mut byte_end = offsets[win_end - 1].1; + + if recursive && win_end < n { + if let Some(cut) = snap_back(text, byte_start, byte_end) { + byte_end = cut; + } + } + chunks.push((byte_start, byte_end)); + + // First token that begins at/after the (possibly snapped) cut. + let cut_tok = offsets.partition_point(|o| o.0 < byte_end); + if cut_tok >= n { + break; + } + start = cut_tok.saturating_sub(overlap).max(start + 1); // step back overlap, always progress + } + chunks +} + +/// Char/byte heuristic for remote API models (no local tokenizer). Windows by an +/// estimated byte budget, snapping to whitespace and never cutting mid-UTF-8. +pub fn chunk_chars(text: &str, max_tokens: usize, overlap_tokens: usize) -> Vec<(usize, usize)> { + if text.is_empty() { + return vec![(0, 0)]; + } + let window = (max_tokens * REMOTE_BYTES_PER_TOKEN).max(1); + if text.len() <= window { + return vec![(0, text.len())]; + } + let overlap = (overlap_tokens * REMOTE_BYTES_PER_TOKEN).min(window / 2); + + let mut chunks = Vec::new(); + let mut start = 0usize; + while start < text.len() { + let mut end = text.floor_char_boundary((start + window).min(text.len())); + if end <= start { + // A single char wider than the window — take at least one char. + end = text.ceil_char_boundary(start + 1).min(text.len()); + } + if end < text.len() { + if let Some(cut) = snap_back(text, start, end) { + end = cut; + } + } + chunks.push((start, end)); + if end >= text.len() { + break; + } + let mut next = end.saturating_sub(overlap); + if next <= start { + next = end; + } + start = text.ceil_char_boundary(next); + } + chunks +} + +/// Count tokens in `text`: exact via the model tokenizer when available, else +/// the conservative byte estimate the remote heuristic uses. +fn count_tokens(text: &str, tokenizer: Option<&Tokenizer>) -> usize { + match tokenizer { + Some(t) => t + .encode(text, false) + .map(|e| e.get_ids().len()) + .unwrap_or(text.len() / REMOTE_BYTES_PER_TOKEN + 1), + None => text.len() / REMOTE_BYTES_PER_TOKEN + 1, + } +} + +/// UAX-29 sentence boundaries — the same standard Elasticsearch's ICU uses — +/// as contiguous byte spans tiling the whole document. +fn sentence_spans(text: &str) -> Vec<(usize, usize)> { + use unicode_segmentation::UnicodeSegmentation; + text.split_sentence_bound_indices() + .map(|(offset, s)| (offset, offset + s.len())) + .collect() +} + +/// Sentence chunking, mirroring Elasticsearch's algorithm: detect sentences +/// (UAX-29), then greedily group whole sentences up to `max_tokens`. A single +/// sentence larger than the budget is split with the token-window splitter (ES +/// splits such a sentence across chunks). `overlap` re-seeds the next chunk with +/// trailing whole sentences summing ≤ `overlap` tokens. +pub fn chunk_sentence( + text: &str, + max_tokens: usize, + overlap: usize, + tokenizer: Option<&Tokenizer>, +) -> Vec<(usize, usize)> { + if text.is_empty() { + return vec![(0, 0)]; + } + let units: Vec<(usize, usize, usize)> = sentence_spans(text) + .into_iter() + .map(|(a, b)| (a, b, count_tokens(&text[a..b], tokenizer))) + .collect(); + // 0/1 sentence (e.g. terminator-free text): nothing to group on — fall back + // to the token-window splitter so an over-long blob still gets divided. + if units.len() <= 1 { + return window(text, max_tokens, overlap, true, tokenizer); + } + + let mut chunks = Vec::new(); + let mut i = 0usize; + while i < units.len() { + // Greedily pack whole sentences up to the budget (always take ≥ 1). + let mut j = i; + let mut sum = 0usize; + while j < units.len() { + let t = units[j].2; + if j > i && sum + t > max_tokens { + break; + } + sum += t; + j += 1; + } + + let start = units[i].0; + let end = units[j - 1].1; + if j == i + 1 && units[i].2 > max_tokens { + // One sentence bigger than the budget — split it like ES does. + for (a, b) in window(&text[start..end], max_tokens, overlap, true, tokenizer) { + chunks.push((start + a, start + b)); + } + } else { + chunks.push((start, end)); + } + + if j >= units.len() { + break; + } + + // Overlap: re-seed with trailing whole sentences summing ≤ overlap, + // never back to i (guarantee forward progress). + let mut next = j; + if overlap > 0 { + let mut acc = 0usize; + let mut k = j; + while k > i + 1 { + let t = units[k - 1].2; + if acc + t > overlap { + break; + } + acc += t; + k -= 1; + } + next = k.max(i + 1); + } + i = next; + } + chunks +} + +/// Enforce `max_chunks`: if exceeded, merge the overflow tail into the last kept +/// chunk so no document text is dropped. `0` ⇒ unlimited. +pub fn cap_chunks(mut chunks: Vec<(usize, usize)>, max_chunks: usize) -> Vec<(usize, usize)> { + if max_chunks == 0 || chunks.len() <= max_chunks { + return chunks; + } + let last_end = chunks.last().map(|c| c.1).unwrap_or(0); + chunks.truncate(max_chunks); + if let Some(last) = chunks.last_mut() { + last.1 = last_end; // extend to cover everything merged in + } + chunks +} + +/// Pull a chunk's end back to the nearest paragraph/line/sentence/space boundary +/// within `[start, end)`, but never past the chunk midpoint (don't over-shrink). +/// Returns the new end byte offset, or `None` if no good boundary was found. +fn snap_back(text: &str, start: usize, end: usize) -> Option { + let floor = start + (end - start) / 2; + for sep in ["\n\n", "\n", ". ", " "] { + if let Some(pos) = text[start..end].rfind(sep) { + let cut = start + pos + sep.len(); + if cut > floor && cut < end { + return Some(cut); + } + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + fn settings(strategy: u32, max: u32, overlap: u32, max_chunks: u32) -> ChunkSettings { + ChunkSettings { + strategy, + max_tokens: max, + overlap_tokens: overlap, + max_chunks, + } + } + + #[test] + fn strategy_classification() { + assert!(!settings(STRATEGY_TRUNCATE, 0, 0, 0).needs_chunking()); + assert!(settings(STRATEGY_MEAN, 0, 0, 0).needs_chunking()); + assert!(settings(STRATEGY_SENTENCE, 0, 0, 0).needs_chunking()); + assert!(settings(STRATEGY_TRUNCATE, 0, 0, 0).is_single_vector()); + assert!(settings(STRATEGY_MEAN, 0, 0, 0).is_single_vector()); + assert!(!settings(STRATEGY_FIXED, 0, 0, 0).is_single_vector()); + assert!(!settings(STRATEGY_SENTENCE, 0, 0, 0).is_single_vector()); + } + + #[test] + fn effective_max_defaults_clamps_and_floors() { + assert_eq!(effective_max(&settings(STRATEGY_MEAN, 0, 0, 0), 512), 510); + assert_eq!( + effective_max(&settings(STRATEGY_MEAN, 100_000, 0, 0), 512), + 510 + ); + assert_eq!( + effective_max(&settings(STRATEGY_MEAN, 256, 0, 0), 8192), + 256 + ); + assert!(effective_max(&settings(STRATEGY_MEAN, 0, 0, 0), 0) >= 1); + } + + #[test] + fn mean_pool_averages_then_normalizes() { + let m = mean_pool(&[vec![1.0, 0.0], vec![0.0, 1.0]]); + let e = std::f32::consts::FRAC_1_SQRT_2; + assert!((m[0] - e).abs() < 1e-6 && (m[1] - e).abs() < 1e-6); + } + + #[test] + fn mean_pool_single_chunk_is_normalized() { + let m = mean_pool(&[vec![3.0, 4.0]]); + assert!((m[0] - 0.6).abs() < 1e-6 && (m[1] - 0.8).abs() < 1e-6); + } + + #[test] + fn row_offsets_prefix_sum() { + assert_eq!(row_offsets_from_counts(&[3, 1, 2]), vec![0, 3, 4, 6]); + assert_eq!(row_offsets_from_counts(&[1, 1, 1]), vec![0, 1, 2, 3]); + assert_eq!(row_offsets_from_counts(&[]), vec![0]); + } + + #[test] + fn short_text_is_one_chunk() { + assert_eq!(chunk_chars("hello world", 100, 0), vec![(0, 11)]); + } + + #[test] + fn chunk_chars_tiles_document_with_no_overlap() { + let text = "word ".repeat(2000); + let chunks = chunk_chars(&text, 100, 0); + assert!(chunks.len() > 1); + assert_eq!(chunks.first().unwrap().0, 0); + assert_eq!(chunks.last().unwrap().1, text.len()); + for w in chunks.windows(2) { + assert_eq!(w[0].1, w[1].0, "no gap/overlap when overlap=0"); + } + for (a, b) in &chunks { + assert!(b > a && text.is_char_boundary(*a) && text.is_char_boundary(*b)); + } + } + + #[test] + fn chunk_chars_overlap_steps_back() { + let text = "word ".repeat(2000); + let chunks = chunk_chars(&text, 100, 20); + assert!(chunks.len() > 1); + for w in chunks.windows(2) { + assert!(w[1].0 < w[0].1, "overlap: next starts before prev end"); + } + assert_eq!(chunks.last().unwrap().1, text.len()); + } + + #[test] + fn chunk_chars_multibyte_never_splits_a_char() { + let text = "🦀 данные ".repeat(500); + let chunks = chunk_chars(&text, 50, 5); + for (a, b) in &chunks { + assert!(text.is_char_boundary(*a) && text.is_char_boundary(*b)); + } + assert_eq!(chunks.last().unwrap().1, text.len()); + } + + #[test] + fn chunk_sentence_splits_at_uax29_boundary() { + let text = "Aaa bbb ccc. Ddd eee fff."; + let chunks = chunk_sentence(text, 5, 0, None); + assert_eq!(chunks.len(), 2); + assert_eq!(chunks[0].0, 0); + assert_eq!(chunks[1].1, text.len()); + assert_eq!(chunks[0].1, chunks[1].0); + assert!((12..=13).contains(&chunks[0].1)); + } + + #[test] + fn chunk_sentence_packs_and_splits_oversized() { + let text = "S one. S two. S three. S four. S five. S six."; + assert_eq!(chunk_sentence(text, 100, 0, None).len(), 1); + assert!(chunk_sentence(text, 3, 0, None).len() > 1); + // terminator-free over-long "sentence" still splits + let blob = "word ".repeat(1000); + assert!(chunk_sentence(&blob, 100, 0, None).len() > 1); + } + + #[test] + fn chunk_text_routes_every_multivector_strategy() { + let text = "word ".repeat(400); + for s in [ + STRATEGY_FIXED, + STRATEGY_RECURSIVE, + STRATEGY_SENTENCE, + STRATEGY_MEAN, + ] { + let chunks = chunk_text(&text, 50, 0, s, None); + assert!(chunks.len() > 1, "strategy {s} should split"); + assert_eq!(chunks.first().unwrap().0, 0); + assert_eq!(chunks.last().unwrap().1, text.len()); + } + } + + #[test] + fn cap_chunks_merges_tail_and_noops() { + assert_eq!( + cap_chunks(vec![(0, 10), (10, 20), (20, 30), (30, 45)], 2), + vec![(0, 10), (10, 45)] + ); + let two = vec![(0, 10), (10, 20)]; + assert_eq!(cap_chunks(two.clone(), 5), two); + assert_eq!(cap_chunks(two.clone(), 0), two); + } +} diff --git a/embeddings/src/error_handling_test.rs b/embeddings/src/error_handling_test.rs index 3b2d3b70..15ac8897 100644 --- a/embeddings/src/error_handling_test.rs +++ b/embeddings/src/error_handling_test.rs @@ -152,6 +152,9 @@ mod tests { ptr: ptr::null(), len: 0, cap: 0, + row_offsets: ptr::null(), + rows: 0, + offsets_cap: 0, }; // Verify error is set and no vectors are present @@ -194,6 +197,9 @@ mod tests { ptr: float_vecs.as_ptr(), len: float_vecs.len(), cap: float_vecs.capacity(), + row_offsets: ptr::null(), + rows: 0, + offsets_cap: 0, }; // Verify successful result structure diff --git a/embeddings/src/ffi.rs b/embeddings/src/ffi.rs index be53847e..eae47276 100644 --- a/embeddings/src/ffi.rs +++ b/embeddings/src/ffi.rs @@ -1,3 +1,4 @@ +use crate::chunk::ChunkSettings; use crate::model::text_model_wrapper::{ FloatVecResult, StringItem, TextModelResult, TextModelWrapper, }; @@ -18,8 +19,13 @@ type LoadModelFn = extern "C" fn( type FreeModelResultFn = extern "C" fn(TextModelResult); -type MakeVectEmbeddingsFn = - extern "C" fn(&TextModelWrapper, *const StringItem, usize, i32) -> FloatVecResult; +type MakeVectEmbeddingsFn = extern "C" fn( + &TextModelWrapper, + *const StringItem, + usize, + *const ChunkSettings, + i32, +) -> FloatVecResult; type FreeVecResultFn = extern "C" fn(FloatVecResult); @@ -62,7 +68,7 @@ pub struct EmbedLib { const VERSION_STR: &[u8] = concat!(env!("EMBEDDINGS_VERSION_STR"), "\0").as_bytes(); const LIB: EmbedLib = EmbedLib { - version: 4usize, + version: 8usize, version_str: VERSION_STR.as_ptr() as *const c_char, load_model: TextModelWrapper::load_model, free_model_result: TextModelWrapper::free_model_result, diff --git a/embeddings/src/lib.rs b/embeddings/src/lib.rs index d7c11f44..dec5fd09 100644 --- a/embeddings/src/lib.rs +++ b/embeddings/src/lib.rs @@ -1,3 +1,4 @@ +mod chunk; mod error; mod ffi; mod model; @@ -19,6 +20,10 @@ mod error_handling_test; #[cfg(test)] mod panic_guard_test; +pub use chunk::{ + ChunkSettings, STRATEGY_FIXED, STRATEGY_MEAN, STRATEGY_RECURSIVE, STRATEGY_SENTENCE, + STRATEGY_TRUNCATE, +}; pub use error::LibError; pub use ffi::{EmbedLib, GetLibFuncs}; pub use model::TextModel; diff --git a/embeddings/src/model/ffi_test.rs b/embeddings/src/model/ffi_test.rs index 322012fa..2ce3a2e0 100644 --- a/embeddings/src/model/ffi_test.rs +++ b/embeddings/src/model/ffi_test.rs @@ -75,8 +75,13 @@ mod tests { model_ptr as *mut std::ffi::c_void, ) }; - let vec_result = - TextModelWrapper::make_vect_embeddings(&wrapper, items.as_ptr(), 1, 0); + let vec_result = TextModelWrapper::make_vect_embeddings( + &wrapper, + items.as_ptr(), + 1, + std::ptr::null(), + 0, + ); assert!(vec_result.error.is_null()); assert_eq!(vec_result.len, 1); TextModelWrapper::free_vec_result(vec_result); @@ -131,6 +136,9 @@ mod tests { ptr: ptr::null(), len: 0, cap: 0, + row_offsets: ptr::null(), + rows: 0, + offsets_cap: 0, }; assert!(result.error.is_null()); @@ -339,6 +347,9 @@ mod tests { ptr: ptr::null(), len: 0, cap: 0, + row_offsets: ptr::null(), + rows: 0, + offsets_cap: 0, }; // Verify error is set @@ -365,6 +376,9 @@ mod tests { ptr: ptr::null(), len: 0, cap: 0, + row_offsets: ptr::null(), + rows: 0, + offsets_cap: 0, }; // Should not crash with null pointers @@ -394,12 +408,15 @@ mod tests { mem::size_of::<*const c_char>() + mem::size_of::() ); - // FloatVecResult should be pointer + two usizes + error pointer + // FloatVecResult: error ptr + embeddings ptr + len + cap + // + row_offsets ptr + rows + offsets_cap assert_eq!( mem::size_of::(), mem::size_of::<*mut c_char>() + mem::size_of::<*const FloatVec>() + mem::size_of::() * 2 + + mem::size_of::<*const usize>() + + mem::size_of::() * 2 ); } @@ -501,4 +518,161 @@ mod tests { fn test_concurrent_qwen_embeddings_via_ffi() { run_concurrent_ffi_embeddings("Qwen/Qwen3-Embedding-0.6B"); } + + /// End-to-end on the cached MiniLM model: all five strategies. truncate/mean + /// return one vector/doc; fixed/recursive/sentence return N vectors/doc + /// grouped by row_offsets. Verifies offsets, normalization, and clean frees. + #[test] + fn test_all_strategies_local_model() { + let model_name = to_c_string("sentence-transformers/all-MiniLM-L6-v2"); + let empty = to_c_string(""); + let loaded = TextModelWrapper::load_model( + model_name.as_ptr(), + model_name.as_bytes().len(), + empty.as_ptr(), + empty.as_bytes().len(), + empty.as_ptr(), + empty.as_bytes().len(), + empty.as_ptr(), + empty.as_bytes().len(), + 0, + false, + ); + if loaded.model.is_null() { + TextModelWrapper::free_model_result(loaded); + eprintln!("skipping: all-MiniLM-L6-v2 not available locally"); + return; + } + let wrapper = + unsafe { std::mem::transmute::<*mut std::ffi::c_void, TextModelWrapper>(loaded.model) }; + + // Long enough to exceed the model's input limit, so truncate drops the + // tail while mean covers the whole document via several chunks. + let long = "Vector search turns text into embeddings for retrieval. ".repeat(100); + let items = [create_string_item(&long)]; + + let first_vec = |res: &FloatVecResult| -> Vec { + assert!(res.error.is_null()); + assert_eq!(res.len, 1, "exactly one vector per input document"); + assert_eq!(res.rows, 1); + assert!(!res.row_offsets.is_null()); + unsafe { + let offs = std::slice::from_raw_parts(res.row_offsets, res.rows + 1); + assert_eq!(offs, &[0usize, 1usize], "trivial offsets for 1 vector/doc"); + let fv = &*res.ptr; + std::slice::from_raw_parts(fv.ptr, fv.len).to_vec() + } + }; + + // truncate (null settings) + let trunc = + TextModelWrapper::make_vect_embeddings(&wrapper, items.as_ptr(), 1, ptr::null(), 0); + let trunc_vec = first_vec(&trunc); + + // mean: small chunk size forces many chunks → embed each → average + let settings = crate::chunk::ChunkSettings { + strategy: crate::chunk::STRATEGY_MEAN, + max_tokens: 32, + overlap_tokens: 0, + max_chunks: 0, + }; + let mean = + TextModelWrapper::make_vect_embeddings(&wrapper, items.as_ptr(), 1, &settings, 0); + let mean_vec = first_vec(&mean); + + assert_eq!(trunc_vec.len(), mean_vec.len(), "same embedding dimension"); + let diff: f32 = trunc_vec + .iter() + .zip(&mean_vec) + .map(|(a, b)| (a - b).abs()) + .sum(); + assert!( + diff > 1e-3, + "mean must differ from truncate on a long document" + ); + let norm: f32 = mean_vec.iter().map(|x| x * x).sum::().sqrt(); + assert!( + (norm - 1.0).abs() < 1e-2, + "mean vector must be L2-normalized" + ); + + // multi-vector strategies: fixed/recursive/sentence keep every chunk + // vector → N vectors/doc, grouped by row_offsets [0, N]. + for strat in [ + crate::chunk::STRATEGY_FIXED, + crate::chunk::STRATEGY_RECURSIVE, + crate::chunk::STRATEGY_SENTENCE, + ] { + let s = crate::chunk::ChunkSettings { + strategy: strat, + max_tokens: 32, + overlap_tokens: 0, + max_chunks: 0, + }; + let res = TextModelWrapper::make_vect_embeddings(&wrapper, items.as_ptr(), 1, &s, 0); + assert!(res.error.is_null(), "strategy {strat} errored"); + assert_eq!(res.rows, 1, "one document"); + assert!( + res.len > 1, + "strategy {strat} must emit multiple chunk vectors, got {}", + res.len + ); + unsafe { + let offs = std::slice::from_raw_parts(res.row_offsets, res.rows + 1); + assert_eq!(offs[0], 0); + assert_eq!(offs[1], res.len, "row 0 owns all {} chunk vectors", res.len); + let fvs = std::slice::from_raw_parts(res.ptr, res.len); + for fv in fvs { + assert_eq!(fv.len, trunc_vec.len(), "chunk vector dimension"); + let v = std::slice::from_raw_parts(fv.ptr, fv.len); + let norm: f32 = v.iter().map(|x| x * x).sum::().sqrt(); + assert!((norm - 1.0).abs() < 1e-2, "chunk vector must be normalized"); + } + } + TextModelWrapper::free_vec_result(res); + } + + // multi-document batch: row_offsets must group N-per-doc correctly, and + // overlap_tokens > 0 must still produce valid output. doc A is long + // (many chunks), doc B is short (one chunk). + let short = "One short sentence."; + let batch = [create_string_item(&long), create_string_item(short)]; + let s = crate::chunk::ChunkSettings { + strategy: crate::chunk::STRATEGY_FIXED, + max_tokens: 32, + overlap_tokens: 4, + max_chunks: 0, + }; + let res = TextModelWrapper::make_vect_embeddings(&wrapper, batch.as_ptr(), 2, &s, 0); + assert!(res.error.is_null()); + assert_eq!(res.rows, 2, "two documents"); + unsafe { + let offs = std::slice::from_raw_parts(res.row_offsets, res.rows + 1); + assert_eq!(offs[0], 0); + assert_eq!(offs[2], res.len, "last offset == total vectors"); + assert!(offs[1] > 1, "doc A (long) has multiple chunks"); + assert_eq!(offs[2] - offs[1], 1, "doc B (short) has exactly one chunk"); + } + TextModelWrapper::free_vec_result(res); + + // max_chunks cap: the long doc is capped to ≤ 3 vectors (tail merged). + let capped = crate::chunk::ChunkSettings { + strategy: crate::chunk::STRATEGY_FIXED, + max_tokens: 32, + overlap_tokens: 0, + max_chunks: 3, + }; + let res = TextModelWrapper::make_vect_embeddings(&wrapper, items.as_ptr(), 1, &capped, 0); + assert!(res.error.is_null()); + assert!( + (1..=3).contains(&res.len), + "max_chunks=3 caps to ≤3 vectors, got {}", + res.len + ); + TextModelWrapper::free_vec_result(res); + + TextModelWrapper::free_vec_result(trunc); + TextModelWrapper::free_vec_result(mean); + TextModelWrapper::free_model_result(loaded); + } } diff --git a/embeddings/src/model/local.rs b/embeddings/src/model/local.rs index aac6e9da..ce87400d 100644 --- a/embeddings/src/model/local.rs +++ b/embeddings/src/model/local.rs @@ -1451,6 +1451,19 @@ impl LocalModel { } } +impl LocalModel { + /// Borrow the loaded tokenizer for token-accurate document chunking. + fn tokenizer(&self) -> &Tokenizer { + match self { + LocalModel::Bert(m) => &m.tokenizer, + LocalModel::T5(m) => &m.tokenizer, + LocalModel::Causal(m) => &m.tokenizer, + LocalModel::Quantized(m) => &m.tokenizer, + LocalModel::Onnx(m) => &m.tokenizer, + } + } +} + impl TextModel for LocalModel { fn predict(&self, texts: &[&str], threads: usize) -> Result>, Box> { // ONNX manages its own worker count internally — no rayon pool involved. @@ -1487,6 +1500,18 @@ impl TextModel for LocalModel { // Local models don't use API keys, so validation is always successful Ok(()) } + + fn chunk( + &self, + text: &str, + max_tokens: usize, + overlap: usize, + strategy: u32, + ) -> Vec<(usize, usize)> { + // Token-accurate split via the loaded tokenizer; chunk_text picks the + // split method (fixed/recursive/sentence) from the strategy. + crate::chunk::chunk_text(text, max_tokens, overlap, strategy, Some(self.tokenizer())) + } } #[cfg(test)] diff --git a/embeddings/src/model/mod.rs b/embeddings/src/model/mod.rs index cfa48d72..6ca2a0d7 100644 --- a/embeddings/src/model/mod.rs +++ b/embeddings/src/model/mod.rs @@ -36,6 +36,21 @@ pub trait TextModel { /// Validates the API key by making a minimal test request to the API. /// For remote models, this makes an actual HTTP request. For local models, this is a no-op. fn validate_api_key(&self) -> Result<(), Box>; + + /// Split one document into chunk byte spans `(start, end)` per `strategy`, + /// sized to `max_tokens` with `overlap` token overlap. Default impl is the + /// char/byte heuristic for remote API models (no local tokenizer); + /// `LocalModel` overrides it with token-accurate chunking via its loaded + /// tokenizer. Used by every strategy except `truncate`. + fn chunk( + &self, + text: &str, + max_tokens: usize, + overlap: usize, + strategy: u32, + ) -> Vec<(usize, usize)> { + crate::chunk::chunk_text(text, max_tokens, overlap, strategy, None) + } } #[repr(C)] @@ -105,6 +120,21 @@ impl TextModel for Model { Model::Local(m) => m.validate_api_key(), } } + + fn chunk( + &self, + text: &str, + max_tokens: usize, + overlap: usize, + strategy: u32, + ) -> Vec<(usize, usize)> { + match self { + Model::OpenAI(m) => m.chunk(text, max_tokens, overlap, strategy), + Model::Voyage(m) => m.chunk(text, max_tokens, overlap, strategy), + Model::Jina(m) => m.chunk(text, max_tokens, overlap, strategy), + Model::Local(m) => m.chunk(text, max_tokens, overlap, strategy), + } + } } /// Refuse local (candle) inference on an emulated x86 CPU. Rosetta 2 and QEMU diff --git a/embeddings/src/model/text_model_wrapper.rs b/embeddings/src/model/text_model_wrapper.rs index 638c8769..9d75474c 100644 --- a/embeddings/src/model/text_model_wrapper.rs +++ b/embeddings/src/model/text_model_wrapper.rs @@ -1,8 +1,14 @@ +use crate::chunk::ChunkSettings; use crate::model::{create_model, Model, ModelOptions, TextModel}; use crate::panic_guard; use std::os::raw::c_char; use std::{ffi::c_void, ptr}; +/// Per-document output vectors (flat across all documents) plus per-document +/// vector counts, used to build the row-offsets sidecar. One inner `Vec` +/// per emitted vector. +type EmbedResult = Result<(Vec>, Vec), Box>; + /// Sentinel written at offset 0 of every live model handle. Lets FFI entry /// points detect garbage, null, or freed pointers handed in by the C++ caller /// and return a clean error instead of dereferencing into UB. @@ -60,13 +66,27 @@ pub struct FloatVec { pub cap: usize, } -/// cbindgen:field-names=[m_szError,m_tEmbedding,len,cap] +/// Embedding result for one `make_vect_embeddings` call. +/// +/// `m_tEmbedding` is a FLAT array of `len` vectors — every input document's +/// vectors concatenated. `m_pRowOffsets` (length `rows + 1`) groups them per +/// input document, Arrow-style: document `i` owns +/// `m_tEmbedding[m_pRowOffsets[i] .. m_pRowOffsets[i + 1]]`. For the v1 +/// strategies (truncate / mean) every document yields exactly one vector, so +/// `len == rows` and the offsets are `[0, 1, ..., rows]`. The sidecar lets a +/// future multi-vector strategy return N vectors per document through this same +/// struct — no second method, cardinality carried as data. +/// +/// cbindgen:field-names=[m_szError,m_tEmbedding,len,cap,m_pRowOffsets,rows,offsets_cap] #[repr(C)] pub struct FloatVecResult { pub error: *mut c_char, pub ptr: *const FloatVec, pub len: usize, pub cap: usize, + pub row_offsets: *const usize, + pub rows: usize, + pub offsets_cap: usize, } #[repr(C)] @@ -201,10 +221,20 @@ impl TextModelWrapper { } } + /// Generate embeddings for a batch of documents. `settings` selects the + /// strategy (null ⇒ `truncate`): + /// - `truncate` — embed the first `max_tokens` tokens (one vector/doc). + /// - `mean` — split the doc, embed every chunk, average into one vector/doc. + /// - `fixed`/`recursive`/`sentence` — split the doc and keep every chunk + /// vector (N vectors/doc). + /// + /// Output is a flat `FloatVec[]`; `m_pRowOffsets` groups it per document + /// (1 vector/doc for truncate/mean, N for the rest). `threads`: 0 = all CPUs. pub extern "C" fn make_vect_embeddings( &self, texts: *const StringItem, count: usize, + settings: *const ChunkSettings, threads: i32, // 0 = use all available CPUs, >0 = cap thread count ) -> FloatVecResult { panic_guard::catch_panic(|| { @@ -217,6 +247,9 @@ impl TextModelWrapper { ptr: ptr::null(), len: 0, cap: 0, + row_offsets: ptr::null(), + rows: 0, + offsets_cap: 0, }; } }; @@ -234,43 +267,111 @@ impl TextModelWrapper { .collect(); let threads = if threads > 0 { threads as usize } else { 0 }; + let settings_ref = unsafe { settings.as_ref() }; + + let strategy = settings_ref + .map(|s| s.strategy) + .unwrap_or(crate::chunk::STRATEGY_TRUNCATE); + + // Each strategy yields a flat Vec> (all documents' output + // vectors) plus per-document output counts. truncate/mean emit one + // vector/doc; fixed/recursive/sentence emit one vector per chunk. + let computed: EmbedResult = if strategy == crate::chunk::STRATEGY_TRUNCATE { + model.predict(&string_refs, threads).map(|vecs| { + let counts = vec![1usize; vecs.len()]; + (vecs, counts) + }) + } else { + let s = settings_ref.unwrap(); + let max = crate::chunk::effective_max(s, model.get_max_input_len()); + let overlap = s.overlap_tokens as usize; + let max_chunks = s.max_chunks as usize; + + // Split every document; remember each document's chunk count. + let mut flat: Vec<&str> = Vec::new(); + let mut chunk_counts: Vec = Vec::with_capacity(string_refs.len()); + for &text in &string_refs { + let spans = crate::chunk::cap_chunks( + model.chunk(text, max, overlap, strategy), + max_chunks, + ); + chunk_counts.push(spans.len()); + for (start, end) in spans { + flat.push(&text[start..end]); + } + } - let mut float_vec_list: Vec = Vec::new(); - let embeddings_list = model.predict(&string_refs, threads); - let c_error = match embeddings_list { - Ok(embeddings_list) => { - for embeddings in embeddings_list.iter() { - let ptr = embeddings.as_ptr(); - let len = embeddings.len(); - let cap = embeddings.capacity(); - let vec = FloatVec { ptr, len, cap }; - float_vec_list.push(vec); + model.predict(&flat, threads).map(|chunk_vecs| { + if strategy == crate::chunk::STRATEGY_MEAN { + // pool each document's chunks into one vector + let mut out = Vec::with_capacity(chunk_counts.len()); + let mut idx = 0usize; + for c in &chunk_counts { + let end = idx + c; + out.push(crate::chunk::mean_pool(&chunk_vecs[idx..end])); + idx = end; + } + let counts = vec![1usize; out.len()]; + (out, counts) + } else { + // fixed/recursive/sentence: keep every chunk vector. + (chunk_vecs, chunk_counts) } + }) + }; + match computed { + Ok((embeddings_list, counts)) => { + let mut float_vec_list: Vec = + Vec::with_capacity(embeddings_list.len()); + for embeddings in embeddings_list.iter() { + float_vec_list.push(FloatVec { + ptr: embeddings.as_ptr(), + len: embeddings.len(), + cap: embeddings.capacity(), + }); + } std::mem::forget(embeddings_list); - ptr::null_mut() - } - Err(e) => { - // Don't push empty vector on error - return error through szError pattern - let c_error = std::ffi::CString::new(e.to_string()).unwrap(); - c_error.into_raw() - } - }; - let vec_result = FloatVecResult { - ptr: float_vec_list.as_ptr(), - len: float_vec_list.len(), - cap: float_vec_list.capacity(), - error: c_error, - }; - std::mem::forget(float_vec_list); - vec_result + // Group the flat vectors per document: truncate/mean → one + // vector/doc (offsets [0,1,..]); fixed/recursive/sentence → + // N vectors/doc, offsets = prefix sums of per-doc counts. + let row_offsets = crate::chunk::row_offsets_from_counts(&counts); + let rows = counts.len(); + + let result = FloatVecResult { + error: ptr::null_mut(), + ptr: float_vec_list.as_ptr(), + len: float_vec_list.len(), + cap: float_vec_list.capacity(), + row_offsets: row_offsets.as_ptr(), + rows, + offsets_cap: row_offsets.capacity(), + }; + std::mem::forget(float_vec_list); + std::mem::forget(row_offsets); + result + } + // Don't push empty vectors on error — surface it via szError. + Err(e) => FloatVecResult { + error: to_c_error(&e.to_string()), + ptr: ptr::null(), + len: 0, + cap: 0, + row_offsets: ptr::null(), + rows: 0, + offsets_cap: 0, + }, + } }) .unwrap_or_else(|msg| FloatVecResult { error: to_c_error(&format!("embeddings: internal error (panic): {msg}")), ptr: ptr::null(), len: 0, cap: 0, + row_offsets: ptr::null(), + rows: 0, + offsets_cap: 0, }) } @@ -293,6 +394,15 @@ impl TextModelWrapper { let _ = Vec::from_raw_parts(result.ptr as *mut FloatVec, result.len, result.cap); } + // Free the per-row offsets sidecar (length = rows + 1). + if !result.row_offsets.is_null() && result.offsets_cap > 0 { + let _ = Vec::from_raw_parts( + result.row_offsets as *mut usize, + result.rows + 1, + result.offsets_cap, + ); + } + // Free the error string if it exists if !result.error.is_null() { let _ = std::ffi::CString::from_raw(result.error); diff --git a/knn/embeddings.cpp b/knn/embeddings.cpp index 9b3a1f48..d995c364 100644 --- a/knn/embeddings.cpp +++ b/knn/embeddings.cpp @@ -265,7 +265,9 @@ bool TextToEmbeddings_c::Convert ( const std::vector & dTexts, assert(pFuncs); // iThreads: 0 = use all available CPUs (default), >0 = cap worker count in the embeddings lib - FloatVecResult tVecResult = pFuncs->make_vect_embeddings ( &m_pModel, dStringItems.data(), dStringItems.size(), iThreads ); + // nullptr ChunkSettings = truncate strategy (today's behavior). Pass a populated + // ChunkSettings (e.g. STRATEGY_MEAN) once the table DDL exposes a chunking option. + FloatVecResult tVecResult = pFuncs->make_vect_embeddings ( &m_pModel, dStringItems.data(), dStringItems.size(), nullptr, iThreads ); if ( tVecResult.m_szError ) { sError = tVecResult.m_szError; @@ -273,6 +275,9 @@ bool TextToEmbeddings_c::Convert ( const std::vector & dTexts, return false; } + // truncate/mean return one vector per input row, so tVecResult.len == rows + // and this 1:1 copy is correct. A future multi-vector strategy (N vectors + // per row) would instead group via tVecResult.m_pRowOffsets[i..i+1]. dEmbeddings.resize ( tVecResult.len ); for ( size_t i = 0; i < tVecResult.len; i++ ) { @@ -312,7 +317,7 @@ knn::EmbeddingsLib_i * LoadEmbeddingsLib ( const std::string & sLibPath, std::st if ( !pLib->Load(sError) ) return nullptr; - const int SUPPORTED_EMBEDDINGS_LIB_VER = 4; + const int SUPPORTED_EMBEDDINGS_LIB_VER = 8; if ( pLib->GetVersion()!=SUPPORTED_EMBEDDINGS_LIB_VER ) { sError = util::FormatStr ( "Unsupported embeddings library version %d (expected %d)", pLib->GetVersion(), SUPPORTED_EMBEDDINGS_LIB_VER );