diff --git a/Cargo.lock b/Cargo.lock index 3adeccf..e4eaa89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,65 +4,119 @@ version = 4 [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] [[package]] -name = "anyhow" -version = "1.0.96" +name = "anstream" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] [[package]] -name = "bitflags" -version = "2.8.0" +name = "anstyle" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] -name = "bumpalo" -version = "3.17.0" +name = "anstyle-parse" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] [[package]] -name = "c2rust-bitfields" -version = "0.3.0" +name = "anstyle-query" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb34f0c0ace43530b2df7f18bc69ee0c4082158aa451ece29602f8c841e73764" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "c2rust-bitfields-derive", + "windows-sys 0.61.2", ] [[package]] -name = "c2rust-bitfields-derive" -version = "0.2.1" +name = "anstyle-wincon" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dd1601a7b828ab874d890e5a895563ca8ad485bdd3d2a359f148c8b72537241" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" -version = "1.2.14" +version = "1.2.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" +checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" dependencies = [ + "find-msvc-tools", "shlex", ] [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "codemap" @@ -80,28 +134,35 @@ dependencies = [ "termcolor", ] +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "console" -version = "0.15.10" +version = "0.15.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" dependencies = [ "encode_unicode", "libc", "once_cell", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] -name = "displaydoc" -version = "0.2.5" +name = "dunce" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "encode_unicode" @@ -109,6 +170,29 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "env_filter" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -117,12 +201,12 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.10" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -132,323 +216,312 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] -name = "form_urlencoded" -version = "1.2.1" +name = "find-msvc-tools" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" -dependencies = [ - "percent-encoding", -] +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] -name = "getrandom" -version = "0.3.1" +name = "foldhash" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" -dependencies = [ - "cfg-if", - "libc", - "wasi", - "windows-targets", -] +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] -name = "hashbrown" -version = "0.15.2" +name = "futures-core" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] -name = "heck" -version = "0.5.0" +name = "futures-task" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] -name = "icu_collections" -version = "1.5.0" +name = "futures-util" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ - "displaydoc", - "yoke", - "zerofrom", - "zerovec", + "futures-core", + "futures-task", + "pin-project-lite", + "slab", ] [[package]] -name = "icu_locid" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_locid_transform" -version = "1.5.0" +name = "getrandom" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", ] [[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - -[[package]] -name = "icu_normalizer" -version = "1.5.0" +name = "hashbrown" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "utf16_iter", - "utf8_iter", - "write16", - "zerovec", + "foldhash", ] [[package]] -name = "icu_normalizer_data" -version = "1.5.0" +name = "hashbrown" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" [[package]] -name = "icu_properties" -version = "1.5.1" +name = "heck" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locid_transform", - "icu_properties_data", - "icu_provider", - "tinystr", - "zerovec", -] +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "icu_properties_data" -version = "1.5.0" +name = "id-arena" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" [[package]] -name = "icu_provider" -version = "1.5.0" +name = "indexmap" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ - "displaydoc", - "icu_locid", - "icu_provider_macros", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerovec", + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", ] [[package]] -name = "icu_provider_macros" -version = "1.5.0" +name = "indoc" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", + "rustversion", ] [[package]] -name = "idna" -version = "1.0.3" +name = "insta" +version = "1.46.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +checksum = "e82db8c87c7f1ccecb34ce0c24399b8a73081427f3c7c50a5d597925356115e4" dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", + "console", + "once_cell", + "similar", + "tempfile", ] [[package]] -name = "idna_adapter" -version = "1.2.0" +name = "is_terminal_polyfill" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" -dependencies = [ - "icu_normalizer", - "icu_properties", -] +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] -name = "indexmap" -version = "2.7.1" +name = "itertools" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ - "equivalent", - "hashbrown", + "either", ] [[package]] -name = "indoc" -version = "2.0.5" +name = "itoa" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] -name = "insta" -version = "1.42.1" +name = "jiff" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c1b125e30d93896b365e156c33dadfffab45ee8400afcbba4752f59de08a86" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ - "console", - "linked-hash-map", - "once_cell", - "pin-project", - "similar", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", ] [[package]] -name = "itoa" -version = "1.0.14" +name = "jiff-static" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" dependencies = [ "once_cell", "wasm-bindgen", ] [[package]] -name = "libc" -version = "0.2.169" +name = "leb128fmt" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] -name = "linked-hash-map" -version = "0.5.6" +name = "libc" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] -name = "linux-raw-sys" -version = "0.4.15" +name = "libm" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] -name = "litemap" -version = "0.7.4" +name = "linux-raw-sys" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "log" -version = "0.4.25" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "memchr" -version = "2.7.4" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "minicov" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27fe9f1cc3c22e1687f9446c2083c4c5fc7f0bcf1c7a86bdbded14985895b4b" +checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" dependencies = [ "cc", "walkdir", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + [[package]] name = "once_cell" -version = "1.20.3" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] -name = "percent-encoding" -version = "2.3.1" +name = "pin-project-lite" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] -name = "pin-project" -version = "1.1.9" +name = "portable-atomic" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5" dependencies = [ - "pin-project-internal", + "portable-atomic", ] [[package]] -name = "pin-project-internal" -version = "1.1.9" +name = "prettyplease" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "quote", - "syn 2.0.98", + "syn", ] [[package]] name = "proc-macro2" -version = "1.0.93" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.38" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "regex" -version = "1.11.1" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -458,9 +531,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -469,35 +542,43 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "rust-sitter" -version = "0.4.5" +version = "0.5.0" dependencies = [ "insta", + "log", "rust-sitter-macro", + "rust-sitter-types", + "serde", + "serde_json", "tempfile", "tree-sitter", - "tree-sitter-c2rust", ] [[package]] name = "rust-sitter-common" -version = "0.4.5" +version = "0.5.0" dependencies = [ + "itertools", + "proc-macro2", "quote", - "syn 2.0.98", + "rust-sitter-types", + "serde_json", + "syn", ] [[package]] name = "rust-sitter-example" -version = "0.4.5" +version = "0.5.0" dependencies = [ "codemap", "codemap-diagnostic", + "env_logger", "insta", "rust-sitter", "rust-sitter-tool", @@ -506,32 +587,42 @@ dependencies = [ [[package]] name = "rust-sitter-macro" -version = "0.4.5" +version = "0.5.0" dependencies = [ "insta", "proc-macro2", "quote", "rust-sitter-common", - "syn 2.0.98", + "rust-sitter-types", + "syn", "tempfile", ] [[package]] name = "rust-sitter-tool" -version = "0.4.5" +version = "0.5.0" dependencies = [ "cc", "insta", "rust-sitter-common", "serde", "serde_json", - "syn 2.0.98", + "syn", "syn-inline-mod", "tempfile", "tree-sitter", "tree-sitter-generate", ] +[[package]] +name = "rust-sitter-types" +version = "0.5.0" +dependencies = [ + "indexmap", + "serde", + "serde_json", +] + [[package]] name = "rustc-hash" version = "2.1.1" @@ -540,22 +631,22 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustix" -version = "0.38.44" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] -name = "ryu" -version = "1.0.19" +name = "rustversion" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "same-file" @@ -568,44 +659,56 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.25" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" dependencies = [ "serde", + "serde_core", ] [[package]] name = "serde" -version = "1.0.218" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.218" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn", ] [[package]] name = "serde_json" -version = "1.0.139" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "indexmap", "itoa", "memchr", - "ryu", "serde", + "serde_core", + "zmij", ] [[package]] @@ -621,22 +724,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] -name = "smallbitvec" -version = "2.6.0" +name = "slab" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d31d263dd118560e1a492922182ab6ca6dc1d03a3bf54e7699993f31a4150e3f" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] -name = "smallvec" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" +name = "smallbitvec" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +checksum = "d31d263dd118560e1a492922182ab6ca6dc1d03a3bf54e7699993f31a4150e3f" [[package]] name = "streaming-iterator" @@ -646,20 +743,9 @@ checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" [[package]] name = "syn" -version = "1.0.109" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.98" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -673,32 +759,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fa6dca1fdb7b2ed46dd534a326725419d4fb10f23d8c85a8b2860e5eb25d0f9" dependencies = [ "proc-macro2", - "syn 2.0.98", -] - -[[package]] -name = "synstructure" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", + "syn", ] [[package]] name = "tempfile" -version = "3.17.1" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e5a0acb1f3f55f65cc4a866c361b2fb2a0ff6366785ae6fbb5f85df07ba230" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ - "cfg-if", "fastrand", "getrandom", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -712,39 +786,34 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.11" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.11" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn", ] [[package]] -name = "tinystr" -version = "0.7.6" +name = "topological-sort" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" -dependencies = [ - "displaydoc", - "zerovec", -] +checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d" [[package]] name = "tree-sitter" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5168a515fe492af54c5cc8800ff8c840be09fa5168de45838afaecd3e008bce4" +version = "0.27.0" +source = "git+https://github.com/jaboatman/tree-sitter?branch=combined#e5a0e2d43efe5ca11fc5c1555390b617880cded6" dependencies = [ "cc", "regex", @@ -754,28 +823,13 @@ dependencies = [ "tree-sitter-language", ] -[[package]] -name = "tree-sitter-c2rust" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1e4909668d7a5e0eb8d66bd9fe2e789106066626056d3bead80dc6b9d5aeee" -dependencies = [ - "c2rust-bitfields", - "once_cell", - "regex", - "regex-syntax", - "streaming-iterator", - "tree-sitter-language", -] - [[package]] name = "tree-sitter-generate" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05beaa8023e08df928535c282cf684c4ca5151475348e73457161f76a50778e8" +version = "0.27.0" +source = "git+https://github.com/jaboatman/tree-sitter?branch=combined#e5a0e2d43efe5ca11fc5c1555390b617880cded6" dependencies = [ - "anyhow", - "heck", + "bitflags", + "dunce", "indexmap", "indoc", "log", @@ -787,45 +841,31 @@ dependencies = [ "serde_json", "smallbitvec", "thiserror", - "tree-sitter", - "url", + "topological-sort", ] [[package]] name = "tree-sitter-language" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4013970217383f67b18aef68f6fb2e8d409bc5755227092d32efb0422ba24b8" +version = "0.1.7" +source = "git+https://github.com/jaboatman/tree-sitter?branch=combined#e5a0e2d43efe5ca11fc5c1555390b617880cded6" [[package]] name = "unicode-ident" -version = "1.0.17" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] -name = "url" -version = "2.5.4" +name = "unicode-xid" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] -name = "utf16_iter" -version = "1.0.5" +name = "utf8parse" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "walkdir" @@ -838,46 +878,44 @@ dependencies = [ ] [[package]] -name = "wasi" -version = "0.13.3+wasi-0.2.2" +name = "wasip2" +version = "1.0.2+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" dependencies = [ - "wit-bindgen-rt", + "wit-bindgen", ] [[package]] -name = "wasm-bindgen" -version = "0.2.100" +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "cfg-if", - "once_cell", - "wasm-bindgen-macro", + "wit-bindgen", ] [[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" +name = "wasm-bindgen" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.98", + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.50" +version = "0.4.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" dependencies = [ "cfg-if", + "futures-util", "js-sys", "once_cell", "wasm-bindgen", @@ -886,9 +924,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -896,55 +934,104 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.98", - "wasm-bindgen-backend", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-bindgen-test" -version = "0.3.50" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66c8d5e33ca3b6d9fa3b4676d774c5778031d27a578c2b007f905acf816152c3" +checksum = "6311c867385cc7d5602463b31825d454d0837a3aba7cdb5e56d5201792a3f7fe" dependencies = [ + "async-trait", + "cast", "js-sys", + "libm", "minicov", + "nu-ansi-term", + "num-traits", + "oorandom", + "serde", + "serde_json", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test-macro", + "wasm-bindgen-test-shared", ] [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.50" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17d5042cc5fa009658f9a7333ef24291b1291a25b6382dd68862a7f3b969f69b" +checksum = "67008cdde4769831958536b0f11b3bdd0380bde882be17fff9c2f34bb4549abd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn", +] + +[[package]] +name = "wasm-bindgen-test-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe29135b180b72b04c74aa97b2b4a2ef275161eff9a6c7955ea9eaedc7e1d4e" + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", ] [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" dependencies = [ "js-sys", "wasm-bindgen", @@ -952,13 +1039,19 @@ dependencies = [ [[package]] name = "winapi-util" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-sys" version = "0.59.0" @@ -968,6 +1061,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -1033,89 +1135,95 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "wit-bindgen-rt" -version = "0.33.0" +name = "wit-bindgen" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" dependencies = [ - "bitflags", + "wit-bindgen-rust-macro", ] [[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - -[[package]] -name = "writeable" -version = "0.5.5" +name = "wit-bindgen-core" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" - -[[package]] -name = "yoke" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", + "anyhow", + "heck", + "wit-parser", ] [[package]] -name = "yoke-derive" -version = "0.7.5" +name = "wit-bindgen-rust" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", - "synstructure", + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", ] [[package]] -name = "zerofrom" -version = "0.1.5" +name = "wit-bindgen-rust-macro" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" dependencies = [ - "zerofrom-derive", + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", ] [[package]] -name = "zerofrom-derive" -version = "0.1.5" +name = "wit-component" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", - "synstructure", + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", ] [[package]] -name = "zerovec" -version = "0.10.4" +name = "wit-parser" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", ] [[package]] -name = "zerovec-derive" -version = "0.10.3" +name = "zmij" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml index 20ca59c..f451268 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,5 +5,15 @@ members = [ "runtime", "tool", "example", - "common", + "common", "types", ] +[workspace.package] +version = "0.5.0" +authors = [ + "Jason Boatman", + "Shadaj Laddad " +] + +[workspace.dependencies] +tree-sitter = { git = "https://github.com/jaboatman/tree-sitter", branch = "combined" } +tree-sitter-generate = { git = "https://github.com/jaboatman/tree-sitter", branch = "combined", default-features = false, features = ["load"] } diff --git a/README.md b/README.md index 550491e..f9726c5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ -# Rust Sitter -[![Crates.io](https://img.shields.io/crates/v/rust-sitter)](https://crates.io/crates/rust-sitter) +# Rust Sitter - Otonoma fork +**This project is a fork of [rust-sitter](https://github.com/hydro-project/rust-sitter). It has been heavily +modified in many breaking ways.** Rust Sitter makes it easy to create efficient parsers in Rust by leveraging the [Tree Sitter](https://tree-sitter.github.io/tree-sitter/) parser generator. With Rust Sitter, you can define your entire grammar with annotations on idiomatic Rust code, and let macros generate the parser and type-safe bindings for you! @@ -7,14 +8,12 @@ Rust Sitter makes it easy to create efficient parsers in Rust by leveraging the First, add Rust/Tree Sitter to your `Cargo.toml`: ```toml [dependencies] -rust-sitter = "0.4.5" +rust-sitter = { git = "https://github.com/otonoma/rust-sitter" } [build-dependencies] -rust-sitter-tool = "0.4.5" +rust-sitter-tool = { git = "https://github.com/otonoma/rust-sitter" } ``` -_Note: By default, Rust Sitter uses a fork of Tree Sitter with a pure-Rust runtime to support `wasm32-unknown-unknown`. To use the standard C runtime instead, disable default features and enable the `tree-sitter-standard` feature_ - The first step is to configure your `build.rs` to compile and link the generated Tree Sitter parser: ```rust @@ -22,45 +21,46 @@ use std::path::PathBuf; fn main() { println!("cargo:rerun-if-changed=src"); - rust_sitter_tool::build_parsers(&PathBuf::from("src/main.rs")); + // Path to the file containing your grammar and any submodules. + rust_sitter_tool::build_parsers("src/grammar/mod.rs")); } ``` ## Defining a Grammar -Now that we have Rust Sitter added to our project, we can define our grammar. Rust Sitter grammars are defined in annotated Rust modules. First, we define the module that will contain our grammar - -```rust -#[rust_sitter::grammar("arithmetic")] -mod grammar { - -} -``` +Now that we have Rust Sitter added to our project, we can define our grammar. Rust Sitter grammars are defined in Rust modules. First, we create a module file for the grammar in `src/grammar/mod.rs`. Note, this can be any module, however, +due to various quirks with the build system it is required that you have one grammar per module, and all types +in the grammar are defined within it, or a submodule of the module. -Then, inside the module, we can define individual AST nodes. For this simple example, we'll define an expression that can be used in a mathematical expression. Note that we annotate this type as `#[rust_sitter::language]` to indicate that it is the root AST type. +Then, inside the module, we can define individual AST nodes. For this simple example, we'll define an expression that can be used in a mathematical expression. Note that we annotate this type as `#[language]` to indicate that it is the root AST type. ```rust -#[rust_sitter::language] +// in ./src/grammar/mod.rs +use rust_sitter::Rule; +#[derive(Rule)] +#[language] pub enum Expr { Number(u32), Add(Box, Box) } ``` -Now that we have the type defined, we must annotate the enum variants to describe how to identify them in the text being parsed. First, we can apply `rust_sitter::leaf` to use a regular expression to match digits corresponding to a number, and define a transformation that parses the resulting string into a `u32`. +Now that we have the type defined, we must annotate the enum variants to describe how to identify them in the text being parsed. First, we can apply `leaf` to use a regular expression to match digits corresponding to a number. +The value will try to extract the value using a default extraction for the type. For numeric types, this +defaults to `FromStr`. You can specify an alternate function using `#[with]`. ```rust Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] u32, ) ``` -For the `Add` variant, things are a bit more complicated. First, we add an extra field corresponding to the `+` that must sit between the two sub-expressions. This can be achieved with `text` parameter of `rust_sitter::leaf`, which instructs the parser to match a specific string. Because we are parsing to `()`, we do not need to provide a transformation. +For the `Add` variant, things are a bit more complicated. First, we add an extra field corresponding to the `+` that must sit between the two sub-expressions. This can be achieved with `text` or `leaf`, which instructs the parser to match a specific string. ```rust Add( Box, - #[rust_sitter::leaf(text = "+")] (), + #[text("+")] (), Box, ) ``` @@ -68,10 +68,10 @@ Add( If we try to compile this grammar, however, we will see ane error due to conflicting parse trees for expressions like `1 + 2 + 3`, which could be parsed as `(1 + 2) + 3` or `1 + (2 + 3)`. We want the former, so we can add a further annotation specifying that we want left-associativity for this rule. ```rust -#[rust_sitter::prec_left(1)] +#[prec_left(1)] Add( Box, - #[rust_sitter::leaf(text = "+")] (), + #[text("+")] (), Box, ) ``` @@ -79,30 +79,29 @@ Add( All together, our grammar looks like this: ```rust -#[rust_sitter::grammar("arithmetic")] -mod grammar { - #[rust_sitter::language] - pub enum Expr { - Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] - u32, - ), - #[rust_sitter::prec_left(1)] - Add( - Box, - #[rust_sitter::leaf(text = "+")] (), - Box, - ) - } +use rust_sitter::Rule; +#[derive(Rule)] +#[language] +pub enum Expr { + Number( + #[leaf(re(r"\d+"))] + u32, + ), + #[prec_left(1)] + Add( + Box, + #[text("+")] (), + Box, + ) } ``` We can then parse text using this grammar: ```rust -dbg!(grammar::parse("1+2+3")); +dbg!(grammar::Expr::parse("1+2+3").into_result()); /* -grammar::parse("1+2+3") = Ok(Add( +grammar::Expr::parse("1+2+3").into_result() = Ok(Add( Add( Number( 1, @@ -123,56 +122,97 @@ grammar::parse("1+2+3") = Ok(Add( ## Type Annotations Rust Sitter supports a number of annotations that can be applied to type and fields in your grammar. These annotations can be used to control how the parser behaves, and how the resulting AST is constructed. -### `#[rust_sitter::language]` +### `#[language]` This annotation marks the entrypoint for parsing, and determines which AST type will be returned from parsing. Only one type in the grammar can be marked as the entrypoint. ```rust -#[rust_sitter::language] +#[derive(Rule)] +#[language] struct Code { ... } ```` -### `#[rust_sitter::extra]` -This annotation marks a node as extra and can safely be skipped while parsing. This is useful for handling whitespace/newlines/comments. +### `#[extras(...)]` +This annotation can be used on the `#[language]` rule to specify a list of extras. These extras are specified +using the same DSL as `#[leaf(...)]` and `#[text(...)]`. These rules are inserted to the `extras` array in the +grammar. ```rust -#[rust_sitter::extra] -struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s")] - _whitespace: (), +#[derive(Rule)] +#[language] +#[extras( + re(r"\s") // allows whitespace in the grammar. +)] +struct Code { + ... } ``` ## Field Annotations -### `#[rust_sitter::leaf(...)]` -The `#[rust_sitter::leaf(...)]` annotation can be used to define a leaf node in the AST. This annotation takes a number of parameters that control how the parser behaves: -- the `pattern` parameter takes a regular expression that is used to match the text of the leaf node. This parameter is required. -- the `text` parameter takes a string that is used to match the text of the leaf node. This parameter is mutually exclusive with `pattern`. -- the `transform` parameter takes a function that is used to transform the matched text (an `&str`) into the desired type. This parameter is optional if the target type is `()`. +### `#[leaf(...)]` and `#[text(...)]` +The `#[leaf(...)]` annotation can be used to define a leaf node in the AST. +`#[text(...)]` is similar, but it does not create a named node in the grammar and cannot be +extracted. It must always be assigned to `()`. + +`leaf` and `text` take an input that looks like the [tree sitter +DSL](https://tree-sitter.github.io/tree-sitter/creating-parsers/2-the-grammar-dsl.html). The supported rules +currently are: +* `choice` +* `optional` +* `seq` +* `re` or `pattern` to specify a regular expression +* literal text + +Others can be added in the future as needed. `leaf` can either be applied to a field in a struct / enum variant (as seen above), or directly on a type with no fields: ```rust -#[rust_sitter::leaf(text = "9")] +#[derive(Rule)] +#[leaf("9")] struct BigDigit; +#[derive(Rule)] enum SmallDigit { - #[rust_sitter::leaf(text = "0")] + #[leaf("0")] Zero, - #[rust_sitter::leaf(text = "1")] + #[leaf("1")] One, } ``` -### `#[rust_sitter::prec(...)]` / `#[rust_sitter::prec_left(...)]` / `#[rust_sitter::prec_right(...)]` +### `#[prec(...)]` / `#[prec_left(...)]` / `#[prec_right(...)]` / `#[prec_dynamic(...)]` This annotation can be used to define a non/left/right-associative operator. This annotation takes a single parameter, which is the precedence level of the operator (higher binds more tightly). -### `#[rust_sitter::skip(...)]` +### `#[immediate]` +Usually, whitespace is optional before each token. This attribute means that the token will only match if there is no whitespace. + +### `#[skip(...)]` This annotation can be used to define a field that does not correspond to anything in the input string, such as some metadata. This annotation takes a single parameter, which is the value that should be used to populate that field at runtime. -### `#[rust_sitter::word]` -This annotation marks the field as a Tree Sitter [word](https://tree-sitter.github.io/tree-sitter/creating-parsers#keywords), which is useful when handling errors involving keywords. Only one field in the grammar can be marked as a word. +### `#[word]` +This annotation marks the field as a Tree Sitter [word](https://tree-sitter.github.io/tree-sitter/creating-parsers#keywords), which is useful when handling errors involving keywords. Like `#[extras]`, the `#[word]` is specified on the `#[language]` implementation: + +```rust +#[derive(Debug, Rule)] +#[language] +#[word(Ident)] +pub struct Language { + // ... +} + +#[derive(Rule)] +#[leaf(re(r"[a-zA-Z_]+"))] +pub struct Ident; +``` + +## Partial AST and Errors +rust-sitter, like tree-sitter, can produce a partial AST along with its errors. Calling `Language::parse` will +produce a `ParseResult` object which includes as much of the AST as it was able to extract, as well as a `Vec` +of all of the parsing errors encountered. This is useful for language servers and other contexts which can +make use of a partial AST. Currently this may not produce the _maximal_ AST, but this may be possible +in the future. ## Special Types Rust Sitter has a few special types that can be used to define more complex grammars. @@ -180,27 +220,24 @@ Rust Sitter has a few special types that can be used to define more complex gram ### `Vec` To parse repeating structures, you can use a `Vec` to parse a list of `T`s. Note that the `Vec` type **cannot** be wrapped in another `Vec` (create additional structs if this is necessary). There are two special attributes that can be applied to a `Vec` field to control the parsing behavior. -The `#[rust_sitter::delimited(...)]` attribute can be used to specify a separator between elements of the list, and takes a parameter of the same format as an unnamed field. For example, we can define a grammar that parses a comma-separated list of expressions: +The `#[sep_by(...)]` attribute can be used to specify a separator between elements of the +list. This is parsed in the same way as `text` and `leaf` and therefore supports all of the listed tree-sitter +grammar above. ```rust pub struct CommaSeparatedExprs { - #[rust_sitter::delimited( - #[rust_sitter::leaf(text = ",")] - () - )] + #[sep_by(",")] numbers: Vec, } ``` -The `#[rust_sitter::repeat(...)]` attribute can be used to specify additional configuration for the parser. Currently, there is only one available parameter: `non_empty`, which takes a boolean that specifies if the list must contain at least one element. For example, we can define a grammar that parses a non-empty comma-separated list of numbers: +The `#[repeat1]` can be used to specify that the list must contain at least, or you can use `#[sep_by1(...)] ```rust pub struct CommaSeparatedExprs { - #[rust_sitter::repeat(non_empty = true)] - #[rust_sitter::delimited( - #[rust_sitter::leaf(text = ",")] - () - )] + #[repeat1] + #[sep_by(",")] + // Or just use #[sep_by1(",")] numbers: Vec, } ``` @@ -210,11 +247,7 @@ To parse optional structures, you can use an `Option` to parse a single `T` o ```rust pub struct CommaSeparatedExprs { - #[rust_sitter::repeat(non_empty = true)] - #[rust_sitter::delimited( - #[rust_sitter::leaf(text = ",")] - () - )] + #[sep_by1(",")] numbers: Vec>, } ``` @@ -224,18 +257,10 @@ When using Rust Sitter to power diagnostic tools, it can be helpful to access sp ```rust pub struct CommaSeparatedExprs { - #[rust_sitter::repeat(non_empty = true)] - #[rust_sitter::delimited( - #[rust_sitter::leaf(text = ",")] - () - )] + #[sep_by1(",")] numbers: Vec>>, } ``` ### `Box` Boxes are automatically constructed around the inner type when parsing, but Rust Sitter doesn't do anything extra beyond that. - -## Debugging - -To view the generated grammar, you can set the `RUST_SITTER_EMIT_ARTIFACTS` environment variable to `true`. This will cause the generated grammar to be written to wherever cargo sets `OUT_DIR` (usually `target/debug/build/-/out`). diff --git a/common/Cargo.toml b/common/Cargo.toml index 41e825f..7190317 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -3,9 +3,9 @@ name = "rust-sitter-common" description = "Shared logic for the Rust Sitter macro and tool" readme = "../README.md" repository = "https://github.com/hydro-project/rust-sitter" -version = "0.4.5" -authors = ["Shadaj Laddad "] -edition = "2021" +version.workspace = true +authors.workspace = true +edition = "2024" license = "MIT" keywords = ["parsing", "codegen"] categories = ["development-tools"] @@ -14,5 +14,10 @@ categories = ["development-tools"] path = "src/lib.rs" [dependencies] +rust-sitter-types = { path = "../types" } syn = { version = "2", features = [ "full", "extra-traits" ] } +proc-macro2 = "1" quote = "1" + +serde_json = "1" +itertools = "0.14" diff --git a/common/src/expansion.rs b/common/src/expansion.rs new file mode 100644 index 0000000..ee29ae2 --- /dev/null +++ b/common/src/expansion.rs @@ -0,0 +1,704 @@ +use std::collections::HashSet; + +use super::*; +use itertools::Itertools; +use rust_sitter_types::grammar::Grammar; +use syn::{parse::Parse, punctuated::Punctuated, spanned::Spanned}; + +#[derive(Debug)] +pub struct RuleDerive { + pub ident: syn::Ident, + pub attrs: Vec, + pub params: RuleParams, + pub data: syn::Data, +} + +impl RuleDerive { + pub fn from_derive_input(d: DeriveInput) -> Result> { + if d.attrs.iter().any(|a| { + let Ok(list) = a.meta.require_list() else { + return false; + }; + let Some(derives) = list + .parse_args_with(Punctuated::::parse_terminated) + .ok() + else { + // Should be infallible to parse derive tokens + return false; + }; + derives + .iter() + .any(|p| p == &parse_quote!(rust_sitter::Rule) || p == &parse_quote!(Rule)) + }) { + Ok(Some(Self::from_derive_input_known(d)?)) + } else { + Ok(None) + } + } + + // Used by the proc macro directly. + pub fn from_derive_input_known(d: DeriveInput) -> Result { + let extras = RuleParams::new(&d.attrs)?; + Ok(Self { + ident: d.ident, + attrs: d.attrs, + params: extras, + data: d.data, + }) + } +} + +/// Generate a single grammar per module. +pub fn generate_grammar(root_file: Vec) -> Result> { + let mut state = ExpansionState::new(); + // for some reason, source_file must be the first key for things to work + state + .grammar + .rules + .insert("source_file".to_string(), RuleDef::BLANK); + + if root_file + .into_iter() + .map(|item| process_item(item, &mut state)) + .flat_map(|r| r.err()) + .reduce(|mut acc, e| { + acc.combine(e); + acc + }) + .map(Err::<(), _>) + .is_some() + { + state.err()?; + // Theoretically, we can use this for something. But, this generate_grammar function is + // only used in `build.rs`, and all of these errors which are useful will be caught + // during macro expansion, which is much more useful for development purposes. + return Ok(None); + } + if state.verify_seen().is_err() { + return Ok(None); + } + + // This error is useful for us and cannot be generated by proc macro expansion. + let (language_ident, language) = state.language_rule.ok_or_else(|| { + Error::new( + Span::call_site(), + "Must specify exactly one root with #[language]", + ) + })?; + let name = language + .name() + .unwrap_or_else(|| language_ident.to_string()); + state.grammar.name = name.clone(); + state.grammar.rules.insert( + "source_file".to_string(), + state + .grammar + .rules + .get(&language_ident.to_string()) + .unwrap() + .clone(), + ); + Ok(Some(state.grammar)) +} + +pub struct ExpansionState { + pub grammar: Grammar, + pub language_rule: Option<(Ident, LanguageExpr)>, + // Accumulated errors. + pub error: Option, +} + +impl Default for ExpansionState { + fn default() -> Self { + Self::new() + } +} + +impl ExpansionState { + pub fn new() -> Self { + Self { + grammar: Grammar { + name: String::new(), + word: None, + rules: Default::default(), + extras: Default::default(), + }, + language_rule: None, + error: None, + } + } + fn err(&mut self) -> Result<()> { + if let Some(err) = self.error.take() { + Err(err) + } else { + Ok(()) + } + } + + fn accumulate_error(&mut self, err: Error) -> Error { + if let Some(inner) = &mut self.error { + inner.combine(err.clone()); + } else { + self.error = Some(err.clone()); + } + err + } + + fn verify_seen(&self) -> Result<()> { + if let Some(e) = self + .grammar + .rules + .values() + .flat_map(|v| self.check_seen_value(v).err()) + .reduce(|mut acc, e| { + acc.combine(e); + acc + }) + { + Err(e) + } else { + Ok(()) + } + } + + fn check_seen_value(&self, value: &RuleDef) -> Result<()> { + // Each value is always a map. + match value { + RuleDef::SYMBOL { name } => { + if !self.grammar.rules.contains_key(name) { + return Err(Error::new( + Span::call_site(), + format!("Symbol found with no corresponding value: {name}"), + )); + } + } + RuleDef::CHOICE { members } | RuleDef::SEQ { members } => { + for member in members { + self.check_seen_value(member)?; + } + } + RuleDef::REPEAT { content } + | RuleDef::REPEAT1 { content } + | RuleDef::PREC_DYNAMIC { value: _, content } + | RuleDef::PREC_LEFT { value: _, content } + | RuleDef::PREC_RIGHT { value: _, content } + | RuleDef::PREC { value: _, content } + | RuleDef::TOKEN { content } + | RuleDef::IMMEDIATE_TOKEN { content } + | RuleDef::RESERVED { + context_name: _, + content, + } => self.check_seen_value(content)?, + _ => return Ok(()), + } + Ok(()) + } + + fn set_language(&mut self, ident: &Ident, lang: LanguageExpr) -> Result<()> { + if let Some((existing, _)) = &self.language_rule { + return Err(self.accumulate_error(Error::new( + lang.path.span(), + format!( + "Language rule already defined as {existing}, found duplicate with {ident}", + ), + ))); + } + self.language_rule = Some((ident.clone(), lang)); + Ok(()) + } + + fn set_word(&mut self, ident: String) -> Result<()> { + if let Some(existing) = &self.grammar.word { + return Err(self.accumulate_error(Error::new( + Span::call_site(), + format!("Word rule already defined as {existing}, found duplicate with {ident}"), + ))); + } + self.grammar.word = Some(ident); + Ok(()) + } +} + +fn process_item(item: Item, ctx: &mut ExpansionState) -> Result<()> { + match item { + Item::Struct(_) | Item::Enum(_) => { + // Try and convert it to a derive. + let stream = item.to_token_stream(); + let input = RuleDerive::from_derive_input(syn::parse2(stream)?)?; + if let Some(input) = input { + // Parse the structure now. + process_rule(input, ctx)?; + } + } + Item::Mod(m) => { + // Recursively process this now. + let (_, items) = m + .content + .ok_or_else(|| Error::new(Span::call_site(), "Module must be inlined"))?; + for item in items { + process_item(item, ctx)?; + } + } + _ => {} + } + + Ok(()) +} + +pub fn process_rule(input: RuleDerive, ctx: &mut ExpansionState) -> Result<()> { + if let Some(l) = &input.params.language { + ctx.set_language(&input.ident, l.clone())?; + } + if let Some(word) = &input.params.word { + let rule = word.evaluate()?; + match rule { + RuleDef::SYMBOL { name } => ctx.set_word(name)?, + + _ => { + return Err(Error::new( + word.span(), + "word must be a reference to another rule", + )); + } + } + } + if let Some(extras) = &input.params.extras { + if input.params.language.is_none() { + return Err(Error::new( + extras.span(), + "Cannot specify extras without #[language]", + )); + } + let (extras, errs): (Vec<_>, Vec<_>) = extras + .iter() + .map(|input| input.evaluate()) + .partition_result(); + let err = errs.into_iter().reduce(|mut acc, n| { + acc.combine(n); + acc + }); + if let Some(err) = err { + return Err(err); + } + ctx.grammar.extras = extras; + } + // if input.extras.word { + // ctx.set_word(&input.ident); + // } + + let ident = input.ident; + + match input.data { + Data::Struct(DataStruct { fields, .. }) => { + gen_struct_or_variant(ident.to_string(), &input.attrs, fields.clone(), false, ctx)?; + } + Data::Enum(DataEnum { variants, .. }) => { + variants + .iter() + .flat_map(|v| { + gen_struct_or_variant( + format!("{}_{}", ident, v.ident), + &v.attrs, + v.fields.clone(), + true, + ctx, + ) + .err() + }) + .reduce(|mut acc, e| { + acc.combine(e); + acc + }) + .map(Err::<(), _>) + .transpose()?; + + let mut members = vec![]; + variants.iter().for_each(|v| { + let variant_path = format!("{}_{}", ident, v.ident); + members.push(RuleDef::SYMBOL { name: variant_path }); + }); + + let rule = RuleDef::CHOICE { members }; + + let rule = input.params.apply(rule)?; + + ctx.grammar.rules.insert(ident.to_string(), rule); + } + Data::Union(_) => return Err(Error::new(ident.span(), "Union not supported")), + } + + Ok(()) +} + +fn gen_field( + ident_str: &str, + leaf_type: Option, + attrs: Vec, +) -> Result<(RuleDef, bool)> { + let precs = RuleParams::new(&attrs)?; + + if precs.language.is_some() { + return Err(Error::new( + leaf_type.span(), + "Cannot specify language on a field", + )); + } + let leaf_attr = attrs.iter().find(|attr| sitter_attr_matches(attr, "leaf")); + + let text_attr = attrs.iter().find(|attr| sitter_attr_matches(attr, "text")); + + if let Some(leaf_attr) = leaf_attr + && text_attr.is_some() + { + return Err(Error::new( + leaf_attr.span(), + "Cannot specify leaf and text at the same time", + )); + } + + if let Some(text) = text_attr { + let input: TsInput = text.parse_args()?; + return Ok((precs.apply(input.evaluate()?)?, false)); + } + + let leaf_input = leaf_attr.map(|a| a.parse_args::()).transpose()?; + + let leaf_type = match leaf_type { + Some(ty) => ty, + None => { + let Some(leaf_input) = leaf_input else { + // TODO: Narrow the span + return Err(Error::new( + Span::call_site(), + "Empty types must have a leaf or text attribute", + )); + }; + return Ok((precs.apply(leaf_input.evaluate()?)?, false)); + } + }; + + let mut skip_over = HashSet::new(); + skip_over.insert("Spanned"); + skip_over.insert("Box"); + + let (inner_type_vec, is_vec) = try_extract_inner_type(&leaf_type, "Vec", &skip_over); + let (inner_type_option, is_option) = try_extract_inner_type(&leaf_type, "Option", &skip_over); + + if !is_vec && !is_option { + if let Some(input) = leaf_input { + let result = input.evaluate()?; + Ok((precs.apply(result)?, is_option)) + } else { + let symbol_name = match filter_inner_type(&leaf_type, &skip_over) { + Type::Path(p) => p.path.require_ident()?.to_string(), + t => return Err(Error::new(t.span(), "Expected a path")), + }; + + Ok((precs.apply(RuleDef::SYMBOL { name: symbol_name })?, false)) + } + } else if is_vec { + let (field_json, field_optional) = gen_field( + ident_str, + Some(inner_type_vec), + leaf_attr.iter().cloned().cloned().collect(), + )?; + + let (delimited_param, repeat_non_empty) = attrs + .iter() + .find_map(|attr| { + if sitter_attr_matches(attr, "sep_by") { + Some((Some(attr), false)) + } else if sitter_attr_matches(attr, "sep_by1") { + Some((Some(attr), true)) + } else if sitter_attr_matches(attr, "repeat1") { + Some((None, true)) + } else { + None + } + }) + .unwrap_or((None, false)); + let delimited_param = delimited_param + .map(|a| a.parse_args::()) + .transpose()?; + + let delimiter_json = delimited_param + .as_ref() + .map(|p| precs.apply(p.evaluate()?)) + .transpose()?; + + let field_rule_non_optional = field_json; + let field_rule = if field_optional { + RuleDef::optional(field_rule_non_optional) + } else { + field_rule_non_optional + }; + + let vec_contents = if let Some(delimiter_json) = delimiter_json { + let content = Box::new(RuleDef::SEQ { + members: vec![delimiter_json, field_rule.clone()], + }); + let delimiter_rule = if field_optional { + RuleDef::REPEAT1 { content } + } else { + RuleDef::REPEAT { content } + }; + RuleDef::SEQ { + members: vec![field_rule, delimiter_rule], + } + } else { + RuleDef::REPEAT1 { + // This FIELD is used only for the macro generation phase to distinguish between + // the two different types of REPEAT1. + content: Box::new(RuleDef::FIELD { + name: ident_str.to_owned(), + content: field_rule.into(), + }), + } + }; + + let vec_contents = precs.apply(vec_contents)?; + + Ok((vec_contents, !repeat_non_empty)) + } else { + // is_option + let (field_json, field_optional) = gen_field(ident_str, Some(inner_type_option), attrs)?; + + if field_optional { + return Err(Error::new( + Span::call_site(), + "Option> is not supported", + )); + } + + Ok((precs.apply(field_json)?, true)) + } +} + +fn gen_struct_or_variant( + path: String, + attrs: &[Attribute], + fields: Fields, + is_variant: bool, + ctx: &mut ExpansionState, +) -> Result<()> { + fn gen_field_optional(field: &Field, ident_str: String) -> Result { + let (field_contents, is_option) = + gen_field(&ident_str, Some(field.ty.clone()), field.attrs.clone())?; + + let core = RuleDef::FIELD { + name: ident_str, + content: field_contents.into(), + }; + + let r = if is_option { + RuleDef::optional(core) + } else { + core + }; + Ok(r) + } + + let (children, errs): (Vec<_>, Vec<_>) = fields + .iter() + .enumerate() + .filter_map(|(i, field)| { + if field + .attrs + .iter() + .any(|attr| sitter_attr_matches(attr, "skip")) + { + None + } else { + let ident_str = field + .ident + .as_ref() + .map(|v| v.to_string()) + .unwrap_or(format!("{i}")); + + Some(gen_field_optional(field, ident_str)) + } + }) + .partition_result(); + let err = errs.into_iter().reduce(|mut acc, e| { + acc.combine(e); + acc + }); + if let Some(err) = err { + return Err(err); + } + + let base_rule = match fields { + Fields::Unit => { + let (field_contents, _is_option) = gen_field("unit", None, attrs.to_owned())?; + if is_variant { + RuleDef::FIELD { + name: "unit".to_owned(), + content: field_contents.into(), + } + } else { + field_contents + } + } + _ => RuleDef::SEQ { members: children }, + }; + + let precs = RuleParams::new(attrs)?; + + ctx.grammar.rules.insert(path, precs.apply(base_rule)?); + Ok(()) +} + +#[derive(Debug)] +pub struct RuleParams { + pub prec_param: Option, + pub prec_left_param: Option, + pub prec_right_param: Option, + pub prec_dynamic_param: Option, + pub language: Option, + pub extras: Option>, + pub word: Option, +} + +impl RuleParams { + fn new(attrs: &[Attribute]) -> Result { + let prec_attr = attrs.iter().find(|attr| sitter_attr_matches(attr, "prec")); + + let prec_param = prec_attr + .map(|a| a.parse_args_with(Expr::parse)) + .transpose()?; + + let prec_left_attr = attrs + .iter() + .find(|attr| sitter_attr_matches(attr, "prec_left")); + + let prec_left_param = prec_left_attr + .map(|a| a.parse_args_with(Expr::parse)) + .transpose()?; + + let prec_right_attr = attrs + .iter() + .find(|attr| sitter_attr_matches(attr, "prec_right")); + + let prec_right_param = prec_right_attr + .map(|a| a.parse_args_with(Expr::parse)) + .transpose()?; + + let prec_dynamic_attr = attrs + .iter() + .find(|attr| sitter_attr_matches(attr, "prec_dynamic")); + + let prec_dynamic_param = prec_dynamic_attr + .map(|a| a.parse_args_with(Expr::parse)) + .transpose()?; + + if let (Some(prec_left), Some(_prec_right)) = (prec_left_attr, prec_right_attr) { + return Err(Error::new( + prec_left.span(), + "only one of prec, prec_left, and prec_right can be specified", + )); + } + + let extras = attrs + .iter() + .find(|a| sitter_attr_matches(a, "extras")) + .map(|a| a.parse_args_with(Punctuated::::parse_terminated)) + .transpose()?; + + let language = attrs + .iter() + .find(|a| sitter_attr_matches(a, "language")) + .map(LanguageExpr::from_attr) + .transpose()?; + let word = attrs + .iter() + .find(|a| sitter_attr_matches(a, "word")) + .map(|a| a.parse_args_with(TsInput::parse)) + .transpose()?; + // TODO: Refactor this and make it a sub-field of the `LanguageExpr` type instead. + if language.is_none() + && let Some(w) = &word + { + return Err(Error::new( + w.span(), + "Cannot specify #[word] on non-language rule", + )); + } + + Ok(Self { + prec_param, + prec_left_param, + prec_right_param, + prec_dynamic_param, + extras, + word, + language, + }) + } + + fn apply(&self, rule: RuleDef) -> Result { + let Self { + prec_param, + prec_left_param, + prec_right_param, + prec_dynamic_param, + .. + } = self; + + let rule = if let Some(Expr::Lit(lit)) = prec_param { + if let Lit::Int(i) = &lit.lit { + let value = i.base10_parse::()?; + RuleDef::PREC { + value: value.into(), + content: Box::new(rule), + } + } else { + return Err(Error::new( + lit.span(), + "Expected integer literal for precedence", + )); + } + } else if let Some(Expr::Lit(lit)) = prec_left_param { + let value = if let Lit::Int(i) = &lit.lit { + i.base10_parse::()? + } else { + return Err(Error::new( + lit.span(), + "Expected integer literal for precedence", + )); + }; + RuleDef::PREC_LEFT { + value: value.into(), + content: Box::new(rule), + } + } else if let Some(Expr::Lit(lit)) = prec_right_param { + let value = if let Lit::Int(i) = &lit.lit { + i.base10_parse::()? + } else { + return Err(Error::new( + lit.span(), + "Expected integer literal for precedence", + )); + }; + RuleDef::PREC_RIGHT { + value: value.into(), + content: Box::new(rule), + } + } else if let Some(Expr::Lit(lit)) = prec_dynamic_param { + if let Lit::Int(i) = &lit.lit { + RuleDef::PREC_DYNAMIC { + value: i.base10_parse::()?, + content: Box::new(rule), + } + } else { + return Err(Error::new( + lit.span(), + "Expected integer literal for precedence", + )); + } + } else { + rule + }; + + Ok(rule) + } +} diff --git a/common/src/lib.rs b/common/src/lib.rs index 8aa06ca..b11cd47 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -1,11 +1,65 @@ +use proc_macro2::Span; +use quote::ToTokens; +use rust_sitter_types::grammar::RuleDef; use std::collections::HashSet; - use syn::{ parse::{Parse, ParseStream}, punctuated::Punctuated, + spanned::Spanned, *, }; +pub mod expansion; + +/// Language expression parsed from an attribute. +/// `#[language]` is the default, additional fields can be provided like so: +/// `#[language(name = "example")]` +#[derive(Debug, Clone)] +pub struct LanguageExpr { + // Useful to hold this for a useful span location on error generation. + pub path: Ident, + pub name: Option, +} + +impl LanguageExpr { + pub fn from_attr(a: &Attribute) -> Result { + let path = a.path().require_ident()?.clone(); + if path != "language" { + panic!("Expected language in LanguageExpr, this is a bug in rust-sitter"); + } + let mut s = Self { path, name: None }; + if matches!(&a.meta, Meta::List(_)) { + let args = + a.parse_args_with(Punctuated::::parse_terminated)?; + for arg in args { + if arg.path == "name" { + if s.name.is_some() { + return Err(Error::new(arg.path.span(), "Duplicate name field")); + } + let value = match arg.expr { + Expr::Lit(ExprLit { + attrs: _, + lit: Lit::Str(s), + }) => s, + _ => { + return Err(Error::new( + arg.expr.span(), + "name must be a literal string", + )); + } + }; + s.name = Some(value.value()); + } + } + } + Ok(s) + } + + pub fn name(&self) -> Option { + self.name.clone() + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct NameValueExpr { pub path: Ident, @@ -23,28 +77,154 @@ impl Parse for NameValueExpr { } } +/// tree-sitter input parsing. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct FieldThenParams { - pub field: Field, - pub comma: Option, - pub params: Punctuated, +pub struct TsInput { + pub expr: Expr, } -impl Parse for FieldThenParams { - fn parse(input: ParseStream) -> syn::Result { - let field = Field::parse_unnamed(input)?; - let comma: Option = input.parse()?; - let params = if comma.is_some() { - Punctuated::parse_terminated_with(input, NameValueExpr::parse)? - } else { - Punctuated::new() +impl Parse for TsInput { + fn parse(input: ParseStream) -> Result { + Ok(Self { + expr: input.parse()?, + }) + } +} + +impl ToTokens for TsInput { + fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { + self.expr.to_tokens(tokens); + } +} + +impl TsInput { + fn new(expr: &Expr) -> Self { + Self { expr: expr.clone() } + } + pub fn evaluate(&self) -> Result { + fn get_str(e: &Expr) -> Result { + let s = match e { + Expr::Lit(ExprLit { + attrs: _, + lit: Lit::Str(f), + }) => f, + _ => return Err(syn::Error::new(e.span(), "expected a string")), + }; + Ok(s.value()) + } + fn get_arg(p: &Punctuated, i: usize, expected: usize) -> Result<&Expr> { + assert!(i < expected); + if p.len() != expected { + // TODO: Fix the span + return Err(syn::Error::new(p.span(), "Too many arguments")); + } + Ok(p.get(i).unwrap()) + } + let def = match &self.expr { + Expr::Lit(ExprLit { + attrs: _, + lit: Lit::Str(s), + }) => RuleDef::STRING { value: s.value() }, + Expr::Call(ExprCall { + attrs: _, + func, + paren_token: _, + args, + }) => { + let name = match &**func { + Expr::Path(ExprPath { + attrs: _, + qself: _, + path, + }) => path.require_ident()?.to_string(), + k => return Err(syn::Error::new(k.span(), "Expected path")), + }; + match name.as_str() { + "optional" => { + let inner = Self::new(get_arg(args, 0, 1)?); + let members = vec![inner.evaluate()?, RuleDef::BLANK]; + RuleDef::CHOICE { members } + } + "seq" => { + let mut members = vec![]; + for arg in args { + let ts = Self::new(arg); + members.push(ts.evaluate()?); + } + RuleDef::SEQ { members } + } + "choice" => { + let mut members = vec![]; + for arg in args { + let ts = Self::new(arg); + members.push(ts.evaluate()?); + } + RuleDef::CHOICE { members } + } + "re" | "pattern" => RuleDef::PATTERN { + value: get_str(get_arg(args, 0, 1)?)?, + flags: None, + }, + "text" => RuleDef::STRING { + value: get_str(get_arg(args, 0, 1)?)?, + }, + "token" => { + let inner = Self::new(get_arg(args, 0, 1)?); + let content = Box::new(inner.evaluate()?); + RuleDef::TOKEN { content } + } + "immediate" => { + let inner = Self::new(get_arg(args, 0, 1)?); + let content = Box::new(inner.evaluate()?); + RuleDef::IMMEDIATE_TOKEN { content } + } + // nodes can be double wrapped in fields, although I'm not sure what happens + // when you ask the cursor for the field name? May not be possible to handle + // that in this case. + "field" => { + let name = get_str(get_arg(args, 0, 2)?)?; + let inner = Self::new(get_arg(args, 1, 2)?); + let content = Box::new(inner.evaluate()?); + RuleDef::FIELD { name, content } + } + k => { + return Err(syn::Error::new( + func.span(), + format!("Unexpected function call `{k}`"), + )); + } + } + } + Expr::Path(ExprPath { + attrs: _, + qself: _, + path, + }) => { + let ident = path.require_ident()?; + RuleDef::SYMBOL { + name: ident.to_string(), + } + } + k => { + return Err(syn::Error::new( + k.span(), + format!("Unexpected input type: {k:?}"), + )); + } }; + Ok(def) + } +} - Ok(FieldThenParams { - field, - comma, - params, - }) +pub fn sitter_attr_matches(attr: &Attribute, name: &str) -> bool { + let path = attr.path(); + if path.segments.len() == 1 { + path.segments[0].ident == name + } else if path.segments.len() == 2 { + // This is no longer possible, we can clean this up. + path.segments[0].ident == "rust_sitter" && path.segments[1].ident == name + } else { + false } } @@ -123,9 +303,9 @@ pub fn wrap_leaf_type(ty: &Type, skip_over: &HashSet<&str>) -> Type { panic!("Expected angle bracketed path"); } } else { - parse_quote!(rust_sitter::WithLeaf<#ty>) + parse_quote!(::rust_sitter::extract::WithLeaf<#ty, _>) } } else { - parse_quote!(rust_sitter::WithLeaf<#ty>) + parse_quote!(::rust_sitter::extract::WithLeaf<#ty, _>) } } diff --git a/example/Cargo.toml b/example/Cargo.toml index 898df92..f0e3e8d 100644 --- a/example/Cargo.toml +++ b/example/Cargo.toml @@ -1,17 +1,13 @@ [package] name = "rust-sitter-example" -version = "0.4.5" -authors = ["Shadaj Laddad "] +version.workspace = true +authors.workspace = true edition = "2021" publish = false -[features] -default = ["tree-sitter-c2rust"] -tree-sitter-c2rust = ["rust-sitter/tree-sitter-c2rust"] -tree-sitter-standard = ["rust-sitter/tree-sitter-standard"] - [dependencies] -rust-sitter = { path = "../runtime", default-features = false } +env_logger = "0.11" +rust-sitter = { path = "../runtime" } codemap = "0.1.3" codemap-diagnostic = "0.1.1" diff --git a/example/build.rs b/example/build.rs index 241bfb2..2a4adee 100644 --- a/example/build.rs +++ b/example/build.rs @@ -1,6 +1,11 @@ -use std::path::PathBuf; - fn main() { println!("cargo:rerun-if-changed=src"); - rust_sitter_tool::build_parsers(&PathBuf::from("src/main.rs")); + let examples = std::fs::read_dir("./src/").unwrap(); + for example in examples { + let example = example.unwrap(); + let path = example.path(); + if path.is_file() && path.file_stem().unwrap().to_str().unwrap() != "main" { + rust_sitter_tool::build_parser(&path); + } + } } diff --git a/example/src/arithmetic.rs b/example/src/arithmetic.rs index 1e8f3f7..36c8c88 100644 --- a/example/src/arithmetic.rs +++ b/example/src/arithmetic.rs @@ -1,44 +1,132 @@ -#[rust_sitter::grammar("arithmetic")] pub mod grammar { - #[rust_sitter::language] - #[derive(PartialEq, Eq, Debug)] + use rust_sitter::Rule; + #[derive(PartialEq, Eq, Debug, Rule)] + #[language] + #[extras( + // whitespace + re(r"\s") + )] pub enum Expression { - Number(#[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] i32), - #[rust_sitter::prec_left(1)] - Sub( - Box, - #[rust_sitter::leaf(text = "-")] (), - Box, - ), - #[rust_sitter::prec_left(2)] - Mul( - Box, - #[rust_sitter::leaf(text = "*")] (), - Box, - ), + Number(#[leaf(pattern(r"\d+"))] i32), + #[prec_left(1)] + Sub(Box, #[leaf("-")] (), Box), + #[prec_left(2)] + Mul(Box, #[leaf("*")] (), Box), + Let(LetExpression), + Complex(ComplexExpression), + Print(PrintExpression), + Vec(VecExpression), + Table(NewTable, #[leaf(";")] (), VecExpression), } - #[rust_sitter::extra] - struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s")] - _whitespace: (), + #[derive(Debug, Clone, PartialEq, Eq, Rule)] + #[leaf(seq("table", "(", ")"))] + pub struct NewTable; + + #[derive(PartialEq, Eq, Debug, Rule)] + pub struct LetExpression { + #[text("let")] + _let: (), + pub var: Ident, + #[text("=")] + _eq: (), + pub val: Box, + } + + #[derive(PartialEq, Eq, Debug, Rule)] + pub enum LogLevel { + #[leaf("info")] + Info, + #[leaf("debug")] + Debug, + #[leaf("trace")] + Trace, + Custom(CustomLevel), + } + + #[derive(PartialEq, Eq, Debug, Rule)] + pub enum Other { + #[leaf("info")] + Info, + #[leaf("debug")] + Debug, + #[leaf("trace")] + Trace, + } + + #[derive(PartialEq, Eq, Debug, Rule)] + pub struct CustomLevel { + #[text("custom")] + _custom: (), + #[text("::")] + _co: (), + pub value: Other, + } + + #[derive(PartialEq, Eq, Debug, Rule)] + pub struct ComplexExpression { + #[text("log")] + _log: (), + #[leaf("optional")] + optional: Option<()>, + pub level: LogLevel, + #[leaf(seq("(", optional(Expression), ")"))] + pub ex: Option<((), Option>, ())>, + // #[leaf(seq(LogLevel, "(", optional(Expression), ")"))] + // pub ident_ex: Option<(LogLevel, (), Option>, ())>, + #[leaf(";")] + _semi: Option<()>, } + + #[derive(PartialEq, Eq, Debug, Rule)] + pub struct VecExpression { + #[text("[")] + _vec: (), + #[sep_by(",")] + #[leaf(seq(Ident, ":", Expression))] + things: Vec<(String, (), Expression)>, + #[text("]")] + _vec_close: (), + other: Box, + } + + #[derive(PartialEq, Eq, Debug, Rule)] + pub struct PrintExpression { + #[text("print")] + _print: (), + #[text("(")] + _lparen: (), + #[sep_by(",")] + inputs: Vec, + #[text(")")] + _rparen: (), + } + + #[derive(PartialEq, Eq, Debug, Rule)] + pub struct Ident(#[leaf(re(r"[a-zA-Z_][a-zA-Z_0-9]*"))] String); } #[cfg(test)] mod tests { use super::*; use grammar::Expression; + use rust_sitter::Language; #[wasm_bindgen_test::wasm_bindgen_test] #[test] fn successful_parses() { - assert_eq!(grammar::parse("1").unwrap(), Expression::Number(1)); + assert_eq!( + grammar::Expression::parse("1").into_result().unwrap(), + Expression::Number(1) + ); - assert_eq!(grammar::parse(" 1").unwrap(), Expression::Number(1)); + assert_eq!( + grammar::Expression::parse(" 1").into_result().unwrap(), + Expression::Number(1) + ); assert_eq!( - grammar::parse("1 - 2").unwrap(), + grammar::Expression::parse("1 - 2").into_result().unwrap(), Expression::Sub( Box::new(Expression::Number(1)), (), @@ -47,7 +135,9 @@ mod tests { ); assert_eq!( - grammar::parse("1 - 2 - 3").unwrap(), + grammar::Expression::parse("1 - 2 - 3") + .into_result() + .unwrap(), Expression::Sub( Box::new(Expression::Sub( Box::new(Expression::Number(1)), @@ -60,7 +150,9 @@ mod tests { ); assert_eq!( - grammar::parse("1 - 2 * 3").unwrap(), + grammar::Expression::parse("1 - 2 * 3") + .into_result() + .unwrap(), Expression::Sub( Box::new(Expression::Number(1)), (), @@ -73,7 +165,9 @@ mod tests { ); assert_eq!( - grammar::parse("1 * 2 * 3").unwrap(), + grammar::Expression::parse("1 * 2 * 3") + .into_result() + .unwrap(), Expression::Mul( Box::new(Expression::Mul( Box::new(Expression::Number(1)), @@ -86,7 +180,9 @@ mod tests { ); assert_eq!( - grammar::parse("1 * 2 - 3").unwrap(), + grammar::Expression::parse("1 * 2 - 3") + .into_result() + .unwrap(), Expression::Sub( Box::new(Expression::Mul( Box::new(Expression::Number(1)), @@ -101,9 +197,9 @@ mod tests { #[test] fn failed_parses() { - insta::assert_debug_snapshot!(grammar::parse("1 + 2")); - insta::assert_debug_snapshot!(grammar::parse("1 - 2 -")); - insta::assert_debug_snapshot!(grammar::parse("a1")); - insta::assert_debug_snapshot!(grammar::parse("1a")); + insta::assert_debug_snapshot!(grammar::Expression::parse("1 + 2")); + insta::assert_debug_snapshot!(grammar::Expression::parse("1 - 2 -")); + insta::assert_debug_snapshot!(grammar::Expression::parse("a1")); + insta::assert_debug_snapshot!(grammar::Expression::parse("1a")); } } diff --git a/example/src/main.rs b/example/src/main.rs index d9615c0..1b29af0 100644 --- a/example/src/main.rs +++ b/example/src/main.rs @@ -1,64 +1,49 @@ -use std::io::Write; +use rust_sitter::Language; +use std::{fmt::Debug, io::Write}; use codemap::CodeMap; use codemap_diagnostic::{ColorConfig, Diagnostic, Emitter, Level, SpanLabel, SpanStyle}; -use rust_sitter::errors::{ParseError, ParseErrorReason}; +use rust_sitter::error::ParseError; mod arithmetic; mod optionals; mod repetitions; mod words; -fn convert_parse_error_to_diagnostics( - file_span: &codemap::Span, - error: &ParseError, - diagnostics: &mut Vec, -) { - match &error.reason { - ParseErrorReason::MissingToken(tok) => diagnostics.push(Diagnostic { - level: Level::Error, - message: format!("Missing token: \"{tok}\""), - code: Some("S000".to_string()), - spans: vec![SpanLabel { - span: file_span.subspan(error.start as u64, error.end as u64), - style: SpanStyle::Primary, - label: Some(format!("missing \"{tok}\"")), - }], - }), - - ParseErrorReason::UnexpectedToken(tok) => diagnostics.push(Diagnostic { - level: Level::Error, - message: format!("Unexpected token: \"{tok}\""), - code: Some("S000".to_string()), - spans: vec![SpanLabel { - span: file_span.subspan(error.start as u64, error.end as u64), - style: SpanStyle::Primary, - label: Some(format!("unexpected \"{tok}\"")), - }], - }), +fn convert_parse_error_to_diagnostics(file_span: &codemap::Span, error: &ParseError) -> Diagnostic { + let mut message = format!("syntax error. reason: {:?}", error.reason); + if !error.lookaheads.is_empty() { + message += &format!( + "\nPossible expected inputs: {}", + error.lookaheads.join(" | ") + ); + } - ParseErrorReason::FailedNode(errors) => { - if errors.is_empty() { - diagnostics.push(Diagnostic { - level: Level::Error, - message: "Failed to parse node".to_string(), - code: Some("S000".to_string()), - spans: vec![SpanLabel { - span: file_span.subspan(error.start as u64, error.end as u64), - style: SpanStyle::Primary, - label: Some("failed".to_string()), - }], - }) - } else { - for error in errors { - convert_parse_error_to_diagnostics(file_span, error, diagnostics); - } - } - } + Diagnostic { + level: Level::Error, + spans: vec![SpanLabel { + span: file_span.subspan( + error.error_position.bytes.start as u64, + error.error_position.bytes.end as u64, + ), + style: SpanStyle::Primary, + label: None, // TODO + }], + code: None, + message, } } fn main() { + env_logger::init(); + let args: Vec<_> = std::env::args().collect(); + let grammar = if args.len() == 1 { + "Expression" + } else if args.len() == 2 { + &args[1] + } else { + panic!("Unexpected inputs") + }; let stdin = std::io::stdin(); loop { @@ -72,19 +57,31 @@ fn main() { break; } - match arithmetic::grammar::parse(input) { - Ok(expr) => println!("{expr:?}"), - Err(errs) => { - let mut codemap = CodeMap::new(); - let file_span = codemap.add_file("".to_string(), input.to_string()); - let mut diagnostics = vec![]; - for error in errs { - convert_parse_error_to_diagnostics(&file_span.span, &error, &mut diagnostics); - } + match grammar { + "Expression" => process_input::(input), + "Repetition" => process_input::(input), + "Optional" => process_input::(input), + "Word" => process_input::(input), + _ => {} + } + } +} - let mut emitter = Emitter::stderr(ColorConfig::Always, Some(&codemap)); - emitter.emit(&diagnostics); - } - }; +fn process_input(input: &str) { + let result = T::parse(input); + match T::parse(input).result { + Some(expr) => println!("{expr:#?}"), + None => eprintln!("Could not parse"), + } + if !result.errors.is_empty() { + let mut codemap = CodeMap::new(); + let file_span = codemap.add_file("".to_string(), input.to_string()); + let mut diagnostics = vec![]; + for error in result.errors { + let d = convert_parse_error_to_diagnostics(&file_span.span, &error); + diagnostics.push(d); + } + let mut emitter = Emitter::stderr(ColorConfig::Always, Some(&codemap)); + emitter.emit(&diagnostics); } } diff --git a/example/src/optionals.rs b/example/src/optionals.rs index 365b749..7eb7f20 100644 --- a/example/src/optionals.rs +++ b/example/src/optionals.rs @@ -1,23 +1,22 @@ -#[rust_sitter::grammar("optionals")] #[allow(dead_code)] -mod grammar { +pub mod grammar { + use rust_sitter::Rule; use rust_sitter::Spanned; - #[rust_sitter::language] - #[derive(Debug)] + #[derive(Debug, Rule)] + #[language] pub struct Language { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] - v: Option, - #[rust_sitter::leaf(text = "_")] + v: Option, + #[leaf("_")] _s: (), t: Spanned>, - #[rust_sitter::leaf(text = ".")] + #[leaf(".")] _d: Option<()>, } - #[derive(Debug)] + #[derive(Debug, Rule)] pub struct Number { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] v: i32, } } @@ -25,16 +24,17 @@ mod grammar { #[cfg(test)] mod tests { use super::*; + use rust_sitter::Language; #[test] fn optional_grammar() { - insta::assert_debug_snapshot!(grammar::parse("_")); - insta::assert_debug_snapshot!(grammar::parse("_.")); - insta::assert_debug_snapshot!(grammar::parse("1_")); - insta::assert_debug_snapshot!(grammar::parse("1_.")); - insta::assert_debug_snapshot!(grammar::parse("1_2")); - insta::assert_debug_snapshot!(grammar::parse("1_2.")); - insta::assert_debug_snapshot!(grammar::parse("_2")); - insta::assert_debug_snapshot!(grammar::parse("_2.")); + insta::assert_debug_snapshot!(grammar::Language::parse("_")); + insta::assert_debug_snapshot!(grammar::Language::parse("_.")); + insta::assert_debug_snapshot!(grammar::Language::parse("1_")); + insta::assert_debug_snapshot!(grammar::Language::parse("1_.")); + insta::assert_debug_snapshot!(grammar::Language::parse("1_2")); + insta::assert_debug_snapshot!(grammar::Language::parse("1_2.")); + insta::assert_debug_snapshot!(grammar::Language::parse("_2")); + insta::assert_debug_snapshot!(grammar::Language::parse("_2.")); } } diff --git a/example/src/repetitions.rs b/example/src/repetitions.rs index df407fa..7b1e1ef 100644 --- a/example/src/repetitions.rs +++ b/example/src/repetitions.rs @@ -1,95 +1,113 @@ -#[rust_sitter::grammar("repetitions")] +#[allow(dead_code)] pub mod grammar { - use rust_sitter::Spanned; + use rust_sitter::{Rule, Spanned}; + #[derive(Debug, Rule)] + #[language] + #[extras(re(r"\s"))] + pub enum Repetitions { + List(NumberList), + ListRep1(NumberListRep1), + ListNoSep(NoSepNumberList), + } - #[rust_sitter::language] - #[derive(Debug)] - #[allow(dead_code)] + #[derive(Debug, Rule)] pub struct NumberList { - #[rust_sitter::repeat(non_empty = true)] - #[rust_sitter::delimited( - #[rust_sitter::leaf(text = ",")] - () - )] - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] - numbers: Spanned>>, + #[text("list")] + _list: (), + #[sep_by(",")] + #[leaf(Number)] + numbers: Spanned>>>, } - #[rust_sitter::extra] - struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s")] - _whitespace: (), + #[derive(Debug, Rule)] + pub struct NumberListRep1 { + #[text("list1")] + _list: (), + #[repeat1] + #[leaf(Number)] + numbers: Spanned>>, } -} -#[rust_sitter::grammar("repetitions_without_delim")] -pub mod grammar2 { - use rust_sitter::Spanned; - - #[rust_sitter::language] - #[derive(Debug)] - #[allow(dead_code)] - pub struct NumberList { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[derive(Debug, Rule)] + pub struct NoSepNumberList { + #[text("list2")] + _list: (), + #[leaf(Number)] numbers: Spanned>>, } - #[rust_sitter::extra] - struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s")] - _whitespace: (), - } + #[derive(Debug, Rule)] + #[leaf(pattern(r"\d+"))] + pub struct Number; } -#[rust_sitter::grammar("repetitions_optional_elem")] -pub mod grammar3 { - use rust_sitter::Spanned; - - #[rust_sitter::language] - #[derive(Debug)] - #[allow(dead_code)] - pub struct NumberList { - #[rust_sitter::delimited( - #[rust_sitter::leaf(text = ",")] - () - )] - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] - numbers: Spanned>>>, - #[rust_sitter::skip(123)] - metadata: u32, - } - - #[rust_sitter::extra] - struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s")] - _whitespace: (), - } -} +// TODO: Currently not allowed, needs to be fixed. +// pub mod grammar2 { +// use rust_sitter::{Rule, Spanned}; +// +// #[derive(Debug, Rule)] +// #[language] +// #[allow(dead_code)] +// pub struct NumberList { +// #[leaf(pattern(r"\d+"))] +// numbers: Spanned>>, +// } +// +// #[derive(Rule)] +// #[extra] +// struct Whitespace { +// #[leaf(pattern(r"\s"))] +// _whitespace: (), +// } +// } +// +// pub mod grammar3 { +// use rust_sitter::{Rule, Spanned}; +// +// #[derive(Debug, Rule)] +// #[language] +// #[allow(dead_code)] +// pub struct NumberList { +// #[sep_by(",")] +// #[leaf(pattern(r"\d+"))] +// numbers: Spanned>>>, +// #[skip(123)] +// metadata: u32, +// } +// +// #[derive(Rule)] +// #[extra] +// struct Whitespace { +// #[leaf(pattern(r"\s"))] +// _whitespace: (), +// } +// } #[cfg(test)] mod tests { use super::*; + use rust_sitter::Language; #[test] fn repetitions_grammar() { - insta::assert_debug_snapshot!(grammar::parse("")); - insta::assert_debug_snapshot!(grammar::parse("1")); - insta::assert_debug_snapshot!(grammar::parse("1, 2")); + insta::assert_debug_snapshot!(grammar::Repetitions::parse("list")); + insta::assert_debug_snapshot!(grammar::Repetitions::parse("list 1")); + insta::assert_debug_snapshot!(grammar::Repetitions::parse("list 1, 2")); } - #[test] - fn repetitions_grammar2() { - insta::assert_debug_snapshot!(grammar2::parse("")); - insta::assert_debug_snapshot!(grammar2::parse("1")); - insta::assert_debug_snapshot!(grammar2::parse("1 2")); - } + // #[test] + // fn repetitions_grammar2() { + // insta::assert_debug_snapshot!(grammar2::parse("")); + // insta::assert_debug_snapshot!(grammar2::parse("1")); + // insta::assert_debug_snapshot!(grammar2::parse("1 2")); + // } - #[test] - fn repetitions_grammar3() { - insta::assert_debug_snapshot!(grammar3::parse("")); - insta::assert_debug_snapshot!(grammar3::parse("1,")); - insta::assert_debug_snapshot!(grammar3::parse("1, 2")); - insta::assert_debug_snapshot!(grammar3::parse("1,, 2")); - insta::assert_debug_snapshot!(grammar3::parse("1,, 2,")); - } + // #[test] + // fn repetitions_grammar3() { + // insta::assert_debug_snapshot!(grammar3::parse("")); + // insta::assert_debug_snapshot!(grammar3::parse("1,")); + // insta::assert_debug_snapshot!(grammar3::parse("1, 2")); + // insta::assert_debug_snapshot!(grammar3::parse("1,, 2")); + // insta::assert_debug_snapshot!(grammar3::parse("1,, 2,")); + // } } diff --git a/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses-2.snap b/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses-2.snap index 86710ae..7040e00 100644 --- a/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses-2.snap +++ b/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses-2.snap @@ -1,15 +1,43 @@ --- source: example/src/arithmetic.rs -expression: "grammar::parse(\"1 - 2 -\")" +expression: "grammar::Expression::parse(\"1 - 2 -\")" --- -Err( - [ +ParseResult { + result: None, + errors: [ ParseError { - reason: MissingToken( - "Expression_Number_0", - ), - start: 7, - end: 7, + node_position: Position { + bytes: 7..7, + start: Point { + line: 1, + column: 8, + }, + end: Point { + line: 1, + column: 8, + }, + }, + error_position: Position { + bytes: 7..7, + start: Point { + line: 1, + column: 8, + }, + end: Point { + line: 1, + column: 8, + }, + }, + lookaheads: [], + reason: Extract { + struct_name: "Expression :: Number", + field_name: "0", + reason: TypeConversion( + ParseIntError { + kind: Empty, + }, + ), + }, }, ], -) +} diff --git a/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses-3.snap b/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses-3.snap index b0598f3..8a1a354 100644 --- a/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses-3.snap +++ b/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses-3.snap @@ -1,23 +1,61 @@ --- source: example/src/arithmetic.rs -expression: "grammar::parse(\"a1\")" +expression: "grammar::Expression::parse(\"a1\")" --- -Err( - [ +ParseResult { + result: Some( + Number( + 1, + ), + ), + errors: [ ParseError { - reason: FailedNode( - [ - ParseError { - reason: UnexpectedToken( - "a", - ), - start: 0, - end: 1, - }, - ], - ), - start: 0, - end: 1, + node_position: Position { + bytes: 0..1, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 2, + }, + }, + error_position: Position { + bytes: 0..1, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 2, + }, + }, + lookaheads: [ + "Expression_Number_token1", + "table", + "let", + "log", + "[", + "print", + "source_file", + "Expression_Number", + "Expression_Sub", + "Expression_Mul", + "Expression_Let", + "Expression_Complex", + "Expression_Print", + "Expression_Vec", + "Expression_Table", + "Expression", + "NewTable", + "LetExpression", + "ComplexExpression", + "VecExpression", + "PrintExpression", + ], + reason: Error, }, ], -) +} diff --git a/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses-4.snap b/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses-4.snap index a7a3cb4..bdf10e6 100644 --- a/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses-4.snap +++ b/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses-4.snap @@ -1,23 +1,68 @@ --- source: example/src/arithmetic.rs -expression: "grammar::parse(\"1a\")" +expression: "grammar::Expression::parse(\"1a\")" --- -Err( - [ +ParseResult { + result: None, + errors: [ ParseError { - reason: FailedNode( - [ - ParseError { - reason: UnexpectedToken( - "a", - ), - start: 1, - end: 2, - }, - ], - ), - start: 1, - end: 2, + node_position: Position { + bytes: 0..2, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 3, + }, + }, + error_position: Position { + bytes: 1..2, + start: Point { + line: 1, + column: 2, + }, + end: Point { + line: 1, + column: 3, + }, + }, + lookaheads: [ + "-", + "*", + ], + reason: Error, + }, + ParseError { + node_position: Position { + bytes: 0..0, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 1, + }, + }, + error_position: Position { + bytes: 0..0, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 1, + }, + }, + lookaheads: [], + reason: Extract { + struct_name: "Expression", + field_name: "root", + reason: MissingEnum, + }, }, ], -) +} diff --git a/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses.snap b/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses.snap index 110e5e2..edeaae8 100644 --- a/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses.snap +++ b/example/src/snapshots/rust_sitter_example__arithmetic__tests__failed_parses.snap @@ -1,23 +1,42 @@ --- source: example/src/arithmetic.rs -expression: "grammar::parse(\"1 + 2\")" +expression: "grammar::Expression::parse(\"1 + 2\")" --- -Err( - [ +ParseResult { + result: Some( + Number( + 2, + ), + ), + errors: [ ParseError { - reason: FailedNode( - [ - ParseError { - reason: UnexpectedToken( - "+", - ), - start: 2, - end: 3, - }, - ], - ), - start: 0, - end: 3, + node_position: Position { + bytes: 0..3, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 4, + }, + }, + error_position: Position { + bytes: 2..3, + start: Point { + line: 1, + column: 3, + }, + end: Point { + line: 1, + column: 4, + }, + }, + lookaheads: [ + "-", + "*", + ], + reason: Error, }, ], -) +} diff --git a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-2.snap b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-2.snap index 3524791..a45beef 100644 --- a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-2.snap +++ b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-2.snap @@ -1,20 +1,30 @@ --- source: example/src/optionals.rs -expression: "grammar::parse(\"_.\")" +expression: "grammar::Language::parse(\"_.\")" --- -Ok( - Language { - v: None, - _s: (), - t: Spanned { - value: None, - span: ( - 1, - 1, +ParseResult { + result: Some( + Language { + v: None, + _s: (), + t: Spanned { + value: None, + position: Position { + bytes: 1..2, + start: Point { + line: 1, + column: 2, + }, + end: Point { + line: 1, + column: 3, + }, + }, + }, + _d: Some( + (), ), }, - _d: Some( - (), - ), - }, -) + ), + errors: [], +} diff --git a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-3.snap b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-3.snap index 944ec51..7d099ce 100644 --- a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-3.snap +++ b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-3.snap @@ -1,20 +1,32 @@ --- source: example/src/optionals.rs -expression: "grammar::parse(\"1_\")" +expression: "grammar::Language::parse(\"1_\")" --- -Ok( - Language { - v: Some( - 1, - ), - _s: (), - t: Spanned { - value: None, - span: ( - 2, - 2, +ParseResult { + result: Some( + Language { + v: Some( + Number { + v: 1, + }, ), + _s: (), + t: Spanned { + value: None, + position: Position { + bytes: 1..2, + start: Point { + line: 1, + column: 2, + }, + end: Point { + line: 1, + column: 3, + }, + }, + }, + _d: None, }, - _d: None, - }, -) + ), + errors: [], +} diff --git a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-4.snap b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-4.snap index e1ce440..53290f5 100644 --- a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-4.snap +++ b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-4.snap @@ -1,22 +1,34 @@ --- source: example/src/optionals.rs -expression: "grammar::parse(\"1_.\")" +expression: "grammar::Language::parse(\"1_.\")" --- -Ok( - Language { - v: Some( - 1, - ), - _s: (), - t: Spanned { - value: None, - span: ( - 2, - 2, +ParseResult { + result: Some( + Language { + v: Some( + Number { + v: 1, + }, + ), + _s: (), + t: Spanned { + value: None, + position: Position { + bytes: 2..3, + start: Point { + line: 1, + column: 3, + }, + end: Point { + line: 1, + column: 4, + }, + }, + }, + _d: Some( + (), ), }, - _d: Some( - (), - ), - }, -) + ), + errors: [], +} diff --git a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-5.snap b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-5.snap index 8615253..2f43687 100644 --- a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-5.snap +++ b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-5.snap @@ -1,24 +1,36 @@ --- source: example/src/optionals.rs -expression: "grammar::parse(\"1_2\")" +expression: "grammar::Language::parse(\"1_2\")" --- -Ok( - Language { - v: Some( - 1, - ), - _s: (), - t: Spanned { - value: Some( +ParseResult { + result: Some( + Language { + v: Some( Number { - v: 2, + v: 1, }, ), - span: ( - 2, - 3, - ), + _s: (), + t: Spanned { + value: Some( + Number { + v: 2, + }, + ), + position: Position { + bytes: 2..3, + start: Point { + line: 1, + column: 3, + }, + end: Point { + line: 1, + column: 4, + }, + }, + }, + _d: None, }, - _d: None, - }, -) + ), + errors: [], +} diff --git a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-6.snap b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-6.snap index fc88456..55173f5 100644 --- a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-6.snap +++ b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-6.snap @@ -1,26 +1,38 @@ --- source: example/src/optionals.rs -expression: "grammar::parse(\"1_2.\")" +expression: "grammar::Language::parse(\"1_2.\")" --- -Ok( - Language { - v: Some( - 1, - ), - _s: (), - t: Spanned { - value: Some( +ParseResult { + result: Some( + Language { + v: Some( Number { - v: 2, + v: 1, }, ), - span: ( - 2, - 3, + _s: (), + t: Spanned { + value: Some( + Number { + v: 2, + }, + ), + position: Position { + bytes: 2..3, + start: Point { + line: 1, + column: 3, + }, + end: Point { + line: 1, + column: 4, + }, + }, + }, + _d: Some( + (), ), }, - _d: Some( - (), - ), - }, -) + ), + errors: [], +} diff --git a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-7.snap b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-7.snap index ff209e0..5b1b4c3 100644 --- a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-7.snap +++ b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-7.snap @@ -1,22 +1,32 @@ --- source: example/src/optionals.rs -expression: "grammar::parse(\"_2\")" +expression: "grammar::Language::parse(\"_2\")" --- -Ok( - Language { - v: None, - _s: (), - t: Spanned { - value: Some( - Number { - v: 2, +ParseResult { + result: Some( + Language { + v: None, + _s: (), + t: Spanned { + value: Some( + Number { + v: 2, + }, + ), + position: Position { + bytes: 1..2, + start: Point { + line: 1, + column: 2, + }, + end: Point { + line: 1, + column: 3, + }, }, - ), - span: ( - 1, - 2, - ), + }, + _d: None, }, - _d: None, - }, -) + ), + errors: [], +} diff --git a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-8.snap b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-8.snap index 70321bb..9e8f944 100644 --- a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-8.snap +++ b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar-8.snap @@ -1,24 +1,34 @@ --- source: example/src/optionals.rs -expression: "grammar::parse(\"_2.\")" +expression: "grammar::Language::parse(\"_2.\")" --- -Ok( - Language { - v: None, - _s: (), - t: Spanned { - value: Some( - Number { - v: 2, +ParseResult { + result: Some( + Language { + v: None, + _s: (), + t: Spanned { + value: Some( + Number { + v: 2, + }, + ), + position: Position { + bytes: 1..2, + start: Point { + line: 1, + column: 2, + }, + end: Point { + line: 1, + column: 3, + }, }, - ), - span: ( - 1, - 2, + }, + _d: Some( + (), ), }, - _d: Some( - (), - ), - }, -) + ), + errors: [], +} diff --git a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar.snap b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar.snap index da6a7b4..5783701 100644 --- a/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar.snap +++ b/example/src/snapshots/rust_sitter_example__optionals__tests__optional_grammar.snap @@ -1,18 +1,28 @@ --- source: example/src/optionals.rs -expression: "grammar::parse(\"_\")" +expression: "grammar::Language::parse(\"_\")" --- -Ok( - Language { - v: None, - _s: (), - t: Spanned { - value: None, - span: ( - 1, - 1, - ), +ParseResult { + result: Some( + Language { + v: None, + _s: (), + t: Spanned { + value: None, + position: Position { + bytes: 0..1, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 2, + }, + }, + }, + _d: None, }, - _d: None, - }, -) + ), + errors: [], +} diff --git a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar-2.snap b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar-2.snap index a2c1782..7e7bce2 100644 --- a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar-2.snap +++ b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar-2.snap @@ -1,23 +1,55 @@ --- source: example/src/repetitions.rs -expression: "grammar::parse(\"1\")" +expression: "grammar::Repetitions::parse(\"list 1\")" --- -Ok( - NumberList { - numbers: Spanned { - value: [ - Spanned { - value: 1, - span: ( - 0, - 1, - ), +ParseResult { + result: Some( + List( + NumberList { + _list: (), + numbers: Spanned { + value: [], + position: Position { + bytes: 0..0, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 1, + }, + }, }, - ], - span: ( - 0, - 1, - ), + }, + ), + ), + errors: [ + ParseError { + node_position: Position { + bytes: 5..6, + start: Point { + line: 1, + column: 6, + }, + end: Point { + line: 1, + column: 7, + }, + }, + error_position: Position { + bytes: 5..6, + start: Point { + line: 1, + column: 6, + }, + end: Point { + line: 1, + column: 7, + }, + }, + lookaheads: [], + reason: Error, }, - }, -) + ], +} diff --git a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar-3.snap b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar-3.snap index 911fce0..f7e4d2e 100644 --- a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar-3.snap +++ b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar-3.snap @@ -1,30 +1,61 @@ --- source: example/src/repetitions.rs -expression: "grammar::parse(\"1, 2\")" +expression: "grammar::Repetitions::parse(\"list 1, 2\")" --- -Ok( - NumberList { - numbers: Spanned { - value: [ - Spanned { - value: 1, - span: ( - 0, - 1, - ), +ParseResult { + result: Some( + List( + NumberList { + _list: (), + numbers: Spanned { + value: [ + Spanned { + value: Some( + 1, + ), + position: Position { + bytes: 5..6, + start: Point { + line: 1, + column: 6, + }, + end: Point { + line: 1, + column: 7, + }, + }, + }, + Spanned { + value: Some( + 2, + ), + position: Position { + bytes: 8..9, + start: Point { + line: 1, + column: 9, + }, + end: Point { + line: 1, + column: 10, + }, + }, + }, + ], + position: Position { + bytes: 5..9, + start: Point { + line: 1, + column: 6, + }, + end: Point { + line: 1, + column: 10, + }, + }, }, - Spanned { - value: 2, - span: ( - 3, - 4, - ), - }, - ], - span: ( - 0, - 4, - ), - }, - }, -) + }, + ), + ), + errors: [], +} diff --git a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar.snap b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar.snap index 33db0bc..d7b1213 100644 --- a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar.snap +++ b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar.snap @@ -1,15 +1,28 @@ --- source: example/src/repetitions.rs -expression: "grammar::parse(\"\")" +expression: "grammar::Repetitions::parse(\"list\")" --- -Err( - [ - ParseError { - reason: FailedNode( - [], - ), - start: 0, - end: 0, - }, - ], -) +ParseResult { + result: Some( + List( + NumberList { + _list: (), + numbers: Spanned { + value: [], + position: Position { + bytes: 0..0, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 1, + }, + }, + }, + }, + ), + ), + errors: [], +} diff --git a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar2-2.snap b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar2-2.snap index c152824..27f29c4 100644 --- a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar2-2.snap +++ b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar2-2.snap @@ -8,16 +8,36 @@ Ok( value: [ Spanned { value: 1, - span: ( + byte_span: ( 0, 1, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 2, + }, + ), }, ], - span: ( + byte_span: ( 0, 1, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 2, + }, + ), }, }, ) diff --git a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar2-3.snap b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar2-3.snap index 1541591..a0be55d 100644 --- a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar2-3.snap +++ b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar2-3.snap @@ -8,23 +8,53 @@ Ok( value: [ Spanned { value: 1, - span: ( + byte_span: ( 0, 1, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 2, + }, + ), }, Spanned { value: 2, - span: ( + byte_span: ( 2, 3, ), + line_span: ( + Point { + line: 1, + column: 3, + }, + Point { + line: 1, + column: 4, + }, + ), }, ], - span: ( + byte_span: ( 0, 3, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 4, + }, + ), }, }, ) diff --git a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar2.snap b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar2.snap index 9b45326..54fa677 100644 --- a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar2.snap +++ b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar2.snap @@ -6,10 +6,20 @@ Ok( NumberList { numbers: Spanned { value: [], - span: ( + byte_span: ( 0, 0, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 1, + }, + ), }, }, ) diff --git a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-2.snap b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-2.snap index 78611b1..58d5a1c 100644 --- a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-2.snap +++ b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-2.snap @@ -10,16 +10,36 @@ Ok( value: Some( 1, ), - span: ( + byte_span: ( 0, 1, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 2, + }, + ), }, ], - span: ( + byte_span: ( 0, 2, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 3, + }, + ), }, metadata: 123, }, diff --git a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-3.snap b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-3.snap index ad53176..4834d86 100644 --- a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-3.snap +++ b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-3.snap @@ -10,25 +10,55 @@ Ok( value: Some( 1, ), - span: ( + byte_span: ( 0, 1, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 2, + }, + ), }, Spanned { value: Some( 2, ), - span: ( + byte_span: ( 3, 4, ), + line_span: ( + Point { + line: 1, + column: 4, + }, + Point { + line: 1, + column: 5, + }, + ), }, ], - span: ( + byte_span: ( 0, 4, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 5, + }, + ), }, metadata: 123, }, diff --git a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-4.snap b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-4.snap index e172dff..3afd20f 100644 --- a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-4.snap +++ b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-4.snap @@ -10,25 +10,55 @@ Ok( value: Some( 1, ), - span: ( + byte_span: ( 0, 1, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 2, + }, + ), }, Spanned { value: Some( 2, ), - span: ( + byte_span: ( 4, 5, ), + line_span: ( + Point { + line: 1, + column: 5, + }, + Point { + line: 1, + column: 6, + }, + ), }, ], - span: ( + byte_span: ( 0, 5, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 6, + }, + ), }, metadata: 123, }, diff --git a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-5.snap b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-5.snap index f9c38ef..a9d5d67 100644 --- a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-5.snap +++ b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3-5.snap @@ -10,25 +10,55 @@ Ok( value: Some( 1, ), - span: ( + byte_span: ( 0, 1, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 2, + }, + ), }, Spanned { value: Some( 2, ), - span: ( + byte_span: ( 4, 5, ), + line_span: ( + Point { + line: 1, + column: 5, + }, + Point { + line: 1, + column: 6, + }, + ), }, ], - span: ( + byte_span: ( 0, 6, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 7, + }, + ), }, metadata: 123, }, diff --git a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3.snap b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3.snap index 6ab2e90..19d7204 100644 --- a/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3.snap +++ b/example/src/snapshots/rust_sitter_example__repetitions__tests__repetitions_grammar3.snap @@ -6,10 +6,20 @@ Ok( NumberList { numbers: Spanned { value: [], - span: ( + byte_span: ( 0, 0, ), + line_span: ( + Point { + line: 1, + column: 1, + }, + Point { + line: 1, + column: 1, + }, + ), }, metadata: 123, }, diff --git a/example/src/snapshots/rust_sitter_example__words__tests__words_grammar-2.snap b/example/src/snapshots/rust_sitter_example__words__tests__words_grammar-2.snap index f396fdb..dc53222 100644 --- a/example/src/snapshots/rust_sitter_example__words__tests__words_grammar-2.snap +++ b/example/src/snapshots/rust_sitter_example__words__tests__words_grammar-2.snap @@ -1,15 +1,70 @@ --- source: example/src/words.rs -expression: "grammar::parse(\"hello\")" +expression: "grammar::Words::parse(\"hello\")" --- -Err( - [ +ParseResult { + result: None, + errors: [ ParseError { - reason: FailedNode( - [], - ), - start: 0, - end: 5, + node_position: Position { + bytes: 0..5, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 6, + }, + }, + error_position: Position { + bytes: 0..5, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 6, + }, + }, + lookaheads: [ + "if", + "source_file", + ], + reason: Error, + }, + ParseError { + node_position: Position { + bytes: 0..5, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 6, + }, + }, + error_position: Position { + bytes: 0..5, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 6, + }, + }, + lookaheads: [], + reason: Extract { + struct_name: "Words", + field_name: "keyword", + reason: FieldExtraction { + message: "fields didn't match, cursor had: None, expected: keyword", + }, + }, }, ], -) +} diff --git a/example/src/snapshots/rust_sitter_example__words__tests__words_grammar-3.snap b/example/src/snapshots/rust_sitter_example__words__tests__words_grammar-3.snap index d9a0e68..62d68a3 100644 --- a/example/src/snapshots/rust_sitter_example__words__tests__words_grammar-3.snap +++ b/example/src/snapshots/rust_sitter_example__words__tests__words_grammar-3.snap @@ -1,15 +1,70 @@ --- source: example/src/words.rs -expression: "grammar::parse(\"ifhello\")" +expression: "grammar::Words::parse(\"ifhello\")" --- -Err( - [ +ParseResult { + result: None, + errors: [ ParseError { - reason: FailedNode( - [], - ), - start: 0, - end: 7, + node_position: Position { + bytes: 0..7, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 8, + }, + }, + error_position: Position { + bytes: 0..7, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 8, + }, + }, + lookaheads: [ + "if", + "source_file", + ], + reason: Error, + }, + ParseError { + node_position: Position { + bytes: 0..7, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 8, + }, + }, + error_position: Position { + bytes: 0..7, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 8, + }, + }, + lookaheads: [], + reason: Extract { + struct_name: "Words", + field_name: "keyword", + reason: FieldExtraction { + message: "fields didn't match, cursor had: None, expected: keyword", + }, + }, }, ], -) +} diff --git a/example/src/snapshots/rust_sitter_example__words__tests__words_grammar-4.snap b/example/src/snapshots/rust_sitter_example__words__tests__words_grammar-4.snap index 1e50599..bbe77b4 100644 --- a/example/src/snapshots/rust_sitter_example__words__tests__words_grammar-4.snap +++ b/example/src/snapshots/rust_sitter_example__words__tests__words_grammar-4.snap @@ -1,10 +1,13 @@ --- source: example/src/words.rs -expression: "grammar::parse(\"if hello\")" +expression: "grammar::Words::parse(\"if hello\")" --- -Ok( - Words { - _keyword: (), - _word: "hello", - }, -) +ParseResult { + result: Some( + Words { + keyword: (), + word: "hello", + }, + ), + errors: [], +} diff --git a/example/src/snapshots/rust_sitter_example__words__tests__words_grammar.snap b/example/src/snapshots/rust_sitter_example__words__tests__words_grammar.snap index 104301e..3d00063 100644 --- a/example/src/snapshots/rust_sitter_example__words__tests__words_grammar.snap +++ b/example/src/snapshots/rust_sitter_example__words__tests__words_grammar.snap @@ -1,15 +1,67 @@ --- source: example/src/words.rs -expression: "grammar::parse(\"if\")" +expression: "grammar::Words::parse(\"if\")" --- -Err( - [ +ParseResult { + result: None, + errors: [ ParseError { - reason: FailedNode( - [], - ), - start: 0, - end: 2, + node_position: Position { + bytes: 0..2, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 3, + }, + }, + error_position: Position { + bytes: 0..2, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 3, + }, + }, + lookaheads: [], + reason: Error, + }, + ParseError { + node_position: Position { + bytes: 0..2, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 3, + }, + }, + error_position: Position { + bytes: 0..2, + start: Point { + line: 1, + column: 1, + }, + end: Point { + line: 1, + column: 3, + }, + }, + lookaheads: [], + reason: Extract { + struct_name: "Words", + field_name: "keyword", + reason: FieldExtraction { + message: "fields didn't match, cursor had: None, expected: keyword", + }, + }, }, ], -) +} diff --git a/example/src/words.rs b/example/src/words.rs index 86ffe4a..68a7fcd 100644 --- a/example/src/words.rs +++ b/example/src/words.rs @@ -1,31 +1,33 @@ -#[rust_sitter::grammar("words")] pub mod grammar { - #[rust_sitter::language] - #[derive(Debug)] + use rust_sitter::Rule; + + #[derive(Debug, Rule)] + #[language] + #[extras(re(r"\s"))] + #[word(Ident)] + #[allow(dead_code)] pub struct Words { - #[rust_sitter::leaf(text = r"if")] - _keyword: (), - #[rust_sitter::word] - #[rust_sitter::leaf(pattern = r"[a-z_]+", transform = |v| v.to_string())] - _word: String, + #[leaf("if")] + keyword: (), + #[leaf(Ident)] + word: String, } - #[rust_sitter::extra] - struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s")] - _whitespace: (), - } + #[derive(Debug, Rule)] + #[leaf(pattern(r"[a-z_]+"))] + pub struct Ident; } #[cfg(test)] mod tests { use super::*; + use rust_sitter::Language; #[test] fn words_grammar() { - insta::assert_debug_snapshot!(grammar::parse("if")); - insta::assert_debug_snapshot!(grammar::parse("hello")); - insta::assert_debug_snapshot!(grammar::parse("ifhello")); - insta::assert_debug_snapshot!(grammar::parse("if hello")); + insta::assert_debug_snapshot!(grammar::Words::parse("if")); + insta::assert_debug_snapshot!(grammar::Words::parse("hello")); + insta::assert_debug_snapshot!(grammar::Words::parse("ifhello")); + insta::assert_debug_snapshot!(grammar::Words::parse("if hello")); } } diff --git a/macro/Cargo.toml b/macro/Cargo.toml index 57d0ac9..e40ac77 100644 --- a/macro/Cargo.toml +++ b/macro/Cargo.toml @@ -3,9 +3,9 @@ name = "rust-sitter-macro" description = "Procedural macros for Rust Sitter" readme = "../README.md" repository = "https://github.com/hydro-project/rust-sitter" -version = "0.4.5" -authors = ["Shadaj Laddad "] -edition = "2021" +version.workspace = true +authors.workspace = true +edition = "2024" license = "MIT" keywords = ["parsing", "codegen"] categories = ["development-tools"] @@ -18,8 +18,9 @@ path = "src/lib.rs" syn = { version = "2", features = [ "full", "extra-traits" ] } quote = "1" proc-macro2 = "1" -rust-sitter-common = { version= "0.4.5", path = "../common" } +rust-sitter-common = { path = "../common" } +rust-sitter-types = { path = "../types" } [dev-dependencies] -insta = "1.39" +insta = "1" tempfile = "3" diff --git a/macro/src/expansion.rs b/macro/src/expansion.rs index 75c2b80..6825a67 100644 --- a/macro/src/expansion.rs +++ b/macro/src/expansion.rs @@ -1,19 +1,14 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use crate::errors::IteratorExt as _; use proc_macro2::Span; -use quote::{quote, ToTokens}; -use rust_sitter_common::*; -use syn::{parse::Parse, punctuated::Punctuated, *}; - -fn is_sitter_attr(attr: &Attribute) -> bool { - attr.path() - .segments - .iter() - .next() - .map(|segment| segment.ident == "rust_sitter") - .unwrap_or(false) -} +use quote::{ToTokens, quote}; +use rust_sitter_common::{ + expansion::{ExpansionState, RuleDerive}, + *, +}; +use rust_sitter_types::grammar::{Grammar, RuleDef}; +use syn::{spanned::Spanned, *}; pub enum ParamOrField { Param(Expr), @@ -29,40 +24,230 @@ impl ToTokens for ParamOrField { } } -fn gen_field(ident_str: String, leaf: Field) -> Expr { - let leaf_type = leaf.ty; +pub fn expand_rule(input: DeriveInput) -> Result { + // At the very beginning, parse out the common rule json. This will pick up all of the errors + // there at compile time, and allow us to cleanly represent them. This is a lot of extra + // compilation time but it is the best we can do for now. Probably isn't noticable in general. + let d = RuleDerive::from_derive_input_known(input.clone())?; + let mut ctx = ExpansionState::new(); + rust_sitter_common::expansion::process_rule(d, &mut ctx)?; + + let ident = input.ident; + let attrs = input.attrs; + let (extract, rule) = match input.data { + Data::Struct(DataStruct { fields, .. }) => { + let extract_expr = gen_struct_or_variant( + fields.clone(), + None, + ident.clone(), + attrs.clone(), + &ctx.grammar, + )?; + + let extract_impl: Item = syn::parse_quote! { + impl ::rust_sitter::Extract for #ident { + type Output = Self; + type LeafFn = (); + #[allow(non_snake_case)] + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, + source: &[u8], + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| { + ::rust_sitter::error::ExtractError::missing_node(ctx) + })?; + #extract_expr + } + } + }; + let ident_str = ident.to_string(); + let rule_impl: Item = syn::parse_quote! { + impl ::rust_sitter::rule::Rule for #ident { + const RULE_NAME: &'static str = #ident_str; + fn produce_ast() -> String { + String::new() + } + } + }; + + (extract_impl, rule_impl) + } + Data::Enum(DataEnum { variants, .. }) => { + let match_cases: Vec = variants + .iter() + .map(|v| { + let variant_path = format!("{}_{}", ident, v.ident); + + let extract_expr = gen_struct_or_variant( + v.fields.clone(), + Some(v.ident.clone()), + ident.clone(), + v.attrs.clone(), + &ctx.grammar, + )?; + Ok(syn::parse_quote! { + #variant_path => return #extract_expr + }) + }) + .sift::>()?; + + let enum_name = &ident; + let ident_str = enum_name.to_string(); + let extract_impl: Item = syn::parse_quote! { + impl ::rust_sitter::Extract for #enum_name { + type Output = Self; + type LeafFn = (); + #[allow(non_snake_case)] + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, + source: &[u8], + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| { + ::rust_sitter::error::ExtractError::missing_node(ctx) + })?; + + let mut cursor = node.walk(); + if !cursor.goto_first_child() { + return Err(::rust_sitter::error::ExtractError::missing_node(ctx)); + } + loop { + let node = cursor.node(); + match node.kind() { + #(#match_cases),*, + k => if !cursor.goto_next_sibling() { + return Err(::rust_sitter::error::ExtractError::missing_enum(ctx)); + } + } + } + } + } + }; + + let rule_impl: Item = syn::parse_quote! { + impl ::rust_sitter::rule::Rule for #enum_name { + const RULE_NAME: &'static str = #ident_str; + fn produce_ast() -> String { + String::new() + } + } + }; + (extract_impl, rule_impl) + } + Data::Union(_) => return Err(Error::new(ident.span(), "Union types not supported")), + }; + + // If it is language, then we need to generate the corresponding functions. + let lang = if let Some((ident, lang)) = ctx.language_rule { + let name = lang.name().unwrap_or_else(|| ident.to_string()); + let tree_sitter_ident = Ident::new(&format!("tree_sitter_{name}"), Span::call_site()); + + let root_type_docstr = format!("[`{ident}`]"); + quote! { + impl ::rust_sitter::rule::Language for #ident { + fn produce_grammar() -> String { + String::new() + } + + fn language() -> ::rust_sitter::tree_sitter::Language { + unsafe extern "C" { + fn #tree_sitter_ident() -> ::rust_sitter::tree_sitter::Language; + } + unsafe { #tree_sitter_ident() } + } + /// Parse an input string according to the grammar. Returns either any parsing errors that happened, or a + #[doc = #root_type_docstr] + /// instance containing the parsed structured data. + fn parse(input: &str) -> ::rust_sitter::ParseResult { + ::rust_sitter::__private::parse(input, Self::language) + } + } + } + } else { + quote! {} + }; + + Ok(quote! { + #lang + #extract + #rule + }) +} + +fn gen_field(ident_str: Option<&str>, leaf: Field, rule: &RuleDef) -> Result { + let leaf_type = &leaf.ty; let leaf_attr = leaf .attrs .iter() - .find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::leaf)); + .find(|attr| sitter_attr_matches(attr, "leaf")); - let leaf_params = leaf_attr.and_then(|a| { - a.parse_args_with(Punctuated::::parse_terminated) - .ok() - }); + let transform = leaf + .attrs + .iter() + .find(|attr| sitter_attr_matches(attr, "transform") || sitter_attr_matches(attr, "with")); - let transform_param = leaf_params.as_ref().and_then(|p| { - p.iter() - .find(|param| param.path == "transform") - .map(|p| p.expr.clone()) - }); + if transform.is_some() && leaf_attr.is_none() { + return Err(Error::new(leaf.span(), "Cannot transform non-leaf nodes")); + } + + let (ident_str, mut should_skip) = match ident_str { + Some(n) => (n, false), + None => ("", true), + }; + + let text_attr = leaf + .attrs + .iter() + .find(|attr| sitter_attr_matches(attr, "text")); + if let Some(text_attr) = text_attr { + if let Some(attr) = leaf_attr { + return Err(Error::new( + attr.span(), + "Cannot use leaf and text at the same time", + )); + } + let text_input = text_attr.parse_args::()?; + text_input.evaluate()?; + should_skip = true; + } + + if should_skip { + // TODO: Handle this correctly. + return Ok(syn::parse_quote!({ + ::rust_sitter::__private::skip_text(state, #ident_str)?; + Ok::<_, ::rust_sitter::extract::ExtractError>(()) + })); + } + + let leaf_input = leaf_attr.map(|a| a.parse_args::()).transpose()?; + // But for now, we just evaluate it to make sure it works correctly. + if let Some(leaf_input) = leaf_input { + leaf_input.evaluate()?; + } + + let extractor: Expr = parse_quote! { ::rust_sitter::extract::BaseExtractor::default() }; - let (leaf_type, closure_expr): (Type, Expr) = match transform_param { + let (leaf_type, leaf_fn): (Type, Expr) = match transform { Some(closure) => { + let closure = closure.parse_args::()?; let mut non_leaf = HashSet::new(); non_leaf.insert("Spanned"); non_leaf.insert("Box"); non_leaf.insert("Option"); - non_leaf.insert("Vec"); - let wrapped_leaf_type = wrap_leaf_type(&leaf_type, &non_leaf); - (wrapped_leaf_type, syn::parse_quote!(Some(&#closure))) + let ty = wrap_leaf_type(leaf_type, &non_leaf); + (ty, closure) } - None => (leaf_type, syn::parse_quote!(None)), + None => (leaf_type.clone(), parse_quote! { () }), }; - syn::parse_quote!({ - ::rust_sitter::__private::extract_field::<#leaf_type,_>(cursor, source, last_idx, #ident_str, #closure_expr) + let extract_state = rule_def_to_extract(rule)?; + + Ok(parse_quote! { + ::rust_sitter::__private::extract_field::<#leaf_type, _>(#extractor, #leaf_fn, state, #extract_state, source, #ident_str) }) } @@ -71,7 +256,16 @@ fn gen_struct_or_variant( variant_ident: Option, containing_type: Ident, container_attrs: Vec, + grammar: &Grammar, ) -> Result { + let path = match &variant_ident { + Some(v) => format!("{containing_type}_{v}"), + None => containing_type.to_string(), + }; + let rule = grammar + .rules + .get(&path) + .expect("Unexpected state, no grammar found"); let children_parsed = if fields == Fields::Unit { let expr = { let dummy_field = Field { @@ -82,11 +276,36 @@ fn gen_struct_or_variant( colon_token: None, ty: Type::Verbatim(quote!(())), // unit type. }; - - gen_field("unit".to_string(), dummy_field) + let ident_str = if variant_ident.is_some() { + Some("unit") + } else { + None + }; + gen_field(ident_str, dummy_field, rule)? }; vec![ParamOrField::Param(expr)] } else { + // Parse out the rule into its appropriate sub parts. + // All top-level rules at this level are guaranteed to be `SEQ` of `FIELD`s. If a field is + // optional, the optional part comes before the `FIELD` definition, although that may be + // unnecessary. However, we don't need to check the fields specifically, because they can be + // determined by the actual field names instead. + let field_grammars: HashMap<_, _> = rule + .as_seq() + .expect("Must be a SEQ") + .iter() + .enumerate() + .zip(&fields) + .map(|((i, def), field)| { + let ident_str = field + .ident + .as_ref() + .map(|v| v.to_string()) + .unwrap_or(format!("{i}")); + (ident_str, def) + }) + .collect(); + fields .iter() .enumerate() @@ -94,7 +313,7 @@ fn gen_struct_or_variant( let expr = if let Some(skip_attrs) = field .attrs .iter() - .find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::skip)) + .find(|attr| sitter_attr_matches(attr, "skip")) { skip_attrs.parse_args::()? } else { @@ -104,7 +323,10 @@ fn gen_struct_or_variant( .map(|v| v.to_string()) .unwrap_or(format!("{i}")); - gen_field(ident_str, field.clone()) + let rule = field_grammars + .get(&ident_str) + .expect("Missing ident grammar"); + gen_field(Some(&ident_str), field.clone(), rule)? }; let field = if let Some(field_name) = &field.ident { @@ -140,205 +362,174 @@ fn gen_struct_or_variant( quote! { { - #expr; - #construct_name + #expr?; + Ok(#construct_name) } } } Fields::Named(_) => quote! { - #construct_name { - #(#children_parsed),* - } + Ok(#construct_name { + #(#children_parsed?),* + }) }, Fields::Unnamed(_) => quote! { - #construct_name( - #(#children_parsed),* - ) + Ok(#construct_name( + #(#children_parsed?),* + )) }, } }; Ok( - syn::parse_quote!(::rust_sitter::__private::extract_struct_or_variant(node, move |cursor, last_idx| #construct_expr)), + syn::parse_quote!(::rust_sitter::__private::extract_struct_or_variant( + stringify!(#construct_name), + node, + move |state| #construct_expr + )), ) } -pub fn expand_grammar(input: ItemMod) -> Result { - let grammar_name = input - .attrs - .iter() - .find_map(|a| { - if a.path() == &syn::parse_quote!(rust_sitter::grammar) { - let grammar_name_expr = a.parse_args_with(Expr::parse).ok(); - if let Some(Expr::Lit(ExprLit { - attrs: _, - lit: Lit::Str(s), - })) = grammar_name_expr - { - Some(Ok(s.value())) - } else { - Some(Err(syn::Error::new( - Span::call_site(), - "Expected a string literal grammar name", - ))) - } - } else { - None +fn rule_def_to_extract(def: &RuleDef) -> Result { + let mut states = vec![]; + // Handle if the top level rule is itself optional. + let optional = if let Some(def) = def.as_optional() { + // Don't propogate the optional to all of the inner states. + rule_def_add_state(def, false, &mut states); + true + } else { + rule_def_add_state(def, false, &mut states); + false + }; + let num_states = states.len() as u32; + let states = states.into_iter().enumerate().map(|(state, value)| { + let state = state as u32; + quote! { + #state => #value, + } + }); + Ok(quote! { + ::rust_sitter::extract::ExtractFieldContext::new(#num_states, #optional, |state| { + match state { + #(#states)* + #num_states => ::rust_sitter::extract::ExtractFieldState::Complete, + _ => ::rust_sitter::extract::ExtractFieldState::Overflow, } }) - .transpose()? - .ok_or_else(|| syn::Error::new(Span::call_site(), "Each grammar must have a name"))?; - - let (brace, new_contents) = input.content.ok_or_else(|| { - syn::Error::new( - Span::call_site(), - "Expected the module to have inline contents (`mod my_module { .. }` syntax)", - ) - })?; + }) +} - let root_type = new_contents - .iter() - .find_map(|item| match item { - Item::Enum(ItemEnum { ident, attrs, .. }) - | Item::Struct(ItemStruct { ident, attrs, .. }) => { - if attrs - .iter() - .any(|attr| attr.path() == &syn::parse_quote!(rust_sitter::language)) - { - Some(ident.clone()) - } else { - None - } +fn rule_def_add_state(def: &RuleDef, optional: bool, states: &mut Vec) { + let s = match def { + RuleDef::SYMBOL { name } => { + // This `grammar` is local to the particular macro expansion and does not include other + // grammars. If it exists here, then we need to return a special state which embeds the + // inner state within it. + quote! { + ::rust_sitter::extract::ExtractFieldState::Str(#name, true, #optional) } - _ => None, - }) - .ok_or_else(|| { - syn::Error::new( - Span::call_site(), - "Each parser must have the root type annotated with `#[rust_sitter::language]`", - ) - })?; - - let mut transformed: Vec = new_contents - .iter() - .cloned() - .map(|c| match c { - Item::Enum(mut e) => { - let match_cases: Vec = e.variants.iter().map(|v| { - let variant_path = format!("{}_{}", e.ident, v.ident); - - let extract_expr = gen_struct_or_variant( - v.fields.clone(), - Some(v.ident.clone()), - e.ident.clone(), - v.attrs.clone(), - )?; - Ok(syn::parse_quote! { - #variant_path => return #extract_expr - }) - }).sift::>()?; - - e.attrs.retain(|a| !is_sitter_attr(a)); - e.variants.iter_mut().for_each(|v| { - v.attrs.retain(|a| !is_sitter_attr(a)); - v.fields.iter_mut().for_each(|f| { - f.attrs.retain(|a| !is_sitter_attr(a)); - }); - }); - - let enum_name = &e.ident; - let extract_impl: Item = syn::parse_quote! { - impl ::rust_sitter::Extract<#enum_name> for #enum_name { - type LeafFn = (); - - #[allow(non_snake_case)] - fn extract(node: Option<::rust_sitter::tree_sitter::Node>, source: &[u8], _last_idx: usize, _leaf_fn: Option<&Self::LeafFn>) -> Self { - let node = node.unwrap(); - - let mut cursor = node.walk(); - assert!(cursor.goto_first_child(), "Could not find a child corresponding to any enum branch"); - loop { - let node = cursor.node(); - match node.kind() { - #(#match_cases),*, - _ => if !cursor.goto_next_sibling() { - panic!("Could not find a child corresponding to any enum branch") - } - } - } - } - } - }; - Ok(vec![Item::Enum(e), extract_impl]) + } + RuleDef::STRING { value } => { + quote! { + ::rust_sitter::extract::ExtractFieldState::Str(#value, false, #optional) } - - Item::Struct(mut s) => { - let struct_name = &s.ident; - let extract_expr = gen_struct_or_variant( - s.fields.clone(), - None, - s.ident.clone(), - s.attrs.clone(), - )?; - - s.attrs.retain(|a| !is_sitter_attr(a)); - s.fields.iter_mut().for_each(|f| { - f.attrs.retain(|a| !is_sitter_attr(a)); - }); - - - let extract_impl: Item = syn::parse_quote! { - impl ::rust_sitter::Extract<#struct_name> for #struct_name { - type LeafFn = (); - - #[allow(non_snake_case)] - fn extract(node: Option<::rust_sitter::tree_sitter::Node>, source: &[u8], last_idx: usize, _leaf_fn: Option<&Self::LeafFn>) -> Self { - let node = node.unwrap(); - #extract_expr - } - } - }; - - Ok(vec![Item::Struct(s), extract_impl]) + } + RuleDef::BLANK => return, + // Not sure what we get here, let's just assume the string is enough though. + RuleDef::PATTERN { .. } => { + // It is not possible to have these in direct field extractions, actually. A quirk of + // tree-sitter, they are always set to `.visible = false`. Maybe we can create a PR + // where PATTERNs can be exposed if they are wrapped in a FIELD. + return; + } + RuleDef::CHOICE { members } => { + // Special handle the optional case. + if let Some(value) = def.as_optional() { + return rule_def_add_state(value, true, states); + } else { + // Returns all choice values in a set. + // Note, that so far this works, however, I think it will fail if we have a tuple + // embedded in here from a local `grammar` rule. + let strs = members.iter().map(|s| match s { + RuleDef::STRING { value } => quote! { (#value, false) }, + RuleDef::SYMBOL { name } => quote! { (#name, true) }, + _ => panic!("CHOICE cannot use {s:#?} currently"), + }); + quote! { + ::rust_sitter::extract::ExtractFieldState::Choice(&[#(#strs),*], #optional) + } } - - o => Ok(vec![o]), - }) - .sift::>()?.into_iter().flatten().collect(); - - let tree_sitter_ident = Ident::new(&format!("tree_sitter_{grammar_name}"), Span::call_site()); - - transformed.push(syn::parse_quote! { - extern "C" { - fn #tree_sitter_ident() -> ::rust_sitter::tree_sitter::Language; } - }); - - transformed.push(syn::parse_quote! { - pub fn language() -> ::rust_sitter::tree_sitter::Language { - unsafe { #tree_sitter_ident() } + // TODO: Handle subfields appropriately? + RuleDef::FIELD { name: _, content } => { + return rule_def_add_state(content, optional, states); } - }); + RuleDef::SEQ { members } => { + return members + .iter() + .for_each(|def| rule_def_add_state(def, optional, states)); + } + RuleDef::PREC_DYNAMIC { value: _, content } + | RuleDef::PREC_LEFT { value: _, content } + | RuleDef::PREC_RIGHT { value: _, content } + | RuleDef::PREC { value: _, content } + | RuleDef::TOKEN { content } + | RuleDef::IMMEDIATE_TOKEN { content } => { + return rule_def_add_state(content, optional, states); + } + RuleDef::ALIAS { .. } => unreachable!("ALIAS not supported in this context"), + RuleDef::REPEAT { content } => { + // In the REPEAT case we only ever generate a rule that looks like this: + // seq($.rule, repeat($.sep, $.rule)), + // so we will have pushed all of the elements of `$.rule` already into the extraction + // state. We just need to insert a `REPEAT` then on the first token. + let seq = content.as_seq().expect("REPEAT was not a sequence"); + let repeat_rule = match seq { + [rule, _] => rule, + _ => panic!("REPEAT had more than two rules"), + }; + // Technically we support anything here as the first rule - it could be a sequence, + // etc. but practically it is only ever literal strings. Maybe SYMBOL. So we will only + // support those for now. + let (value, named) = match repeat_rule { + RuleDef::SYMBOL { name } => (name, true), + RuleDef::STRING { value } => (value, false), + _ => panic!("sep_by can only use SYMBOL or STRING currently"), + }; + quote! { + ::rust_sitter::extract::ExtractFieldState::Repeat(#value, #named) + } + } + RuleDef::REPEAT1 { content } => match &**content { + RuleDef::SEQ { members } => { + let repeat_rule = match members.as_slice() { + [rule, _] => rule, + _ => panic!("REPEAT1 had more than two rules"), + }; + let (value, named) = match repeat_rule { + RuleDef::SYMBOL { name } => (name, true), + RuleDef::STRING { value } => (value, false), + _ => panic!("sep_by can only use SYMBOL or STRING currently"), + }; + quote! { + ::rust_sitter::extract::ExtractFieldState::Repeat(#value, #named) + } + } + RuleDef::FIELD { name: _, content } => { + // Add all the inner states of the REPEAT1, then conclude with a repeat state. + // Note, that currently we don't support inner repeats, although we could, it would + // look like a tuple (e.g. `(u32, Vec, ())`), and to support that we just + // need the repeat enum to hold the state it should return to instead of just + // returning to zero. It also will need to know when the field ends - that is, it + // would need to check that the first value isn't the repeat value. + rule_def_add_state(content, optional, states); + quote! { + ::rust_sitter::extract::ExtractFieldState::Repeat1 + } + } + _ => panic!("Unsupported input in REPEAT1"), + }, + RuleDef::RESERVED { .. } => unreachable!("RESERVED not supported in this context"), + }; - let root_type_docstr = format!("[`{root_type}`]"); - transformed.push(syn::parse_quote! { - /// Parse an input string according to the grammar. Returns either any parsing errors that happened, or a - #[doc = #root_type_docstr] - /// instance containing the parsed structured data. - pub fn parse(input: &str) -> core::result::Result<#root_type, Vec<::rust_sitter::errors::ParseError>> { - ::rust_sitter::__private::parse::<#root_type>(input, language) - } - }); - - let mut filtered_attrs = input.attrs; - filtered_attrs.retain(|a| !is_sitter_attr(a)); - Ok(ItemMod { - attrs: filtered_attrs, - vis: input.vis, - unsafety: None, - mod_token: input.mod_token, - ident: input.ident, - content: Some((brace, transformed)), - semi: input.semi, - }) + states.push(s); } diff --git a/macro/src/lib.rs b/macro/src/lib.rs index d65d083..833a666 100644 --- a/macro/src/lib.rs +++ b/macro/src/lib.rs @@ -1,226 +1,59 @@ -use quote::ToTokens; -use syn::{parse_macro_input, ItemMod}; +use syn::{DeriveInput, parse_macro_input}; mod errors; mod expansion; +// mod grammar; use expansion::*; -#[proc_macro_attribute] -/// Marks the top level AST node where parsing should start. -/// -/// ## Example -/// ```ignore -/// #[rust_sitter::language] -/// pub struct Code { -/// ... -/// } -/// ``` -pub fn language( - _attr: proc_macro::TokenStream, - item: proc_macro::TokenStream, -) -> proc_macro::TokenStream { - item -} - -#[proc_macro_attribute] -/// This annotation marks a node as extra, which can safely be skipped while parsing. -/// This is useful for handling whitespace/newlines/comments. -/// -/// ## Example -/// ```ignore -/// #[rust_sitter::extra] -/// struct Whitespace { -/// #[rust_sitter::leaf(pattern = r"\s")] -/// _whitespace: (), -/// } -/// ``` -pub fn extra( - _attr: proc_macro::TokenStream, - item: proc_macro::TokenStream, -) -> proc_macro::TokenStream { - item -} - -#[proc_macro_attribute] -/// Defines a field which matches a specific token in the source string. -/// The token can be defined by passing one of two arguments -/// - `text`: a string literal that will be exactly matched -/// - `pattern`: a regular expression that will be matched against the source string -/// -/// If the resulting token needs to be converted into a richer type at runtime, -/// such as a number, then the `transform` argument can be used to specify a function -/// that will be called with the token's text. -/// -/// The attribute can also be applied to a struct or enum variant with no fields. -/// -/// ## Examples -/// -/// Using the `leaf` attribute on a field: -/// ```ignore -/// Number( -/// #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] -/// u32 -/// ) -/// ``` -/// -/// Using the attribute on a unit struct or unit enum variant: -/// ```ignore -/// #[rust_sitter::leaf(text = "9")] -/// struct BigDigit; -/// -/// enum SmallDigit { -/// #[rust_sitter::leaf(text = "0")] -/// Zero, -/// #[rust_sitter::leaf(text = "1")] -/// One, -/// } -/// ``` -/// -pub fn leaf( - _attr: proc_macro::TokenStream, - item: proc_macro::TokenStream, -) -> proc_macro::TokenStream { - item -} - -#[proc_macro_attribute] -/// Defines a field that does not correspond to anything in the input string, -/// such as some metadata. Takes a single, unnamed argument, which is the value -/// used to populate the field at runtime. -/// -/// ## Example -/// ```ignore -/// struct MyNode { -/// ..., -/// #[rust_sitter::skip(false)] -/// node_visited: bool -/// } -/// ``` -pub fn skip( - _attr: proc_macro::TokenStream, - item: proc_macro::TokenStream, -) -> proc_macro::TokenStream { - item -} - -#[proc_macro_attribute] -/// Defines a precedence level for a non-terminal that has no associativity. -/// -/// This annotation takes a single, unnamed parameter, which specifies the precedence level. -/// This is used to resolve conflicts with other non-terminals, so that the one with the higher -/// precedence will bind more tightly (appear lower in the parse tree). -/// -/// ## Example -/// ```ignore -/// #[rust_sitter::prec(1)] -/// PriorityExpr(Box, Box) -/// ``` -pub fn prec( - _attr: proc_macro::TokenStream, - item: proc_macro::TokenStream, -) -> proc_macro::TokenStream { - item -} - -#[proc_macro_attribute] -/// Defines a precedence level for a non-terminal that should be left-associative. -/// For example, with subtraction we expect 1 - 2 - 3 to be parsed as (1 - 2) - 3, -/// which corresponds to a left-associativity. -/// -/// This annotation takes a single, unnamed parameter, which specifies the precedence level. -/// This is used to resolve conflicts with other non-terminals, so that the one with the higher -/// precedence will bind more tightly (appear lower in the parse tree). -/// -/// ## Example -/// ```ignore -/// #[rust_sitter::prec_left(1)] -/// Subtract(Box, Box) -/// ``` -pub fn prec_left( - _attr: proc_macro::TokenStream, - item: proc_macro::TokenStream, -) -> proc_macro::TokenStream { - item -} - -#[proc_macro_attribute] -/// Defines a precedence level for a non-terminal that should be right-associative. -/// For example, with cons we could have 1 :: 2 :: 3 to be parsed as 1 :: (2 :: 3), -/// which corresponds to a right-associativity. -/// -/// This annotation takes a single, unnamed parameter, which specifies the precedence level. -/// This is used to resolve conflicts with other non-terminals, so that the one with the higher -/// precedence will bind more tightly (appear lower in the parse tree). -/// -/// ## Example -/// ```ignore -/// #[rust_sitter::prec_right(1)] -/// Cons(Box, Box) -/// ``` -pub fn prec_right( - _attr: proc_macro::TokenStream, - item: proc_macro::TokenStream, -) -> proc_macro::TokenStream { - item -} - -#[proc_macro_attribute] -/// On `Vec<_>` typed fields, specifies a non-terminal that should be parsed in between the elements. -/// The [`rust_sitter::repeat`] annotation must be used on the field as well. -/// -/// This annotation takes a single, unnamed argument, which specifies a field type to parse. This can -/// either be a reference to another type, or can be defined as a `leaf` field. Generally, the argument -/// is parsed using the same rules as an unnamed field of an enum variant. -/// -/// ## Example -/// ```ignore -/// #[rust_sitter::delimited( -/// #[rust_sitter::leaf(text = ",")] -/// () -/// )] -/// numbers: Vec -/// ``` -pub fn delimited( - _attr: proc_macro::TokenStream, - item: proc_macro::TokenStream, -) -> proc_macro::TokenStream { - item -} - -#[proc_macro_attribute] -/// On `Vec<_>` typed fields, specifies additional config for how the repeated elements should -/// be parsed. In particular, this annotation takes the following named arguments: -/// - `non_empty` - if this argument is `true`, then there must be at least one element parsed -/// -/// ## Example -/// ```ignore -/// #[rust_sitter::repeat(non_empty = true)] -/// numbers: Vec -/// ``` -pub fn repeat( - _attr: proc_macro::TokenStream, - item: proc_macro::TokenStream, -) -> proc_macro::TokenStream { - item -} - -/// Mark a module to be analyzed for a Rust Sitter grammar. Takes a single, unnamed argument, which -/// specifies the name of the grammar. This name must be unique across all Rust Sitter grammars within -/// a compilation unit. -#[proc_macro_attribute] -pub fn grammar( - attr: proc_macro::TokenStream, - input: proc_macro::TokenStream, -) -> proc_macro::TokenStream { - let attr_tokens: proc_macro2::TokenStream = attr.into(); - let module: ItemMod = parse_macro_input!(input); - let expanded = expand_grammar(syn::parse_quote! { - #[rust_sitter::grammar[#attr_tokens]] - #module - }) - .map(ToTokens::into_token_stream) - .unwrap_or_else(syn::Error::into_compile_error); - proc_macro::TokenStream::from(expanded) +// // TODO: Make a direct grammar function... +// This would allow us to write something like: +// struct Function { +// name: String, +// inputs: Vec, +// } +// grammar! { +// rule: seq("function", $.ident, "(", repeat($.input), ")") -> |id, inputs| Function { name, +// inputs: inputs.into() }; +// +// ident: /re/; +// input: seq($.ident, ":", $.ident); +// +// } +// #[proc_macro] +// pub fn grammar(input: proc_macro::TokenStream) -> proc_macro::TokenStream { +// grammar::parse_grammar_macro(input) +// } + +#[proc_macro_derive( + Rule, + // Alternatively, we can instead have one helper like `tree(...)` - generally looks cleaner. + attributes( + // Helper + language, + word, + leaf, + text, + prec, + prec_left, + prec_right, + prec_dynamic, + // TODO: This will instead be on a derive(Language) as well as others like conflicts, + // externals, inline, word, supertypes, etc. to fill out the full grammar specification. + extras, + with, + transform, + sep_by, + // Helper! + sep_by1, + repeat1, + skip, + ) +)] +pub fn derive_rule(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let input = parse_macro_input!(input as DeriveInput); + expand_rule(input) + .unwrap_or_else(syn::Error::into_compile_error) + .into() } #[cfg(test)] @@ -230,11 +63,33 @@ mod tests { use std::process::Command; use quote::ToTokens; - use syn::{parse_quote, Result}; + use syn::{ItemMod, Result, parse_quote}; use tempfile::tempdir; - use super::expand_grammar; - + use crate::expand_rule; + + // Allows expanding multiple rules at once. + fn expand_grammar(input: ItemMod) -> ItemMod { + let (_, items) = input.content.unwrap(); + let mut output = vec![]; + for item in items { + let stream = item.to_token_stream(); + // This might not actually work... + if let Ok(parsed) = syn::parse2(stream.clone()) { + let result = expand_rule(parsed).unwrap(); + output.push(result); + } else { + output.push(stream); + } + } + let mod_name = input.ident; + + parse_quote! { + mod #mod_name { + #(#output)* + } + } + } fn rustfmt_code(code: &str) -> String { let dir = tempdir().unwrap(); let file_path = dir.path().join("temp.rs"); @@ -262,17 +117,18 @@ mod tests { fn enum_transformed_fields() -> Result<()> { insta::assert_snapshot!(rustfmt_code( &expand_grammar(parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + use rust_sitter::Rule; + #[derive(Rule)] + #[language] pub enum Expression { Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse::().unwrap())] + #[leaf(re(r"\d+"))] i32 ), } } - })? + }) .to_token_stream() .to_string() )); @@ -284,22 +140,22 @@ mod tests { fn enum_recursive() -> Result<()> { insta::assert_snapshot!(rustfmt_code( &expand_grammar(parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub enum Expression { Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] i32 ), Neg( - #[rust_sitter::leaf(text = "-")] + #[leaf("-")] (), Box ), } } - })? + }) .to_token_stream() .to_string() )); @@ -311,24 +167,24 @@ mod tests { fn enum_prec_left() -> Result<()> { insta::assert_snapshot!(rustfmt_code( &expand_grammar(parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub enum Expression { Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(pattern(r"\d+"))] i32 ), - #[rust_sitter::prec_left(1)] + #[prec_left(1)] Sub( Box, - #[rust_sitter::leaf(text = "-")] + #[leaf("-")] (), Box ), } } - })? + }) .to_token_stream() .to_string() )); @@ -340,22 +196,23 @@ mod tests { fn struct_extra() -> Result<()> { insta::assert_snapshot!(rustfmt_code( &expand_grammar(parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub enum Expression { Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] i32, + #[leaf(re(r"\d+"))] i32, ), } - #[rust_sitter::extra] + #[derive(Rule)] + #[extra] struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s")] + #[leaf(pattern(r"\s"))] _whitespace: (), } } - })? + }) .to_token_stream() .to_string() )); @@ -367,21 +224,22 @@ mod tests { fn grammar_unboxed_field() -> Result<()> { insta::assert_snapshot!(rustfmt_code( &expand_grammar(parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub struct Language { e: Expression, } + #[derive(rust_sitter::Rule)] pub enum Expression { Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v: &str| v.parse::().unwrap())] + #[leaf(re(r"\d+"))] i32 ), } } - })? + }) .to_token_stream() .to_string() )); @@ -393,25 +251,27 @@ mod tests { fn struct_repeat() -> Result<()> { insta::assert_snapshot!(rustfmt_code( &expand_grammar(parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub struct NumberList { numbers: Vec, } + #[derive(rust_sitter::Rule)] pub struct Number { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] v: i32 } - #[rust_sitter::extra] + #[derive(rust_sitter::Rule)] + #[extra] struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s")] + #[leaf(pattern(r"\s"))] _whitespace: (), } } - })? + }) .to_token_stream() .to_string() )); @@ -423,21 +283,22 @@ mod tests { fn struct_optional() -> Result<()> { insta::assert_snapshot!(rustfmt_code( &expand_grammar(parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub struct Language { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] v: Option, t: Option, } + #[derive(rust_sitter::Rule)] pub struct Number { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] v: i32 } } - })? + }) .to_token_stream() .to_string() )); @@ -449,22 +310,23 @@ mod tests { fn enum_with_unamed_vector() -> Result<()> { insta::assert_snapshot!(rustfmt_code( &expand_grammar(parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { + #[derive(rust_sitter::Rule)] pub struct Number { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] value: u32 } - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub enum Expr { Numbers( - #[rust_sitter::repeat(non_empty = true)] + #[repeat1] Vec ) } } - })? + }) .to_token_stream() .to_string() )); @@ -476,22 +338,22 @@ mod tests { fn enum_with_named_field() -> Result<()> { insta::assert_snapshot!(rustfmt_code( &expand_grammar(parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub enum Expr { Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(pattern(r"\d+"))] u32 ), Neg { - #[rust_sitter::leaf(text = "!")] + #[leaf("!")] _bang: (), value: Box, } } } - })? + }) .to_token_stream() .to_string() )); @@ -503,27 +365,29 @@ mod tests { fn spanned_in_vec() -> Result<()> { insta::assert_snapshot!(rustfmt_code( &expand_grammar(parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - use rust_sitter::Spanned; + use rust_sitter::{Rule, Spanned}; - #[rust_sitter::language] + #[derive(Rule)] + #[language] pub struct NumberList { numbers: Vec>, } + #[derive(Rule)] pub struct Number { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] v: i32 } - #[rust_sitter::extra] + #[derive(Rule)] + #[extra] struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s")] + #[leaf(pattern(r"\s"))] _whitespace: (), } } - })? + }) .to_token_stream() .to_string() )); diff --git a/macro/src/snapshots/rust_sitter_macro__tests__enum_prec_left.snap b/macro/src/snapshots/rust_sitter_macro__tests__enum_prec_left.snap index e9ae783..1ad76c1 100644 --- a/macro/src/snapshots/rust_sitter_macro__tests__enum_prec_left.snap +++ b/macro/src/snapshots/rust_sitter_macro__tests__enum_prec_left.snap @@ -1,91 +1,132 @@ --- source: macro/src/lib.rs -expression: "rustfmt_code(&expand_grammar(parse_quote! {\n #[rust_sitter :: grammar(\"test\")] mod grammar\n {\n #[rust_sitter :: language] pub enum Expression\n {\n Number(#[rust_sitter ::\n leaf(pattern = r\"\\d+\", transform = | v |\n v.parse().unwrap())] i32), #[rust_sitter :: prec_left(1)]\n Sub(Box < Expression >, #[rust_sitter :: leaf(text = \"-\")]\n (), Box < Expression >),\n }\n }\n })?.to_token_stream().to_string())" +expression: "rustfmt_code(&expand_grammar(parse_quote!\n{\n mod grammar\n {\n #[derive(rust_sitter::Rule)] #[language] pub enum Expression\n {\n Number(#[leaf(pattern(r\"\\d+\"))] i32), #[prec_left(1)]\n Sub(Box, #[leaf(\"-\")] (), Box),\n }\n }\n}).to_token_stream().to_string())" --- mod grammar { - pub enum Expression { - Number(i32), - Sub(Box, (), Box), + impl ::rust_sitter::rule::Language for Expression { + fn produce_grammar() -> String { + String::new() + } + fn language() -> ::rust_sitter::tree_sitter::Language { + unsafe extern "C" { + fn tree_sitter_Expression() -> ::rust_sitter::tree_sitter::Language; + } + unsafe { tree_sitter_Expression() } + } + #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] + #[doc = "[`Expression`]"] + #[doc = r" instance containing the parsed structured data."] + fn parse(input: &str) -> ::rust_sitter::ParseResult { + ::rust_sitter::__private::parse(input, Self::language) + } } - impl ::rust_sitter::Extract for Expression { + impl ::rust_sitter::Extract for Expression { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - _last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; let mut cursor = node.walk(); - assert!( - cursor.goto_first_child(), - "Could not find a child corresponding to any enum branch" - ); + if !cursor.goto_first_child() { + return Err(::rust_sitter::error::ExtractError::missing_node(ctx)); + } loop { let node = cursor.node(); match node.kind() { "Expression_Number" => { return ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Expression::Number), node, - move |cursor, last_idx| { - Expression::Number({ - ::rust_sitter::__private::extract_field::< - rust_sitter::WithLeaf, - _, - >( - cursor, source, last_idx, "0", Some(&|v| v.parse().unwrap()) - ) - }) + move |state| { + Ok(Expression::Number( + ::rust_sitter::__private::extract_field::( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 0u32, + false, + |state| { + match state { 0u32 => :: rust_sitter :: extract :: ExtractFieldState :: Complete , _ => :: rust_sitter :: extract :: ExtractFieldState :: Overflow , } + }, + ), + source, + "0", + )?, + )) }, ) } "Expression_Sub" => { return ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Expression::Sub), node, - move |cursor, last_idx| { - Expression::Sub( - { - ::rust_sitter::__private::extract_field::, _>( - cursor, source, last_idx, "0", None, - ) - }, - { - ::rust_sitter::__private::extract_field::<(), _>( - cursor, source, last_idx, "1", None, - ) - }, - { - ::rust_sitter::__private::extract_field::, _>( - cursor, source, last_idx, "2", None, - ) - }, - ) + move |state| { + Ok(Expression::Sub( + ::rust_sitter::__private::extract_field::, _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 1u32, + false, + |state| { + match state { 0u32 => :: rust_sitter :: extract :: ExtractFieldState :: Str ("Expression" , true , false) , 1u32 => :: rust_sitter :: extract :: ExtractFieldState :: Complete , _ => :: rust_sitter :: extract :: ExtractFieldState :: Overflow , } + }, + ), + source, + "0", + )?, + ::rust_sitter::__private::extract_field::<(), _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 1u32, + false, + |state| { + match state { 0u32 => :: rust_sitter :: extract :: ExtractFieldState :: Str ("-" , false , false) , 1u32 => :: rust_sitter :: extract :: ExtractFieldState :: Complete , _ => :: rust_sitter :: extract :: ExtractFieldState :: Overflow , } + }, + ), + source, + "1", + )?, + ::rust_sitter::__private::extract_field::, _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 1u32, + false, + |state| { + match state { 0u32 => :: rust_sitter :: extract :: ExtractFieldState :: Str ("Expression" , true , false) , 1u32 => :: rust_sitter :: extract :: ExtractFieldState :: Complete , _ => :: rust_sitter :: extract :: ExtractFieldState :: Overflow , } + }, + ), + source, + "2", + )?, + )) }, ) } - _ => { + k => { if !cursor.goto_next_sibling() { - panic!("Could not find a child corresponding to any enum branch") + return Err(::rust_sitter::error::ExtractError::missing_enum(ctx)); } } } } } } - extern "C" { - fn tree_sitter_test() -> ::rust_sitter::tree_sitter::Language; - } - pub fn language() -> ::rust_sitter::tree_sitter::Language { - unsafe { tree_sitter_test() } - } - #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] - #[doc = "[`Expression`]"] - #[doc = r" instance containing the parsed structured data."] - pub fn parse( - input: &str, - ) -> core::result::Result> { - ::rust_sitter::__private::parse::(input, language) + impl ::rust_sitter::rule::Rule for Expression { + const RULE_NAME: &'static str = "Expression"; + fn produce_ast() -> String { + String::new() + } } } - diff --git a/macro/src/snapshots/rust_sitter_macro__tests__enum_recursive.snap b/macro/src/snapshots/rust_sitter_macro__tests__enum_recursive.snap index 3cadc92..99bda4f 100644 --- a/macro/src/snapshots/rust_sitter_macro__tests__enum_recursive.snap +++ b/macro/src/snapshots/rust_sitter_macro__tests__enum_recursive.snap @@ -1,86 +1,118 @@ --- source: macro/src/lib.rs -expression: "rustfmt_code(&expand_grammar(parse_quote! {\n #[rust_sitter :: grammar(\"test\")] mod grammar\n {\n #[rust_sitter :: language] pub enum Expression\n {\n Number(#[rust_sitter ::\n leaf(pattern = r\"\\d+\", transform = | v |\n v.parse().unwrap())] i32),\n Neg(#[rust_sitter :: leaf(text = \"-\")] (), Box < Expression\n >),\n }\n }\n })?.to_token_stream().to_string())" +expression: "rustfmt_code(&expand_grammar(parse_quote!\n{\n mod grammar\n {\n #[derive(rust_sitter::Rule)] #[language] pub enum Expression\n {\n Number(#[leaf(re(r\"\\d+\"))] i32),\n Neg(#[leaf(\"-\")] (), Box),\n }\n }\n}).to_token_stream().to_string())" --- mod grammar { - pub enum Expression { - Number(i32), - Neg((), Box), + impl ::rust_sitter::rule::Language for Expression { + fn produce_grammar() -> String { + String::new() + } + fn language() -> ::rust_sitter::tree_sitter::Language { + unsafe extern "C" { + fn tree_sitter_Expression() -> ::rust_sitter::tree_sitter::Language; + } + unsafe { tree_sitter_Expression() } + } + #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] + #[doc = "[`Expression`]"] + #[doc = r" instance containing the parsed structured data."] + fn parse(input: &str) -> ::rust_sitter::ParseResult { + ::rust_sitter::__private::parse(input, Self::language) + } } - impl ::rust_sitter::Extract for Expression { + impl ::rust_sitter::Extract for Expression { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - _last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; let mut cursor = node.walk(); - assert!( - cursor.goto_first_child(), - "Could not find a child corresponding to any enum branch" - ); + if !cursor.goto_first_child() { + return Err(::rust_sitter::error::ExtractError::missing_node(ctx)); + } loop { let node = cursor.node(); match node.kind() { "Expression_Number" => { return ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Expression::Number), node, - move |cursor, last_idx| { - Expression::Number({ - ::rust_sitter::__private::extract_field::< - rust_sitter::WithLeaf, - _, - >( - cursor, source, last_idx, "0", Some(&|v| v.parse().unwrap()) - ) - }) + move |state| { + Ok(Expression::Number( + ::rust_sitter::__private::extract_field::( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 0u32, + false, + |state| { + match state { 0u32 => :: rust_sitter :: extract :: ExtractFieldState :: Complete , _ => :: rust_sitter :: extract :: ExtractFieldState :: Overflow , } + }, + ), + source, + "0", + )?, + )) }, ) } "Expression_Neg" => { return ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Expression::Neg), node, - move |cursor, last_idx| { - Expression::Neg( - { - ::rust_sitter::__private::extract_field::<(), _>( - cursor, source, last_idx, "0", None, - ) - }, - { - ::rust_sitter::__private::extract_field::, _>( - cursor, source, last_idx, "1", None, - ) - }, - ) + move |state| { + Ok(Expression::Neg( + ::rust_sitter::__private::extract_field::<(), _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 1u32, + false, + |state| { + match state { 0u32 => :: rust_sitter :: extract :: ExtractFieldState :: Str ("-" , false , false) , 1u32 => :: rust_sitter :: extract :: ExtractFieldState :: Complete , _ => :: rust_sitter :: extract :: ExtractFieldState :: Overflow , } + }, + ), + source, + "0", + )?, + ::rust_sitter::__private::extract_field::, _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 1u32, + false, + |state| { + match state { 0u32 => :: rust_sitter :: extract :: ExtractFieldState :: Str ("Expression" , true , false) , 1u32 => :: rust_sitter :: extract :: ExtractFieldState :: Complete , _ => :: rust_sitter :: extract :: ExtractFieldState :: Overflow , } + }, + ), + source, + "1", + )?, + )) }, ) } - _ => { + k => { if !cursor.goto_next_sibling() { - panic!("Could not find a child corresponding to any enum branch") + return Err(::rust_sitter::error::ExtractError::missing_enum(ctx)); } } } } } } - extern "C" { - fn tree_sitter_test() -> ::rust_sitter::tree_sitter::Language; - } - pub fn language() -> ::rust_sitter::tree_sitter::Language { - unsafe { tree_sitter_test() } - } - #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] - #[doc = "[`Expression`]"] - #[doc = r" instance containing the parsed structured data."] - pub fn parse( - input: &str, - ) -> core::result::Result> { - ::rust_sitter::__private::parse::(input, language) + impl ::rust_sitter::rule::Rule for Expression { + const RULE_NAME: &'static str = "Expression"; + fn produce_ast() -> String { + String::new() + } } } - diff --git a/macro/src/snapshots/rust_sitter_macro__tests__enum_transformed_fields.snap b/macro/src/snapshots/rust_sitter_macro__tests__enum_transformed_fields.snap index 3f0cb88..92795ce 100644 --- a/macro/src/snapshots/rust_sitter_macro__tests__enum_transformed_fields.snap +++ b/macro/src/snapshots/rust_sitter_macro__tests__enum_transformed_fields.snap @@ -1,70 +1,81 @@ --- source: macro/src/lib.rs -expression: "rustfmt_code(&expand_grammar(parse_quote! {\n #[rust_sitter :: grammar(\"test\")] mod grammar\n {\n #[rust_sitter :: language] pub enum Expression\n {\n Number(#[rust_sitter ::\n leaf(pattern = r\"\\d+\", transform = | v | v.parse :: < i32 >\n ().unwrap())] i32),\n }\n }\n })?.to_token_stream().to_string())" +expression: "rustfmt_code(&expand_grammar(parse_quote!\n{\n mod grammar\n {\n use rust_sitter::Rule; #[derive(Rule)] #[language] pub enum Expression\n { Number(#[leaf(re(r\"\\d+\"))] i32), }\n }\n}).to_token_stream().to_string())" --- mod grammar { - pub enum Expression { - Number(i32), + use rust_sitter::Rule; + impl ::rust_sitter::rule::Language for Expression { + fn produce_grammar() -> String { + String::new() + } + fn language() -> ::rust_sitter::tree_sitter::Language { + unsafe extern "C" { + fn tree_sitter_Expression() -> ::rust_sitter::tree_sitter::Language; + } + unsafe { tree_sitter_Expression() } + } + #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] + #[doc = "[`Expression`]"] + #[doc = r" instance containing the parsed structured data."] + fn parse(input: &str) -> ::rust_sitter::ParseResult { + ::rust_sitter::__private::parse(input, Self::language) + } } - impl ::rust_sitter::Extract for Expression { + impl ::rust_sitter::Extract for Expression { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - _last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; let mut cursor = node.walk(); - assert!( - cursor.goto_first_child(), - "Could not find a child corresponding to any enum branch" - ); + if !cursor.goto_first_child() { + return Err(::rust_sitter::error::ExtractError::missing_node(ctx)); + } loop { let node = cursor.node(); match node.kind() { "Expression_Number" => { return ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Expression::Number), node, - move |cursor, last_idx| { - Expression::Number({ - ::rust_sitter::__private::extract_field::< - rust_sitter::WithLeaf, - _, - >( - cursor, + move |state| { + Ok(Expression::Number( + ::rust_sitter::__private::extract_field::( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 0u32, + false, + |state| { + match state { 0u32 => :: rust_sitter :: extract :: ExtractFieldState :: Complete , _ => :: rust_sitter :: extract :: ExtractFieldState :: Overflow , } + }, + ), source, - last_idx, "0", - Some(&|v| v.parse::().unwrap()), - ) - }) + )?, + )) }, ) } - _ => { + k => { if !cursor.goto_next_sibling() { - panic!("Could not find a child corresponding to any enum branch") + return Err(::rust_sitter::error::ExtractError::missing_enum(ctx)); } } } } } } - extern "C" { - fn tree_sitter_test() -> ::rust_sitter::tree_sitter::Language; - } - pub fn language() -> ::rust_sitter::tree_sitter::Language { - unsafe { tree_sitter_test() } - } - #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] - #[doc = "[`Expression`]"] - #[doc = r" instance containing the parsed structured data."] - pub fn parse( - input: &str, - ) -> core::result::Result> { - ::rust_sitter::__private::parse::(input, language) + impl ::rust_sitter::rule::Rule for Expression { + const RULE_NAME: &'static str = "Expression"; + fn produce_ast() -> String { + String::new() + } } } - diff --git a/macro/src/snapshots/rust_sitter_macro__tests__enum_with_named_field.snap b/macro/src/snapshots/rust_sitter_macro__tests__enum_with_named_field.snap index b6fc87c..d66293f 100644 --- a/macro/src/snapshots/rust_sitter_macro__tests__enum_with_named_field.snap +++ b/macro/src/snapshots/rust_sitter_macro__tests__enum_with_named_field.snap @@ -1,84 +1,124 @@ --- source: macro/src/lib.rs -expression: "rustfmt_code(&expand_grammar(parse_quote! {\n #[rust_sitter :: grammar(\"test\")] mod grammar\n {\n #[rust_sitter :: language] pub enum Expr\n {\n Number(#[rust_sitter ::\n leaf(pattern = r\"\\d+\", transform = | v |\n v.parse().unwrap())] u32), Neg\n {\n #[rust_sitter :: leaf(text = \"!\")] _bang : (), value : Box <\n Expr >,\n }\n }\n }\n })?.to_token_stream().to_string())" +expression: "rustfmt_code(&expand_grammar(parse_quote!\n{\n mod grammar\n {\n #[derive(rust_sitter::Rule)] #[language] pub enum Expr\n {\n Number(#[leaf(pattern(r\"\\d+\"))] u32), Neg\n { #[leaf(\"!\")] _bang: (), value: Box, }\n }\n }\n}).to_token_stream().to_string())" --- mod grammar { - pub enum Expr { - Number(u32), - Neg { _bang: (), value: Box }, + impl ::rust_sitter::rule::Language for Expr { + fn produce_grammar() -> String { + String::new() + } + fn language() -> ::rust_sitter::tree_sitter::Language { + unsafe extern "C" { + fn tree_sitter_Expr() -> ::rust_sitter::tree_sitter::Language; + } + unsafe { tree_sitter_Expr() } + } + #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] + #[doc = "[`Expr`]"] + #[doc = r" instance containing the parsed structured data."] + fn parse(input: &str) -> ::rust_sitter::ParseResult { + ::rust_sitter::__private::parse(input, Self::language) + } } - impl ::rust_sitter::Extract for Expr { + impl ::rust_sitter::Extract for Expr { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - _last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; let mut cursor = node.walk(); - assert!( - cursor.goto_first_child(), - "Could not find a child corresponding to any enum branch" - ); + if !cursor.goto_first_child() { + return Err(::rust_sitter::error::ExtractError::missing_node(ctx)); + } loop { let node = cursor.node(); match node.kind() { "Expr_Number" => { return ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Expr::Number), node, - move |cursor, last_idx| { - Expr::Number({ - ::rust_sitter::__private::extract_field::< - rust_sitter::WithLeaf, - _, - >( - cursor, source, last_idx, "0", Some(&|v| v.parse().unwrap()) - ) - }) + move |state| { + Ok(Expr::Number(::rust_sitter::__private::extract_field::< + u32, + _, + >( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 0u32, + false, + |state| match state { + 0u32 => { + ::rust_sitter::extract::ExtractFieldState::Complete + } + _ => { + ::rust_sitter::extract::ExtractFieldState::Overflow + } + }, + ), + source, + "0", + )?)) }, ) } "Expr_Neg" => { return ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Expr::Neg), node, - move |cursor, last_idx| Expr::Neg { - _bang: { - ::rust_sitter::__private::extract_field::<(), _>( - cursor, source, last_idx, "_bang", None, - ) - }, - value: { - ::rust_sitter::__private::extract_field::, _>( - cursor, source, last_idx, "value", None, - ) - }, + move |state| { + Ok(Expr::Neg { + _bang: ::rust_sitter::__private::extract_field::<(), _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 1u32, + false, + |state| { + match state { 0u32 => :: rust_sitter :: extract :: ExtractFieldState :: Str ("!" , false , false) , 1u32 => :: rust_sitter :: extract :: ExtractFieldState :: Complete , _ => :: rust_sitter :: extract :: ExtractFieldState :: Overflow , } + }, + ), + source, + "_bang", + )?, + value: ::rust_sitter::__private::extract_field::, _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 1u32, + false, + |state| { + match state { 0u32 => :: rust_sitter :: extract :: ExtractFieldState :: Str ("Expr" , true , false) , 1u32 => :: rust_sitter :: extract :: ExtractFieldState :: Complete , _ => :: rust_sitter :: extract :: ExtractFieldState :: Overflow , } + }, + ), + source, + "value", + )?, + }) }, ) } - _ => { + k => { if !cursor.goto_next_sibling() { - panic!("Could not find a child corresponding to any enum branch") + return Err(::rust_sitter::error::ExtractError::missing_enum(ctx)); } } } } } } - extern "C" { - fn tree_sitter_test() -> ::rust_sitter::tree_sitter::Language; - } - pub fn language() -> ::rust_sitter::tree_sitter::Language { - unsafe { tree_sitter_test() } - } - #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] - #[doc = "[`Expr`]"] - #[doc = r" instance containing the parsed structured data."] - pub fn parse( - input: &str, - ) -> core::result::Result> { - ::rust_sitter::__private::parse::(input, language) + impl ::rust_sitter::rule::Rule for Expr { + const RULE_NAME: &'static str = "Expr"; + fn produce_ast() -> String { + String::new() + } } } - diff --git a/macro/src/snapshots/rust_sitter_macro__tests__enum_with_unamed_vector.snap b/macro/src/snapshots/rust_sitter_macro__tests__enum_with_unamed_vector.snap index 892ee17..bb45fd8 100644 --- a/macro/src/snapshots/rust_sitter_macro__tests__enum_with_unamed_vector.snap +++ b/macro/src/snapshots/rust_sitter_macro__tests__enum_with_unamed_vector.snap @@ -1,91 +1,134 @@ --- source: macro/src/lib.rs -expression: "rustfmt_code(&expand_grammar(parse_quote! {\n #[rust_sitter :: grammar(\"test\")] mod grammar\n {\n pub struct Number\n {\n #[rust_sitter ::\n leaf(pattern = r\"\\d+\", transform = | v |\n v.parse().unwrap())] value : u32\n } #[rust_sitter :: language] pub enum Expr\n {\n Numbers(#[rust_sitter :: repeat(non_empty = true)] Vec <\n Number >)\n }\n }\n })?.to_token_stream().to_string())" +expression: "rustfmt_code(&expand_grammar(parse_quote!\n{\n mod grammar\n {\n #[derive(rust_sitter::Rule)] pub struct Number\n { #[leaf(re(r\"\\d+\"))] value: u32 } #[derive(rust_sitter::Rule)]\n #[language] pub enum Expr { Numbers(#[repeat1] Vec) }\n }\n}).to_token_stream().to_string())" --- mod grammar { - pub struct Number { - value: u32, - } - impl ::rust_sitter::Extract for Number { + impl ::rust_sitter::Extract for Number { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); - ::rust_sitter::__private::extract_struct_or_variant(node, move |cursor, last_idx| { - Number { - value: { - ::rust_sitter::__private::extract_field::, _>( - cursor, + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; + ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Number), + node, + move |state| { + Ok(Number { + value: ::rust_sitter::__private::extract_field::( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 0u32, + false, + |state| match state { + 0u32 => ::rust_sitter::extract::ExtractFieldState::Complete, + _ => ::rust_sitter::extract::ExtractFieldState::Overflow, + }, + ), source, - last_idx, "value", - Some(&|v| v.parse().unwrap()), - ) - }, - } - }) + )?, + }) + }, + ) } } - pub enum Expr { - Numbers(Vec), + impl ::rust_sitter::rule::Rule for Number { + const RULE_NAME: &'static str = "Number"; + fn produce_ast() -> String { + String::new() + } } - impl ::rust_sitter::Extract for Expr { + impl ::rust_sitter::rule::Language for Expr { + fn produce_grammar() -> String { + String::new() + } + fn language() -> ::rust_sitter::tree_sitter::Language { + unsafe extern "C" { + fn tree_sitter_Expr() -> ::rust_sitter::tree_sitter::Language; + } + unsafe { tree_sitter_Expr() } + } + #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] + #[doc = "[`Expr`]"] + #[doc = r" instance containing the parsed structured data."] + fn parse(input: &str) -> ::rust_sitter::ParseResult { + ::rust_sitter::__private::parse(input, Self::language) + } + } + impl ::rust_sitter::Extract for Expr { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - _last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; let mut cursor = node.walk(); - assert!( - cursor.goto_first_child(), - "Could not find a child corresponding to any enum branch" - ); + if !cursor.goto_first_child() { + return Err(::rust_sitter::error::ExtractError::missing_node(ctx)); + } loop { let node = cursor.node(); match node.kind() { "Expr_Numbers" => { return ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Expr::Numbers), node, - move |cursor, last_idx| { - Expr::Numbers({ - ::rust_sitter::__private::extract_field::, _>( - cursor, source, last_idx, "0", None, - ) - }) + move |state| { + Ok(Expr::Numbers(::rust_sitter::__private::extract_field::< + Vec, + _, + >( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 2u32, + false, + |state| match state { + 0u32 => ::rust_sitter::extract::ExtractFieldState::Str( + "Number", true, false, + ), + 1u32 => { + ::rust_sitter::extract::ExtractFieldState::Repeat1 + } + 2u32 => { + ::rust_sitter::extract::ExtractFieldState::Complete + } + _ => { + ::rust_sitter::extract::ExtractFieldState::Overflow + } + }, + ), + source, + "0", + )?)) }, ) } - _ => { + k => { if !cursor.goto_next_sibling() { - panic!("Could not find a child corresponding to any enum branch") + return Err(::rust_sitter::error::ExtractError::missing_enum(ctx)); } } } } } } - extern "C" { - fn tree_sitter_test() -> ::rust_sitter::tree_sitter::Language; - } - pub fn language() -> ::rust_sitter::tree_sitter::Language { - unsafe { tree_sitter_test() } - } - #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] - #[doc = "[`Expr`]"] - #[doc = r" instance containing the parsed structured data."] - pub fn parse( - input: &str, - ) -> core::result::Result> { - ::rust_sitter::__private::parse::(input, language) + impl ::rust_sitter::rule::Rule for Expr { + const RULE_NAME: &'static str = "Expr"; + fn produce_ast() -> String { + String::new() + } } } - diff --git a/macro/src/snapshots/rust_sitter_macro__tests__grammar_unboxed_field.snap b/macro/src/snapshots/rust_sitter_macro__tests__grammar_unboxed_field.snap index 5e28401..9606554 100644 --- a/macro/src/snapshots/rust_sitter_macro__tests__grammar_unboxed_field.snap +++ b/macro/src/snapshots/rust_sitter_macro__tests__grammar_unboxed_field.snap @@ -1,94 +1,127 @@ --- source: macro/src/lib.rs -expression: "rustfmt_code(&expand_grammar(parse_quote! {\n #[rust_sitter :: grammar(\"test\")] mod grammar\n {\n #[rust_sitter :: language] pub struct Language\n { e : Expression, } pub enum Expression\n {\n Number(#[rust_sitter ::\n leaf(pattern = r\"\\d+\", transform = | v : & str | v.parse ::\n < i32 > ().unwrap())] i32),\n }\n }\n })?.to_token_stream().to_string())" +expression: "rustfmt_code(&expand_grammar(parse_quote!\n{\n mod grammar\n {\n #[derive(rust_sitter::Rule)] #[language] pub struct Language\n { e: Expression, } #[derive(rust_sitter::Rule)] pub enum Expression\n { Number(#[leaf(re(r\"\\d+\"))] i32), }\n }\n}).to_token_stream().to_string())" --- mod grammar { - pub struct Language { - e: Expression, + impl ::rust_sitter::rule::Language for Language { + fn produce_grammar() -> String { + String::new() + } + fn language() -> ::rust_sitter::tree_sitter::Language { + unsafe extern "C" { + fn tree_sitter_Language() -> ::rust_sitter::tree_sitter::Language; + } + unsafe { tree_sitter_Language() } + } + #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] + #[doc = "[`Language`]"] + #[doc = r" instance containing the parsed structured data."] + fn parse(input: &str) -> ::rust_sitter::ParseResult { + ::rust_sitter::__private::parse(input, Self::language) + } } - impl ::rust_sitter::Extract for Language { + impl ::rust_sitter::Extract for Language { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); - ::rust_sitter::__private::extract_struct_or_variant(node, move |cursor, last_idx| { - Language { - e: { - ::rust_sitter::__private::extract_field::( - cursor, source, last_idx, "e", None, - ) - }, - } - }) + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; + ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Language), + node, + move |state| { + Ok(Language { + e: ::rust_sitter::__private::extract_field::( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 1u32, + false, + |state| match state { + 0u32 => ::rust_sitter::extract::ExtractFieldState::Str( + "Expression", + true, + false, + ), + 1u32 => ::rust_sitter::extract::ExtractFieldState::Complete, + _ => ::rust_sitter::extract::ExtractFieldState::Overflow, + }, + ), + source, + "e", + )?, + }) + }, + ) } } - pub enum Expression { - Number(i32), + impl ::rust_sitter::rule::Rule for Language { + const RULE_NAME: &'static str = "Language"; + fn produce_ast() -> String { + String::new() + } } - impl ::rust_sitter::Extract for Expression { + impl ::rust_sitter::Extract for Expression { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - _last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; let mut cursor = node.walk(); - assert!( - cursor.goto_first_child(), - "Could not find a child corresponding to any enum branch" - ); + if !cursor.goto_first_child() { + return Err(::rust_sitter::error::ExtractError::missing_node(ctx)); + } loop { let node = cursor.node(); match node.kind() { "Expression_Number" => { return ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Expression::Number), node, - move |cursor, last_idx| { - Expression::Number({ - ::rust_sitter::__private::extract_field::< - rust_sitter::WithLeaf, - _, - >( - cursor, + move |state| { + Ok(Expression::Number( + ::rust_sitter::__private::extract_field::( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 0u32, + false, + |state| { + match state { 0u32 => :: rust_sitter :: extract :: ExtractFieldState :: Complete , _ => :: rust_sitter :: extract :: ExtractFieldState :: Overflow , } + }, + ), source, - last_idx, "0", - Some(&|v: &str| v.parse::().unwrap()), - ) - }) + )?, + )) }, ) } - _ => { + k => { if !cursor.goto_next_sibling() { - panic!("Could not find a child corresponding to any enum branch") + return Err(::rust_sitter::error::ExtractError::missing_enum(ctx)); } } } } } } - extern "C" { - fn tree_sitter_test() -> ::rust_sitter::tree_sitter::Language; - } - pub fn language() -> ::rust_sitter::tree_sitter::Language { - unsafe { tree_sitter_test() } - } - #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] - #[doc = "[`Language`]"] - #[doc = r" instance containing the parsed structured data."] - pub fn parse( - input: &str, - ) -> core::result::Result> { - ::rust_sitter::__private::parse::(input, language) + impl ::rust_sitter::rule::Rule for Expression { + const RULE_NAME: &'static str = "Expression"; + fn produce_ast() -> String { + String::new() + } } } - diff --git a/macro/src/snapshots/rust_sitter_macro__tests__spanned_in_vec.snap b/macro/src/snapshots/rust_sitter_macro__tests__spanned_in_vec.snap index 9efa021..7484aa3 100644 --- a/macro/src/snapshots/rust_sitter_macro__tests__spanned_in_vec.snap +++ b/macro/src/snapshots/rust_sitter_macro__tests__spanned_in_vec.snap @@ -1,102 +1,152 @@ --- source: macro/src/lib.rs -expression: "rustfmt_code(&expand_grammar(parse_quote! {\n #[rust_sitter :: grammar(\"test\")] mod grammar\n {\n use rust_sitter :: Spanned ; #[rust_sitter :: language] pub\n struct NumberList { numbers : Vec < Spanned < Number >>, }\n pub struct Number\n {\n #[rust_sitter ::\n leaf(pattern = r\"\\d+\", transform = | v |\n v.parse().unwrap())] v : i32\n } #[rust_sitter :: extra] struct Whitespace\n {\n #[rust_sitter :: leaf(pattern = r\"\\s\")] _whitespace : (),\n }\n }\n })?.to_token_stream().to_string())" +expression: "rustfmt_code(&expand_grammar(parse_quote!\n{\n mod grammar\n {\n use rust_sitter::{Rule, Spanned}; #[derive(Rule)] #[language] pub\n struct NumberList { numbers: Vec>, } #[derive(Rule)]\n pub struct Number { #[leaf(re(r\"\\d+\"))] v: i32 } #[derive(Rule)]\n #[extra] struct Whitespace\n { #[leaf(pattern(r\"\\s\"))] _whitespace: (), }\n }\n}).to_token_stream().to_string())" --- mod grammar { - use rust_sitter::Spanned; - pub struct NumberList { - numbers: Vec>, + use rust_sitter::{Rule, Spanned}; + impl ::rust_sitter::rule::Language for NumberList { + fn produce_grammar() -> String { + String::new() + } + fn language() -> ::rust_sitter::tree_sitter::Language { + unsafe extern "C" { + fn tree_sitter_NumberList() -> ::rust_sitter::tree_sitter::Language; + } + unsafe { tree_sitter_NumberList() } + } + #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] + #[doc = "[`NumberList`]"] + #[doc = r" instance containing the parsed structured data."] + fn parse(input: &str) -> ::rust_sitter::ParseResult { + ::rust_sitter::__private::parse(input, Self::language) + } } - impl ::rust_sitter::Extract for NumberList { + impl ::rust_sitter::Extract for NumberList { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); - ::rust_sitter::__private::extract_struct_or_variant(node, move |cursor, last_idx| { - NumberList { - numbers: { - ::rust_sitter::__private::extract_field::>, _>( - cursor, source, last_idx, "numbers", None, - ) - }, - } - }) + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; + ::rust_sitter::__private::extract_struct_or_variant( + stringify!(NumberList), + node, + move |state| { + Ok(NumberList { + numbers: ::rust_sitter::__private::extract_field::>, _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new(2u32, true, |state| { + match state { + 0u32 => ::rust_sitter::extract::ExtractFieldState::Str( + "Number", true, false, + ), + 1u32 => ::rust_sitter::extract::ExtractFieldState::Repeat1, + 2u32 => ::rust_sitter::extract::ExtractFieldState::Complete, + _ => ::rust_sitter::extract::ExtractFieldState::Overflow, + } + }), + source, + "numbers", + )?, + }) + }, + ) } } - pub struct Number { - v: i32, + impl ::rust_sitter::rule::Rule for NumberList { + const RULE_NAME: &'static str = "NumberList"; + fn produce_ast() -> String { + String::new() + } } - impl ::rust_sitter::Extract for Number { + impl ::rust_sitter::Extract for Number { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); - ::rust_sitter::__private::extract_struct_or_variant(node, move |cursor, last_idx| { - Number { - v: { - ::rust_sitter::__private::extract_field::, _>( - cursor, + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; + ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Number), + node, + move |state| { + Ok(Number { + v: ::rust_sitter::__private::extract_field::( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 0u32, + false, + |state| match state { + 0u32 => ::rust_sitter::extract::ExtractFieldState::Complete, + _ => ::rust_sitter::extract::ExtractFieldState::Overflow, + }, + ), source, - last_idx, "v", - Some(&|v| v.parse().unwrap()), - ) - }, - } - }) + )?, + }) + }, + ) } } - struct Whitespace { - _whitespace: (), + impl ::rust_sitter::rule::Rule for Number { + const RULE_NAME: &'static str = "Number"; + fn produce_ast() -> String { + String::new() + } } - impl ::rust_sitter::Extract for Whitespace { + impl ::rust_sitter::Extract for Whitespace { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); - ::rust_sitter::__private::extract_struct_or_variant(node, move |cursor, last_idx| { - Whitespace { - _whitespace: { - ::rust_sitter::__private::extract_field::<(), _>( - cursor, + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; + ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Whitespace), + node, + move |state| { + Ok(Whitespace { + _whitespace: ::rust_sitter::__private::extract_field::<(), _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 0u32, + false, + |state| match state { + 0u32 => ::rust_sitter::extract::ExtractFieldState::Complete, + _ => ::rust_sitter::extract::ExtractFieldState::Overflow, + }, + ), source, - last_idx, "_whitespace", - None, - ) - }, - } - }) + )?, + }) + }, + ) } } - extern "C" { - fn tree_sitter_test() -> ::rust_sitter::tree_sitter::Language; - } - pub fn language() -> ::rust_sitter::tree_sitter::Language { - unsafe { tree_sitter_test() } - } - #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] - #[doc = "[`NumberList`]"] - #[doc = r" instance containing the parsed structured data."] - pub fn parse( - input: &str, - ) -> core::result::Result> { - ::rust_sitter::__private::parse::(input, language) + impl ::rust_sitter::rule::Rule for Whitespace { + const RULE_NAME: &'static str = "Whitespace"; + fn produce_ast() -> String { + String::new() + } } } - diff --git a/macro/src/snapshots/rust_sitter_macro__tests__struct_extra.snap b/macro/src/snapshots/rust_sitter_macro__tests__struct_extra.snap index 07cea34..5dc70d9 100644 --- a/macro/src/snapshots/rust_sitter_macro__tests__struct_extra.snap +++ b/macro/src/snapshots/rust_sitter_macro__tests__struct_extra.snap @@ -1,94 +1,122 @@ --- source: macro/src/lib.rs -expression: "rustfmt_code(&expand_grammar(parse_quote! {\n #[rust_sitter :: grammar(\"test\")] mod grammar\n {\n #[rust_sitter :: language] pub enum Expression\n {\n Number(#[rust_sitter ::\n leaf(pattern = r\"\\d+\", transform = | v |\n v.parse().unwrap())] i32,),\n } #[rust_sitter :: extra] struct Whitespace\n {\n #[rust_sitter :: leaf(pattern = r\"\\s\")] _whitespace : (),\n }\n }\n })?.to_token_stream().to_string())" +expression: "rustfmt_code(&expand_grammar(parse_quote!\n{\n mod grammar\n {\n #[derive(rust_sitter::Rule)] #[language] pub enum Expression\n { Number(#[leaf(re(r\"\\d+\"))] i32,), } #[derive(Rule)] #[extra] struct\n Whitespace { #[leaf(pattern(r\"\\s\"))] _whitespace: (), }\n }\n}).to_token_stream().to_string())" --- mod grammar { - pub enum Expression { - Number(i32), + impl ::rust_sitter::rule::Language for Expression { + fn produce_grammar() -> String { + String::new() + } + fn language() -> ::rust_sitter::tree_sitter::Language { + unsafe extern "C" { + fn tree_sitter_Expression() -> ::rust_sitter::tree_sitter::Language; + } + unsafe { tree_sitter_Expression() } + } + #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] + #[doc = "[`Expression`]"] + #[doc = r" instance containing the parsed structured data."] + fn parse(input: &str) -> ::rust_sitter::ParseResult { + ::rust_sitter::__private::parse(input, Self::language) + } } - impl ::rust_sitter::Extract for Expression { + impl ::rust_sitter::Extract for Expression { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - _last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; let mut cursor = node.walk(); - assert!( - cursor.goto_first_child(), - "Could not find a child corresponding to any enum branch" - ); + if !cursor.goto_first_child() { + return Err(::rust_sitter::error::ExtractError::missing_node(ctx)); + } loop { let node = cursor.node(); match node.kind() { "Expression_Number" => { return ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Expression::Number), node, - move |cursor, last_idx| { - Expression::Number({ - ::rust_sitter::__private::extract_field::< - rust_sitter::WithLeaf, - _, - >( - cursor, source, last_idx, "0", Some(&|v| v.parse().unwrap()) - ) - }) + move |state| { + Ok(Expression::Number( + ::rust_sitter::__private::extract_field::( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 0u32, + false, + |state| { + match state { 0u32 => :: rust_sitter :: extract :: ExtractFieldState :: Complete , _ => :: rust_sitter :: extract :: ExtractFieldState :: Overflow , } + }, + ), + source, + "0", + )?, + )) }, ) } - _ => { + k => { if !cursor.goto_next_sibling() { - panic!("Could not find a child corresponding to any enum branch") + return Err(::rust_sitter::error::ExtractError::missing_enum(ctx)); } } } } } } - struct Whitespace { - _whitespace: (), + impl ::rust_sitter::rule::Rule for Expression { + const RULE_NAME: &'static str = "Expression"; + fn produce_ast() -> String { + String::new() + } } - impl ::rust_sitter::Extract for Whitespace { + impl ::rust_sitter::Extract for Whitespace { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); - ::rust_sitter::__private::extract_struct_or_variant(node, move |cursor, last_idx| { - Whitespace { - _whitespace: { - ::rust_sitter::__private::extract_field::<(), _>( - cursor, + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; + ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Whitespace), + node, + move |state| { + Ok(Whitespace { + _whitespace: ::rust_sitter::__private::extract_field::<(), _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 0u32, + false, + |state| match state { + 0u32 => ::rust_sitter::extract::ExtractFieldState::Complete, + _ => ::rust_sitter::extract::ExtractFieldState::Overflow, + }, + ), source, - last_idx, "_whitespace", - None, - ) - }, - } - }) + )?, + }) + }, + ) } } - extern "C" { - fn tree_sitter_test() -> ::rust_sitter::tree_sitter::Language; - } - pub fn language() -> ::rust_sitter::tree_sitter::Language { - unsafe { tree_sitter_test() } - } - #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] - #[doc = "[`Expression`]"] - #[doc = r" instance containing the parsed structured data."] - pub fn parse( - input: &str, - ) -> core::result::Result> { - ::rust_sitter::__private::parse::(input, language) + impl ::rust_sitter::rule::Rule for Whitespace { + const RULE_NAME: &'static str = "Whitespace"; + fn produce_ast() -> String { + String::new() + } } } - diff --git a/macro/src/snapshots/rust_sitter_macro__tests__struct_optional.snap b/macro/src/snapshots/rust_sitter_macro__tests__struct_optional.snap index 16c8d23..07117a4 100644 --- a/macro/src/snapshots/rust_sitter_macro__tests__struct_optional.snap +++ b/macro/src/snapshots/rust_sitter_macro__tests__struct_optional.snap @@ -1,82 +1,121 @@ --- source: macro/src/lib.rs -expression: "rustfmt_code(&expand_grammar(parse_quote! {\n #[rust_sitter :: grammar(\"test\")] mod grammar\n {\n #[rust_sitter :: language] pub struct Language\n {\n #[rust_sitter ::\n leaf(pattern = r\"\\d+\", transform = | v |\n v.parse().unwrap())] v : Option < i32 >, t : Option < Number\n >,\n } pub struct Number\n {\n #[rust_sitter ::\n leaf(pattern = r\"\\d+\", transform = | v |\n v.parse().unwrap())] v : i32\n }\n }\n })?.to_token_stream().to_string())" +expression: "rustfmt_code(&expand_grammar(parse_quote!\n{\n mod grammar\n {\n #[derive(rust_sitter::Rule)] #[language] pub struct Language\n { #[leaf(re(r\"\\d+\"))] v: Option, t: Option, }\n #[derive(rust_sitter::Rule)] pub struct Number\n { #[leaf(re(r\"\\d+\"))] v: i32 }\n }\n}).to_token_stream().to_string())" --- mod grammar { - pub struct Language { - v: Option, - t: Option, + impl ::rust_sitter::rule::Language for Language { + fn produce_grammar() -> String { + String::new() + } + fn language() -> ::rust_sitter::tree_sitter::Language { + unsafe extern "C" { + fn tree_sitter_Language() -> ::rust_sitter::tree_sitter::Language; + } + unsafe { tree_sitter_Language() } + } + #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] + #[doc = "[`Language`]"] + #[doc = r" instance containing the parsed structured data."] + fn parse(input: &str) -> ::rust_sitter::ParseResult { + ::rust_sitter::__private::parse(input, Self::language) + } } - impl ::rust_sitter::Extract for Language { + impl ::rust_sitter::Extract for Language { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); - ::rust_sitter::__private::extract_struct_or_variant(node, move |cursor, last_idx| { - Language { - v: { - ::rust_sitter::__private::extract_field::< - Option>, - _, - >( - cursor, source, last_idx, "v", Some(&|v| v.parse().unwrap()) - ) - }, - t: { - ::rust_sitter::__private::extract_field::, _>( - cursor, source, last_idx, "t", None, - ) - }, - } - }) + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; + ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Language), + node, + move |state| { + Ok(Language { + v: ::rust_sitter::__private::extract_field::, _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new(0u32, true, |state| { + match state { + 0u32 => ::rust_sitter::extract::ExtractFieldState::Complete, + _ => ::rust_sitter::extract::ExtractFieldState::Overflow, + } + }), + source, + "v", + )?, + t: ::rust_sitter::__private::extract_field::, _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new(1u32, true, |state| { + match state { + 0u32 => ::rust_sitter::extract::ExtractFieldState::Str( + "Number", true, false, + ), + 1u32 => ::rust_sitter::extract::ExtractFieldState::Complete, + _ => ::rust_sitter::extract::ExtractFieldState::Overflow, + } + }), + source, + "t", + )?, + }) + }, + ) } } - pub struct Number { - v: i32, + impl ::rust_sitter::rule::Rule for Language { + const RULE_NAME: &'static str = "Language"; + fn produce_ast() -> String { + String::new() + } } - impl ::rust_sitter::Extract for Number { + impl ::rust_sitter::Extract for Number { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); - ::rust_sitter::__private::extract_struct_or_variant(node, move |cursor, last_idx| { - Number { - v: { - ::rust_sitter::__private::extract_field::, _>( - cursor, + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; + ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Number), + node, + move |state| { + Ok(Number { + v: ::rust_sitter::__private::extract_field::( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 0u32, + false, + |state| match state { + 0u32 => ::rust_sitter::extract::ExtractFieldState::Complete, + _ => ::rust_sitter::extract::ExtractFieldState::Overflow, + }, + ), source, - last_idx, "v", - Some(&|v| v.parse().unwrap()), - ) - }, - } - }) + )?, + }) + }, + ) } } - extern "C" { - fn tree_sitter_test() -> ::rust_sitter::tree_sitter::Language; - } - pub fn language() -> ::rust_sitter::tree_sitter::Language { - unsafe { tree_sitter_test() } - } - #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] - #[doc = "[`Language`]"] - #[doc = r" instance containing the parsed structured data."] - pub fn parse( - input: &str, - ) -> core::result::Result> { - ::rust_sitter::__private::parse::(input, language) + impl ::rust_sitter::rule::Rule for Number { + const RULE_NAME: &'static str = "Number"; + fn produce_ast() -> String { + String::new() + } } } - diff --git a/macro/src/snapshots/rust_sitter_macro__tests__struct_repeat.snap b/macro/src/snapshots/rust_sitter_macro__tests__struct_repeat.snap index 16728fa..0113824 100644 --- a/macro/src/snapshots/rust_sitter_macro__tests__struct_repeat.snap +++ b/macro/src/snapshots/rust_sitter_macro__tests__struct_repeat.snap @@ -1,101 +1,151 @@ --- source: macro/src/lib.rs -expression: "rustfmt_code(&expand_grammar(parse_quote! {\n #[rust_sitter :: grammar(\"test\")] mod grammar\n {\n #[rust_sitter :: language] pub struct NumberList\n { numbers : Vec < Number >, } pub struct Number\n {\n #[rust_sitter ::\n leaf(pattern = r\"\\d+\", transform = | v |\n v.parse().unwrap())] v : i32\n } #[rust_sitter :: extra] struct Whitespace\n {\n #[rust_sitter :: leaf(pattern = r\"\\s\")] _whitespace : (),\n }\n }\n })?.to_token_stream().to_string())" +expression: "rustfmt_code(&expand_grammar(parse_quote!\n{\n mod grammar\n {\n #[derive(rust_sitter::Rule)] #[language] pub struct NumberList\n { numbers: Vec, } #[derive(rust_sitter::Rule)] pub struct\n Number { #[leaf(re(r\"\\d+\"))] v: i32 } #[derive(rust_sitter::Rule)]\n #[extra] struct Whitespace\n { #[leaf(pattern(r\"\\s\"))] _whitespace: (), }\n }\n}).to_token_stream().to_string())" --- mod grammar { - pub struct NumberList { - numbers: Vec, + impl ::rust_sitter::rule::Language for NumberList { + fn produce_grammar() -> String { + String::new() + } + fn language() -> ::rust_sitter::tree_sitter::Language { + unsafe extern "C" { + fn tree_sitter_NumberList() -> ::rust_sitter::tree_sitter::Language; + } + unsafe { tree_sitter_NumberList() } + } + #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] + #[doc = "[`NumberList`]"] + #[doc = r" instance containing the parsed structured data."] + fn parse(input: &str) -> ::rust_sitter::ParseResult { + ::rust_sitter::__private::parse(input, Self::language) + } } - impl ::rust_sitter::Extract for NumberList { + impl ::rust_sitter::Extract for NumberList { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); - ::rust_sitter::__private::extract_struct_or_variant(node, move |cursor, last_idx| { - NumberList { - numbers: { - ::rust_sitter::__private::extract_field::, _>( - cursor, source, last_idx, "numbers", None, - ) - }, - } - }) + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; + ::rust_sitter::__private::extract_struct_or_variant( + stringify!(NumberList), + node, + move |state| { + Ok(NumberList { + numbers: ::rust_sitter::__private::extract_field::, _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new(2u32, true, |state| { + match state { + 0u32 => ::rust_sitter::extract::ExtractFieldState::Str( + "Number", true, false, + ), + 1u32 => ::rust_sitter::extract::ExtractFieldState::Repeat1, + 2u32 => ::rust_sitter::extract::ExtractFieldState::Complete, + _ => ::rust_sitter::extract::ExtractFieldState::Overflow, + } + }), + source, + "numbers", + )?, + }) + }, + ) } } - pub struct Number { - v: i32, + impl ::rust_sitter::rule::Rule for NumberList { + const RULE_NAME: &'static str = "NumberList"; + fn produce_ast() -> String { + String::new() + } } - impl ::rust_sitter::Extract for Number { + impl ::rust_sitter::Extract for Number { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); - ::rust_sitter::__private::extract_struct_or_variant(node, move |cursor, last_idx| { - Number { - v: { - ::rust_sitter::__private::extract_field::, _>( - cursor, + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; + ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Number), + node, + move |state| { + Ok(Number { + v: ::rust_sitter::__private::extract_field::( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 0u32, + false, + |state| match state { + 0u32 => ::rust_sitter::extract::ExtractFieldState::Complete, + _ => ::rust_sitter::extract::ExtractFieldState::Overflow, + }, + ), source, - last_idx, "v", - Some(&|v| v.parse().unwrap()), - ) - }, - } - }) + )?, + }) + }, + ) } } - struct Whitespace { - _whitespace: (), + impl ::rust_sitter::rule::Rule for Number { + const RULE_NAME: &'static str = "Number"; + fn produce_ast() -> String { + String::new() + } } - impl ::rust_sitter::Extract for Whitespace { + impl ::rust_sitter::Extract for Whitespace { + type Output = Self; type LeafFn = (); #[allow(non_snake_case)] - fn extract( - node: Option<::rust_sitter::tree_sitter::Node>, + fn extract<'tree>( + ctx: &mut ::rust_sitter::extract::ExtractContext, + node: Option<::rust_sitter::tree_sitter::Node<'tree>>, source: &[u8], - last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) -> Self { - let node = node.unwrap(); - ::rust_sitter::__private::extract_struct_or_variant(node, move |cursor, last_idx| { - Whitespace { - _whitespace: { - ::rust_sitter::__private::extract_field::<(), _>( - cursor, + _l: Self::LeafFn, + ) -> Result> { + let node = node.ok_or_else(|| ::rust_sitter::error::ExtractError::missing_node(ctx))?; + ::rust_sitter::__private::extract_struct_or_variant( + stringify!(Whitespace), + node, + move |state| { + Ok(Whitespace { + _whitespace: ::rust_sitter::__private::extract_field::<(), _>( + ::rust_sitter::extract::BaseExtractor::default(), + (), + state, + ::rust_sitter::extract::ExtractFieldContext::new( + 0u32, + false, + |state| match state { + 0u32 => ::rust_sitter::extract::ExtractFieldState::Complete, + _ => ::rust_sitter::extract::ExtractFieldState::Overflow, + }, + ), source, - last_idx, "_whitespace", - None, - ) - }, - } - }) + )?, + }) + }, + ) } } - extern "C" { - fn tree_sitter_test() -> ::rust_sitter::tree_sitter::Language; - } - pub fn language() -> ::rust_sitter::tree_sitter::Language { - unsafe { tree_sitter_test() } - } - #[doc = r" Parse an input string according to the grammar. Returns either any parsing errors that happened, or a"] - #[doc = "[`NumberList`]"] - #[doc = r" instance containing the parsed structured data."] - pub fn parse( - input: &str, - ) -> core::result::Result> { - ::rust_sitter::__private::parse::(input, language) + impl ::rust_sitter::rule::Rule for Whitespace { + const RULE_NAME: &'static str = "Whitespace"; + fn produce_ast() -> String { + String::new() + } } } - diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 718e312..ac43368 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -3,9 +3,9 @@ name = "rust-sitter" description = "A package for defining tree-sitter grammars alongside Rust logic" readme = "../README.md" repository = "https://github.com/hydro-project/rust-sitter" -version = "0.4.5" -authors = ["Shadaj Laddad "] -edition = "2021" +version.workspace = true +authors.workspace = true +edition = "2024" license = "MIT" keywords = ["parsing", "codegen"] categories = ["development-tools"] @@ -13,15 +13,14 @@ categories = ["development-tools"] [lib] path = "src/lib.rs" -[features] -default = ["tree-sitter-c2rust"] -tree-sitter-c2rust = ["tree-sitter-runtime-c2rust"] -tree-sitter-standard = ["tree-sitter-runtime-standard"] - [dependencies] -tree-sitter-runtime-c2rust = { package = "tree-sitter-c2rust", version = "0.25.2", optional = true } -tree-sitter-runtime-standard = { package = "tree-sitter", version = "0.25.2", optional = true } -rust-sitter-macro = { version = "0.4.5", path = "../macro" } +tree-sitter.workspace = true +rust-sitter-macro = { path = "../macro" } +rust-sitter-types = { path = "../types" } +log = "0.4" +# This one could be optional. +serde_json = "1" +serde = { version = "1", features = ["derive"] } [dev-dependencies] insta = "1.39" diff --git a/runtime/src/__private.rs b/runtime/src/__private.rs index 5858c52..c880220 100644 --- a/runtime/src/__private.rs +++ b/runtime/src/__private.rs @@ -4,80 +4,166 @@ //! They need to be public so they can be accessed at all (\*cough\* macro hygiene), but //! they are not intended to actually be called in any other circumstance. -use crate::{tree_sitter, Extract}; +use crate::{ + Extract, ExtractContext, Extractor, + extract::{ExtractFieldContext, ExtractFieldIterator, Result}, +}; +use log::trace; -pub fn extract_struct_or_variant( - node: tree_sitter::Node, - construct_expr: impl Fn(&mut Option, &mut usize) -> T, -) -> T { +pub fn extract_struct_or_variant<'tree, T>( + struct_name: &'static str, + node: tree_sitter::Node<'tree>, + construct_expr: impl for<'t> Fn(&mut ExtractStructState<'t>) -> Result<'t, T>, +) -> Result<'tree, T> { + trace!("extract_struct_or_variant node.kind={}", node.kind()); + trace!("extract_struct_or_variant node={}", node); + trace!( + "extract_struct_or_variant node.child_count={}", + node.child_count() + ); let mut parent_cursor = node.walk(); - construct_expr( - &mut if parent_cursor.goto_first_child() { - Some(parent_cursor) - } else { - None - }, - &mut node.start_byte(), - ) + let has_children = parent_cursor.goto_first_child(); + let mut state = ExtractStructState { + struct_name, + cursor: Some(parent_cursor), + has_children, + last_idx: node.start_byte(), + last_pt: node.start_position(), + // error: ExtractError::empty(), + }; + construct_expr(&mut state) +} + +pub struct ExtractStructState<'tree> { + struct_name: &'static str, + cursor: Option>, + has_children: bool, + last_idx: usize, + last_pt: tree_sitter::Point, + // TODO: Use this. + // error: ExtractError, } -pub fn extract_field, T>( - cursor_opt: &mut Option, +pub fn extract_field<'tree, T: Extract, E: Extractor>( + extractor: E, + leaf_fn: T::LeafFn, + state: &mut ExtractStructState<'tree>, + field_state: ExtractFieldContext, source: &[u8], - last_idx: &mut usize, - field_name: &str, - closure_ref: Option<<::LeafFn>, -) -> T { - if let Some(cursor) = cursor_opt.as_mut() { + field_name: &'static str, +) -> Result<'tree, T::Output> { + trace!( + "extract_field struct_name={} field_name={field_name}", + state.struct_name + ); + let mut ctx = ExtractContext { + last_idx: state.last_idx, + last_pt: state.last_pt, + field_name, + struct_name: state.struct_name, + }; + if state.has_children { + if let Some(cursor) = state.cursor.as_mut() { + trace!("extract_field has_children: {}", cursor.node()); + let mut iter = ExtractFieldIterator::new( + field_state, + cursor, + state.struct_name, + field_name, + source, + ); + + // Start the iterator. + // Iteration requires knowing if there is a valid starting state or not. + iter.advance_state()?; + + let result = extractor.do_extract_field(&mut ctx, &mut iter, source, leaf_fn)?; + iter.finalize()?; + Ok(result) + } else { + extractor.do_extract(&mut ctx, None, source, leaf_fn) + } + } else if let Some(cursor) = state.cursor.as_mut() { + let n = cursor.node(); + if !cursor.goto_next_sibling() { + state.cursor = None; + } + extractor.do_extract(&mut ctx, Some(n), source, leaf_fn) + } else { + extractor.do_extract(&mut ctx, None, source, leaf_fn) + } +} + +// TODO: Handle errors in this one too. +pub fn skip_text<'tree>( + state: &mut ExtractStructState<'tree>, + field_name: &'static str, +) -> Result<'tree, ()> { + trace!( + "skip field: {field_name:?}, has cursor: {}", + state.cursor.is_some() + ); + if let Some(cursor) = state.cursor.as_mut() { + trace!( + "skip field: expects: {field_name:?}, has: {:?}", + cursor.field_name() + ); loop { - let n = cursor.node(); + if cursor.node().is_extra() { + if !cursor.goto_next_sibling() { + state.cursor = None; + return Ok(()); + } + continue; + } if let Some(name) = cursor.field_name() { if name == field_name { - let out = LT::extract(Some(n), source, *last_idx, closure_ref); - if !cursor.goto_next_sibling() { - *cursor_opt = None; - }; - - *last_idx = n.end_byte(); - - return out; + state.cursor = None; + return Ok(()); + } } else { - return LT::extract(None, source, *last_idx, closure_ref); + return Ok(()); } } else { - *last_idx = n.end_byte(); - } - - if !cursor.goto_next_sibling() { - return LT::extract(None, source, *last_idx, closure_ref); + return Ok(()); } } - } else { - LT::extract(None, source, *last_idx, closure_ref) } + + Ok(()) } -pub fn parse>( +pub fn parse( input: &str, language: impl Fn() -> tree_sitter::Language, -) -> core::result::Result> { - let mut parser = crate::tree_sitter::Parser::new(); +) -> crate::ParseResult { + let mut parser = tree_sitter::Parser::new(); parser.set_language(&language()).unwrap(); - let tree = parser.parse(input, None).unwrap(); + if matches!(std::env::var("RUST_SITTER_PARSER_LOG").as_deref(), Ok("1")) { + parser.set_logger(Some(Box::new(|_t, m| log::debug!("parser::{m}")))); + } + let tree = parser.parse(input, None).expect("Failed to parse"); let root_node = tree.root_node(); + let mut errors = vec![]; if root_node.has_error() { - let mut errors = vec![]; - crate::errors::collect_parsing_errors(&root_node, input.as_bytes(), &mut errors); - - Err(errors) - } else { - Ok(>::extract( - Some(root_node), - input.as_bytes(), - 0, - None, - )) + crate::error::collect_parsing_errors(&root_node, &mut errors); } + let mut ctx = ExtractContext { + last_pt: Default::default(), + last_idx: 0, + field_name: "root", + struct_name: T::rule_name(), + }; + let result = ::extract(&mut ctx, Some(root_node), input.as_bytes(), ()); + let result = match result { + Err(e) => { + // These are actually not really useful yet. + e.accumulate_parse_errors(&mut errors); + None + } + Ok(o) => Some(o), + }; + crate::ParseResult { result, errors } } diff --git a/runtime/src/error.rs b/runtime/src/error.rs new file mode 100644 index 0000000..33db2cd --- /dev/null +++ b/runtime/src/error.rs @@ -0,0 +1,488 @@ +use log::trace; +use std::{collections::HashSet, ops::Range}; + +use crate::{ExtractContext, Point, Position, extract::ExtractFieldIterator}; + +/// A high level parsing error with useful information extracted already. +#[derive(Debug)] +pub struct ParseError { + /// Position within the source code of the full node which failed to parse. + /// This can be used in combination with `error_position` to indicate a greater context of where + /// an error occurred. + pub node_position: Position, + pub error_position: Position, + /// Possible next tokens that were expected. + pub lookaheads: Vec<&'static str>, + pub reason: ParseErrorReason, +} + +#[derive(Debug)] +pub enum ParseErrorReason { + Missing(&'static str), + Error, + Extract { + struct_name: &'static str, + field_name: &'static str, + reason: ExtractErrorReason, + }, +} + +impl ParseError { + pub fn is_missing(&self) -> bool { + matches!(&self.reason, ParseErrorReason::Missing(_)) + } +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}:{} to {}:{}, {}", + self.error_position.start.line, + self.error_position.start.column, + self.error_position.end.line, + self.error_position.end.column, + self.reason + ) + } +} + +impl std::fmt::Display for ParseErrorReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ParseErrorReason::Missing(kind) => write!(f, "missing {kind}"), + ParseErrorReason::Error => f.write_str("parse error"), + // ParseErrorReason::FailedExtract { field } => { + // write!(f, "failed extraction of field: {field}") + // } + ParseErrorReason::Extract { + struct_name, + field_name, + reason, + } => { + write!( + f, + "extraction error for {struct_name}::{field_name}. Reason: {reason}" + ) + } + } + } +} + +/// A low level error which just wraps the error node and exposes many fields around it. +#[derive(Debug)] +pub struct NodeError<'a> { + node: tree_sitter::Node<'a>, +} + +impl<'a> NodeError<'a> { + pub fn to_parse_error(&self) -> ParseError { + // Handle missing shift. + let mut node_position = Position::new(self.node_byte_range(), self.point_range()); + let mut error_position = Position::new( + self.first_error_byte_range(), + self.first_error_point_range(), + ); + trace!("error node: {}", self.node); + trace!("error node: {:?}", self.node); + trace!("error node parent: {:?}", self.node.parent()); + if self.node.is_missing() + && let Some(parent) = self.node.parent() + { + trace!("attempting missing shift: {}", parent.to_sexp()); + // Find where the missing node is located in the parent, then shift it backwards by + // removing any extra nodes in its place. + // let mut c = parent.walk(); + // let idx = parent.children(&mut c) + // // defers to pointer equality, which is what we want in this case. + // .position(|n| n == self.node) + // .unwrap(); + // c.reset(self.node); + // Doesn't work, the cursor iterator doesn't work correctly. + // dbg!(self.node.prev_sibling()); + // while dbg!(c.goto_previous_sibling()) && c.node().is_extra() { + // debug!("shifting past extra: {}", c.node()); + // } + // Use parent node for node_position and this node for error_position. + let mut node = self.node; + let mut has_shifted = false; + while let Some(n) = node.prev_sibling() { + node = n; + if !has_shifted { + has_shifted = node.is_extra(); + } + trace!("shifting past extra: {}", n); + if !node.is_extra() { + break; + } + } + + if has_shifted { + trace!("shifted to node: {}", node.kind()); + let range = node.byte_range(); + let range = range.end..range.end; + let new_err = Position::new( + range, + (node.start_position().into(), node.end_position().into()), + ); + let new_pos = Position::new( + parent.byte_range(), + (parent.start_position().into(), parent.end_position().into()), + ); + trace!("shifted position from {error_position:?} to {new_pos:?}"); + error_position = new_err; + node_position = new_pos; + } + } + ParseError { + node_position, + error_position, + lookaheads: self.lookahead().map(|l| l.collect()).unwrap_or_default(), + reason: if self.node.is_missing() { + ParseErrorReason::Missing(self.node.kind()) + } else { + ParseErrorReason::Error + }, + } + } + /// Full range of the node which failed to parse. + pub fn node_byte_range(&self) -> Range { + self.node.byte_range() + } + + /// Byte range of the portion of the text which created the error. + pub fn error_byte_range(&self) -> Range { + self.node.error_byte_range().unwrap() + } + + pub fn point_range(&self) -> (Point, Point) { + let start = self.node.start_position(); + let end = self.node.end_position(); + (Point::from_tree_sitter(start), Point::from_tree_sitter(end)) + } + + pub fn error_point_range(&self) -> (Point, Point) { + let start = self.node.error_start_position().unwrap(); + let end = self.node.error_end_position().unwrap(); + (Point::from_tree_sitter(start), Point::from_tree_sitter(end)) + } + + pub fn first_error_point_range(&self) -> (Point, Point) { + match self.node.error_child(0) { + None => self.error_point_range(), + Some(c) => { + let start = c.start_position(); + let end = c.end_position(); + (Point::from_tree_sitter(start), Point::from_tree_sitter(end)) + } + } + } + + pub fn first_error_byte_range(&self) -> Range { + match self.node.error_child(0) { + None => self.error_byte_range(), + Some(c) => c.byte_range(), + } + } + + pub fn is_missing(&self) -> bool { + self.node.is_missing() + } + + pub fn lookahead( + &self, + // grammar: Option<&'a crate::grammar::Grammar>, + ) -> Option> { + let (state, reachable, filter) = if self.node.is_missing() { + // Handle the lookahead appropriately for missing. + let state = self.node.parse_state(); + (state, None, true) + } else { + // Find the endpoint. + // let (node, ctx) = match self.node.error_child(0) { + // Some(c) => (c, self.node.child(0).unwrap()), + // None => (self.node, self.node), + // }; + let node = match self.node.error_child(0) { + Some(c) => c, + None => self.node, + }; + + // Find the first context node type and compute reachable set. + // let reachable = if let Some(grammar) = grammar { + // dbg!(grammar.reachable_set(dbg!(ctx.kind()))) + // } else { + // None + // }; + let reachable = None; + + let state = node.parse_state(); + // NOTE: We may want to always filter these. + (state, reachable, false) + }; + + if state == 0 { + return None; + } + + let language = self.node.language().to_owned(); + let it = language.lookahead_iterator(state)?; + + Some(ErrorLookahead { + it, + language, + filter_non_action: filter, + state, + reachable, + }) + } +} + +struct ErrorLookahead<'a> { + it: tree_sitter::LookaheadIterator, + language: tree_sitter::Language, + filter_non_action: bool, + state: u16, + reachable: Option>, +} + +impl Iterator for ErrorLookahead<'_> { + type Item = &'static str; + fn next(&mut self) -> Option { + loop { + self.it.next()?; + let sym = self.it.current_symbol(); + // skip the end symbol, it isn't useful here. + if sym == 0 { + continue; + } + if self.filter_non_action && !self.it.has_actions() { + continue; + } + // Maybe we want this to be optional as well? + // Filter out "extra" nodes. + if self.state == self.language.next_state(self.state, sym) { + continue; + } + + let sym_name = self.it.current_symbol_name(); + + if let Some(reachable) = &self.reachable + && !reachable.contains(sym_name) + { + continue; + } + + return Some(sym_name); + } + } +} + +#[derive(Debug)] +pub struct ExtractError<'a> { + inner: Vec>, +} + +#[derive(Debug)] +struct ExtractErrorInner<'a> { + /// Span of the node which failed to extract. + position: crate::Position, + field_name: &'static str, + struct_name: &'static str, + node: Option>, + reason: ExtractErrorReason, +} + +impl<'a> ExtractError<'a> { + pub(crate) fn empty() -> Self { + Self { inner: vec![] } + } + + pub(crate) fn prop(self) -> Result<(), Self> { + if self.inner.is_empty() { + Ok(()) + } else { + Err(self) + } + } + + pub(crate) fn new( + struct_name: &'static str, + field_name: &'static str, + position: crate::Position, + reason: ExtractErrorReason, + ) -> Self { + Self { + inner: vec![ExtractErrorInner { + // TODO: Provide this where possible. + node: None, + position, + field_name, + struct_name, + reason, + }], + } + } + + pub(crate) fn new_ctx( + ctx: &ExtractContext, + position: crate::Position, + reason: ExtractErrorReason, + ) -> Self { + Self::new(ctx.struct_name, ctx.field_name, position, reason) + } + + pub(crate) fn merge(&mut self, err: ExtractError<'a>) { + self.inner.extend(err.inner); + } + + pub(crate) fn type_conversion( + ctx: &ExtractContext, + n: tree_sitter::Node<'_>, + e: impl std::error::Error + Send + Sync + 'static, + ) -> Self { + let position = crate::Position::from_node(n); + Self::new( + ctx.struct_name, + ctx.field_name, + position, + ExtractErrorReason::TypeConversion(Box::new(e)), + ) + } + + pub(crate) fn field_extraction( + ctx: &ExtractFieldIterator<'_, '_>, + msg: impl Into, + ) -> Self { + let msg = msg.into(); + log::error!( + "field_extraction error: {}::{}, msg={}", + ctx.struct_name, + ctx.field_name, + msg + ); + let position = ctx.position(); + Self::new( + ctx.struct_name, + ctx.field_name, + position, + ExtractErrorReason::FieldExtraction { message: msg }, + ) + } + + #[allow(dead_code)] + pub(crate) fn accumulate_parse_errors(self, errors: &mut Vec) { + for inner in self.inner { + let err = ParseError { + node_position: inner.position.clone(), + error_position: inner.position, + lookaheads: vec![], + reason: ParseErrorReason::Extract { + struct_name: inner.struct_name, + field_name: inner.field_name, + reason: inner.reason, + }, + }; + errors.push(err); + } + } + + pub fn missing_node(ctx: &ExtractContext) -> Self { + let position = crate::Position { + // TODO: This should be fixed to actually have the full range from the outer node. + bytes: ctx.last_idx..ctx.last_idx, + start: Point::from_tree_sitter(ctx.last_pt), + end: Point::from_tree_sitter(ctx.last_pt), + }; + Self::new_ctx(ctx, position, ExtractErrorReason::MissingNode) + } + + pub fn missing_enum(ctx: &ExtractContext) -> Self { + let position = crate::Position { + // TODO: This should be fixed to actually have the full range from the outer node. + bytes: ctx.last_idx..ctx.last_idx, + start: Point::from_tree_sitter(ctx.last_pt), + end: Point::from_tree_sitter(ctx.last_pt), + }; + Self::new_ctx(ctx, position, ExtractErrorReason::MissingEnum) + } + + pub fn position(&self) -> &Position { + &self.inner[0].position + } + + pub fn reason(&self) -> &ExtractErrorReason { + &self.inner[0].reason + } +} + +#[derive(Debug)] +pub enum ExtractErrorReason { + FieldExtraction { + message: String, + }, + MissingNode, + MissingEnum, + /// Parsed OK, but failed to extract to the given type. + TypeConversion(Box), +} + +impl std::fmt::Display for ExtractErrorReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::MissingNode => write!(f, "missing node in extraction"), + Self::MissingEnum => write!(f, "missing enum in extraction",), + Self::FieldExtraction { message } => write!(f, "field extraction failure: {message}"), + Self::TypeConversion(error) => write!(f, "type conversion: {error}"), + } + } +} + +impl<'a> IntoIterator for ExtractError<'a> { + type Item = ExtractError<'a>; + type IntoIter = ErrorIntoIter<'a>; + fn into_iter(self) -> Self::IntoIter { + ErrorIntoIter { + iter: self.inner.into_iter(), + } + } +} + +pub struct ErrorIntoIter<'a> { + iter: std::vec::IntoIter>, +} + +impl<'a> Iterator for ErrorIntoIter<'a> { + type Item = ExtractError<'a>; + fn next(&mut self) -> Option { + Some(ExtractError { + inner: vec![self.iter.next()?], + }) + } +} +/// Given the root node of a Tree Sitter parsing result, accumulates all +/// errors that were emitted. +pub fn collect_parsing_errors(node: &tree_sitter::Node<'_>, errors: &mut Vec) { + collect_node_errors(*node, |err| errors.push(err.to_parse_error())); +} + +pub fn collect_node_errors<'a, F>(node: tree_sitter::Node<'a>, mut f: F) +where + F: FnMut(NodeError<'a>), +{ + collect_node_errors_(node, &mut f); + // I couldn't figure out how to get this to compile well. + fn collect_node_errors_<'a, F>(node: tree_sitter::Node<'a>, f: &mut F) + where + F: FnMut(NodeError<'a>), + { + if node.is_error() || node.is_missing() { + f(NodeError { node }); + } else if node.has_error() { + // A node somewhere down in the tree from here has an error, recursively find it. + let mut cursor = node.walk(); + node.children(&mut cursor) + .for_each(|c| collect_node_errors_(c, f)); + } + } +} diff --git a/runtime/src/extract.rs b/runtime/src/extract.rs new file mode 100644 index 0000000..cac4e94 --- /dev/null +++ b/runtime/src/extract.rs @@ -0,0 +1,365 @@ +use super::Node; +pub mod field; +pub use crate::error::ExtractError; +pub use field::{ExtractFieldContext, ExtractFieldIterator, ExtractFieldState}; + +pub type Result<'a, T> = std::result::Result>; + +/// Structs which can perform extractions. This allows an extractor to carry additional state +/// around the extraction (see for example, `WithLeafExtractor`). +pub trait Extractor { + fn do_extract<'tree>( + self, + ctx: &mut ExtractContext, + node: Option>, + source: &[u8], + leaf_fn: E::LeafFn, + ) -> Result<'tree, E::Output>; + + fn do_extract_field<'cursor, 'tree>( + self, + ctx: &mut ExtractContext, + it: &mut ExtractFieldIterator<'cursor, 'tree>, + source: &[u8], + leaf_fn: E::LeafFn, + ) -> Result<'tree, E::Output>; + + fn map(self, next: F) -> MapExtractor + where + F: FnOnce(E) -> O, + Self: Sized, + { + MapExtractor::new(self, next) + } +} + +/// Defines the logic used to convert a node in a Tree Sitter tree to +/// the corresponding Rust type. +pub trait Extract: Sized { + type LeafFn; + type Output; + fn extract<'tree>( + ctx: &mut ExtractContext, + node: Option>, + source: &[u8], + leaf_fn: Self::LeafFn, + ) -> Result<'tree, Self::Output>; + + fn extract_field<'cursor, 'tree>( + ctx: &mut ExtractContext, + it: &mut ExtractFieldIterator<'cursor, 'tree>, + source: &[u8], + leaf_fn: Self::LeafFn, + ) -> Result<'tree, Self::Output> { + let node = it.next_node()?; + Self::extract(ctx, node, source, leaf_fn) + } +} + +pub struct ExtractContext { + pub last_idx: usize, + pub last_pt: tree_sitter::Point, + pub field_name: &'static str, + pub struct_name: &'static str, +} + +/// Default extractor which simply delegates to the `Extract` implementation. +#[derive(Default)] +pub struct BaseExtractor {} + +impl Extractor for BaseExtractor { + fn do_extract<'tree>( + self, + ctx: &mut ExtractContext, + node: Option>, + source: &[u8], + leaf_fn: E::LeafFn, + ) -> Result<'tree, E::Output> { + E::extract(ctx, node, source, leaf_fn) + } + + fn do_extract_field<'cursor, 'tree>( + self, + ctx: &mut ExtractContext, + it: &mut ExtractFieldIterator<'cursor, 'tree>, + source: &[u8], + leaf_fn: E::LeafFn, + ) -> Result<'tree, E::Output> { + E::extract_field(ctx, it, source, leaf_fn) + } +} + +/// Transforms leaf nodes from one output type to another. +pub struct MapExtractor { + _e: std::marker::PhantomData, + base: B, + f: F, +} + +impl MapExtractor { + pub fn new(base: B, f: F) -> MapExtractor { + MapExtractor { + _e: std::marker::PhantomData, + base, + f, + } + } +} + +impl Extractor for MapExtractor +where + B: Extractor, + E: Extract, + O: Extract, + F: FnOnce(E::Output) -> O::Output, +{ + fn do_extract<'tree>( + self, + ctx: &mut ExtractContext, + node: Option>, + source: &[u8], + leaf_fn: O::LeafFn, + ) -> Result<'tree, O::Output> { + Ok((self.f)(self.base.do_extract(ctx, node, source, leaf_fn)?)) + } + + fn do_extract_field<'cursor, 'tree>( + self, + _ctx: &mut ExtractContext, + _it: &mut ExtractFieldIterator<'cursor, 'tree>, + _source: &[u8], + _leaf_fn: E::LeafFn, + ) -> Result<'tree, O::Output> { + todo!() + } +} + +/// Map for `#[with(...)]` +pub struct WithLeaf { + _phantom: std::marker::PhantomData, + _f: std::marker::PhantomData, +} + +impl Extract for WithLeaf +where + F: FnOnce(&str) -> L, +{ + type LeafFn = F; + type Output = L; + + fn extract<'tree>( + ctx: &mut ExtractContext, + node: Option>, + source: &[u8], + leaf_fn: Self::LeafFn, + ) -> Result<'tree, L> { + let node = match node { + Some(n) => n, + None => return Err(ExtractError::missing_node(ctx)), + }; + let text = node.utf8_text(source).unwrap(); + Ok(leaf_fn(text)) + } +} + +// Common implementations for various types. + +impl Extract for () { + type LeafFn = (); + type Output = (); + fn extract<'tree>( + _ctx: &mut ExtractContext, + _node: Option>, + _source: &[u8], + _l: (), + ) -> Result<'tree, ()> { + Ok(()) + } +} + +impl Extract for Option { + type LeafFn = T::LeafFn; + type Output = Option; + fn extract<'tree>( + ctx: &mut ExtractContext, + node: Option>, + source: &[u8], + l: T::LeafFn, + ) -> Result<'tree, Option> { + node.map(|n| T::extract(ctx, Some(n), source, l)) + .transpose() + } + + fn extract_field<'cursor, 'tree>( + ctx: &mut ExtractContext, + it: &mut ExtractFieldIterator<'cursor, 'tree>, + source: &[u8], + l: T::LeafFn, + ) -> Result<'tree, Option> { + if it.current_node().is_some() { + Ok(Some(T::extract_field(ctx, it, source, l)?)) + } else { + it.advance_state()?; + Ok(None) + } + } +} + +impl Extract for Box { + type LeafFn = T::LeafFn; + type Output = Box; + fn extract<'tree>( + ctx: &mut ExtractContext, + node: Option>, + source: &[u8], + l: Self::LeafFn, + ) -> Result<'tree, Self::Output> { + Ok(Box::new(T::extract(ctx, node, source, l)?)) + } + + fn extract_field<'cursor, 'tree>( + ctx: &mut ExtractContext, + it: &mut ExtractFieldIterator<'cursor, 'tree>, + source: &[u8], + l: Self::LeafFn, + ) -> Result<'tree, Self::Output> { + Ok(Box::new(T::extract_field(ctx, it, source, l)?)) + } +} + +impl Extract for Vec +where + T::LeafFn: Clone, +{ + type LeafFn = T::LeafFn; + type Output = Vec; + fn extract<'tree>( + _ctx: &mut ExtractContext, + node: Option>, + _source: &[u8], + _l: Self::LeafFn, + ) -> Result<'tree, Self::Output> { + match node { + None => Ok(vec![]), + Some(n) if n.child_count() == 0 => Ok(vec![]), + _ => panic!("Cannot be implemented on Vec"), + } + } + + fn extract_field<'cursor, 'tree>( + ctx: &mut ExtractContext, + it: &mut ExtractFieldIterator<'cursor, 'tree>, + source: &[u8], + leaf_fn: Self::LeafFn, + ) -> Result<'tree, Self::Output> { + let mut out = vec![]; + let mut error = ExtractError::empty(); + while it.is_valid() { + let n = it.current_node(); + match T::extract_field(ctx, it, source, leaf_fn.clone()) { + Ok(t) => out.push(t), + Err(e) => error.merge(e), + } + if let Some(n) = n { + ctx.last_idx = n.end_byte(); + ctx.last_pt = n.end_position(); + } + } + error.prop()?; + Ok(out) + } +} + +macro_rules! extract_from_str { + ($t:ty) => { + impl Extract for $t { + type LeafFn = (); + type Output = $t; + fn extract<'tree>( + ctx: &mut ExtractContext, + node: Option>, + source: &[u8], + _l: (), + ) -> Result<'tree, Self> { + let node = match node { + Some(n) => n, + None => { + return Err(ExtractError::missing_node(ctx)); + } + }; + let text = node.utf8_text(source).expect("No text found for node"); + match text.parse() { + Ok(t) => Ok(t), + Err(e) => Err(ExtractError::type_conversion(ctx, node, e)), + } + } + } + }; +} + +extract_from_str!(u8); +extract_from_str!(i8); +extract_from_str!(u16); +extract_from_str!(i16); +extract_from_str!(u32); +extract_from_str!(i32); +extract_from_str!(u64); +extract_from_str!(i64); +// NOTE: These two may not work as intended due to rounding issues. +extract_from_str!(f32); +extract_from_str!(f64); +// Sort of silly, but keeps it general. +extract_from_str!(String); + +macro_rules! extract_for_tuple { + ($($t:ident),*) => { + impl<$($t: Extract),*> Extract for ($($t),*) + where + $(<$t as Extract>::LeafFn: Default),* + { + type LeafFn = (); + type Output = Self; + fn extract<'tree>( + _ctx: &mut ExtractContext, + _node: Option>, + _source: &[u8], + _l: (), + ) -> Result<'tree, Self> { + panic!("Cannot be implemented on tuples") + } + + fn extract_field<'cursor, 'tree>(ctx: &mut ExtractContext, it: &mut ExtractFieldIterator<'cursor, 'tree>, source: &[u8], _l: ()) -> Result<'tree, Self> { + // NOTE: Nested tuples are not supported as it stands. + Ok(( + $( + $t::extract_field(ctx, it, source, Default::default())? + ),* + )) + } + + } + + }; +} + +extract_for_tuple!(T1, T2); +extract_for_tuple!(T1, T2, T3); +extract_for_tuple!(T1, T2, T3, T4); +extract_for_tuple!(T1, T2, T3, T4, T5); +extract_for_tuple!(T1, T2, T3, T4, T5, T6); +extract_for_tuple!(T1, T2, T3, T4, T5, T6, T7); +extract_for_tuple!(T1, T2, T3, T4, T5, T6, T7, T8); +// Good enough, can maybe generate all of these with a macro if we are clever enough. + +// Would like this to extract optionals specifically if they exist - probably means if a node is +// present then it is true. Might be too magic though. +// impl Extract for bool { +// type LeafFn = (); +// fn extract( +// node: Option, +// source: &[u8], +// last_idx: usize, +// leaf_fn: Option<&Self::LeafFn>, +// ) -> bool { +// } +// } diff --git a/runtime/src/extract/field.rs b/runtime/src/extract/field.rs new file mode 100644 index 0000000..b621b34 --- /dev/null +++ b/runtime/src/extract/field.rs @@ -0,0 +1,349 @@ +use crate::error::ExtractError; + +use super::Result; +use log::trace; +use tree_sitter::Node; + +pub struct ExtractFieldIterator<'cursor, 'tree: 'cursor> { + pub(crate) cursor: &'cursor mut tree_sitter::TreeCursor<'tree>, + pub(crate) field_name: &'static str, + pub(crate) struct_name: &'static str, + pub(crate) ctx: ExtractFieldContext, + pub(crate) source: &'cursor [u8], + pub(crate) current: NodeIterState<'tree>, + pub(crate) did_advance: bool, + pub(crate) final_node: Option>, +} + +pub struct ExtractFieldContext { + state_fn: fn(u32) -> ExtractFieldState, + state: u32, + num_states: u32, + optional: bool, +} + +impl ExtractFieldContext { + pub fn new( + num_states: u32, + optional: bool, + // repeat_type: RepeatType, + state_fn: fn(u32) -> ExtractFieldState, + ) -> Self { + Self { + state_fn, + state: 0, + num_states, + optional, + } + } +} + +#[derive(Debug)] +pub enum ExtractFieldState { + // expected string, is_named, is_optional + Str(&'static str, bool, bool), + // Current implementation only really supports doing this with a list of strings. + Choice(&'static [(&'static str, bool)], bool), + Repeat(&'static str, bool), + Repeat1, + Complete, + // State went too far. + Overflow, +} + +impl<'cursor, 'tree: 'cursor> ExtractFieldIterator<'cursor, 'tree> { + fn skip_extras(&mut self) { + loop { + if self.cursor.node().is_extra() { + if !self.cursor.goto_next_sibling() { + return; + } + continue; + } + return; + } + } + + fn advance_cursor(&mut self) { + if let NodeIterState::Node(n) = self.current { + self.final_node = n; + } + self.did_advance = self.cursor.goto_next_sibling(); + } + + fn set_complete(&mut self) { + if let NodeIterState::Node(n) = self.current { + self.final_node = n; + } + self.current = NodeIterState::Complete; + } + + pub fn advance_state(&mut self) -> Result<'tree, ()> { + if self.current == NodeIterState::Complete { + trace!("advance_state: verifying completion"); + self.finalize()?; + return Ok(()); + } + self.skip_extras(); + let n = self.cursor.node(); + trace!( + "advance_state: field_name={}, cursor.field_name={:?}, state={}, num_states={}, optional={}, node={}, node.kind={}", + self.field_name, + self.cursor.field_name(), + self.ctx.state, + self.ctx.num_states, + self.ctx.optional, + n, + n.kind() + ); + + trace!( + "advance_state: node_string={}", + n.utf8_text(self.source).unwrap() + ); + + let state = (self.ctx.state_fn)(self.ctx.state); + self.ctx.state += 1; + trace!("advance_state: got state={:?}", state); + match state { + ExtractFieldState::Str(expected, named, optional) => { + let cursor_field = self.cursor.field_name(); + let field_name = self.field_name; + if cursor_field != Some(field_name) { + trace!("advance_state: field names didn't match"); + // TODO: It would be generally lovely to clean up this logic throughout. + if optional { + trace!("advance_state: state didn't match, but optional, skipping"); + self.current = NodeIterState::Node(None); + return Ok(()); + } + + // Check if we have an optional overall. + self.handle_optional_err(|| { + format!( + "fields didn't match, cursor had: {:?}, expected: {}", + cursor_field, field_name + ) + })?; + return Ok(()); + } + if n.kind() == expected && n.is_named() == named { + trace!("advance_state: state matched, advancing iteration"); + // advance the cursor and return the current node. + self.advance_cursor(); + self.current = NodeIterState::Node(Some(n)); + Ok(()) + } else if optional { + trace!("advance_state: state didn't match, but optional, skipping"); + self.current = NodeIterState::Node(None); + Ok(()) + } else { + self.handle_optional_err(|| "state didn't match".into())?; + Ok(()) + } + } + ExtractFieldState::Choice(values, optional) => { + let cursor_field = self.cursor.field_name(); + let field_name = self.field_name; + if cursor_field != Some(field_name) { + trace!("advance_state: field names didn't match"); + if optional { + trace!("advance_state: state didn't match, but optional, skipping"); + self.current = NodeIterState::Node(None); + return Ok(()); + } + self.handle_optional_err(|| { + format!( + "fields didn't match, cursor had: {:?}, expected: {}", + cursor_field, field_name + ) + })?; + return Ok(()); + } + for (value, named) in values { + if n.kind() == *value && n.is_named() == *named { + // Found one. + self.advance_cursor(); + self.current = NodeIterState::Node(Some(n)); + return Ok(()); + } + } + if optional { + self.current = NodeIterState::Node(None); + Ok(()) + } else { + self.handle_optional_err(|| "none of the choice values matched".into())?; + Ok(()) + } + } + ExtractFieldState::Repeat(expected, named) => { + trace!("advance_state: repeat state: expected={expected}, named={named}"); + if !self.did_advance { + // We reached the end of the cursor state, we can advance to the end. + self.ctx.state = self.ctx.num_states + 1; + self.set_complete(); + return Ok(()); + } + // Check if the state matches the repeat and then start over from the beginning. If + // it doesn't, then we need to advance again and we should hit the complete state + // after that. + let cursor_field = self.cursor.field_name(); + let field_name = self.field_name; + if cursor_field != Some(field_name) { + trace!("advance_state: field names didn't match in repeat, completing state"); + self.ctx.state = self.ctx.num_states + 1; + self.set_complete(); + // Check if we have an optional overall. + // self.handle_optional_err(|| { + // format!( + // "fields didn't match, cursor had: {:?}, expected: {}", + // cursor_field, field_name + // ) + // })?; + return Ok(()); + } + if n.kind() == expected && n.is_named() == named { + trace!("advance_state: repeat state matched, resetting iteration"); + // Advance past the repeat symbol and start over. + self.advance_cursor(); + self.ctx.state = 0; + self.advance_state()?; + Ok(()) + } else { + self.handle_optional_err(|| "state didn't match".into())?; + Ok(()) + } + } + ExtractFieldState::Repeat1 => { + trace!("advance_state: repeat1 state"); + if !self.did_advance { + self.ctx.state = self.ctx.num_states + 1; + self.set_complete(); + return Ok(()); + } + let cursor_field = self.cursor.field_name(); + let field_name = self.field_name; + if cursor_field != Some(field_name) { + trace!("advance_state: field names didn't match in repeat, completing state"); + self.ctx.state = self.ctx.num_states + 1; + self.set_complete(); + // Check if we have an optional overall. + // self.handle_optional_err(|| { + // format!( + // "fields didn't match, cursor had: {:?}, expected: {}", + // cursor_field, field_name + // ) + // })?; + Ok(()) + } else { + trace!("advance_state: field names matched, triggering repeat"); + // No repeat symbol in this case, we just are at the next repeat node already. + self.ctx.state = 0; + self.advance_state()?; + Ok(()) + } + } + ExtractFieldState::Complete => { + trace!("advance_state: got complete state"); + self.set_complete(); + Ok(()) + } + ExtractFieldState::Overflow => { + self.handle_optional_err(|| "state overflowed".into())?; + Ok(()) + } + } + } + + pub fn next_node(&mut self) -> Result<'tree, Option>> { + let node = self.current_node(); + self.advance_state()?; + Ok(node) + } + + pub fn current_node(&self) -> Option> { + match self.current { + NodeIterState::Node(n) => { + trace!("current_node: {:?}", n.map(|n| n.kind())); + n + } + NodeIterState::Complete => None, + // TODO: Should error? + NodeIterState::Start => None, + } + } + + pub fn is_valid(&self) -> bool { + matches!(self.current, NodeIterState::Node(_)) + } + + pub fn finalize(&self) -> Result<'tree, ()> { + let state = self.ctx.state; + let expected = self.ctx.num_states + 1; + if state != expected { + return Err(ExtractError::field_extraction( + self, + format!("Could not finalize, was in state: {state}, expected: {expected}"), + )); + } + Ok(()) + } +} + +// Some helpers. +impl<'cursor, 'tree> ExtractFieldIterator<'cursor, 'tree> { + fn handle_optional_err(&mut self, f: F) -> Result<'tree, ()> + where + F: FnOnce() -> String, + { + if self.ctx.state == 1 && self.ctx.optional { + trace!("advance_state: optional, outputting None"); + self.ctx.state = self.ctx.num_states + 1; + self.set_complete(); + Ok(()) + } else { + Err(ExtractError::field_extraction(self, f())) + } + } + + pub(crate) fn new( + ctx: ExtractFieldContext, + cursor: &'cursor mut tree_sitter::TreeCursor<'tree>, + struct_name: &'static str, + field_name: &'static str, + source: &'cursor [u8], + ) -> Self { + Self { + cursor, + final_node: None, + current: NodeIterState::Start, + did_advance: false, + source, + field_name, + struct_name, + ctx, + } + } + + pub(crate) fn position(&self) -> crate::Position { + match self.current { + NodeIterState::Node(Some(n)) => crate::Position::from_node(n), + _ => crate::Position::from_node(self.cursor.node()), + } + } + + pub(crate) fn final_position(&self) -> crate::Position { + match self.final_node { + Some(n) => crate::Position::from_node(n), + _ => self.position(), + } + } +} + +#[derive(Default, Clone, Copy, PartialEq)] +pub(crate) enum NodeIterState<'tree> { + Node(Option>), + #[default] + Start, + Complete, +} diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 9343cee..716eee3 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -1,121 +1,63 @@ pub mod __private; +pub mod error; +pub mod extract; +pub mod rule; +pub use rust_sitter_types::grammar; -use std::ops::Deref; - -pub use rust_sitter_macro::*; - -#[cfg(feature = "tree-sitter-standard")] -pub use tree_sitter_runtime_standard as tree_sitter; +pub use rule::Language; -#[cfg(feature = "tree-sitter-c2rust")] -pub use tree_sitter_runtime_c2rust as tree_sitter; +pub use extract::{Extract, ExtractContext, Extractor}; +use serde::{Deserialize, Serialize}; -/// Defines the logic used to convert a node in a Tree Sitter tree to -/// the corresponding Rust type. -pub trait Extract { - type LeafFn: ?Sized; - fn extract( - node: Option, - source: &[u8], - last_idx: usize, - leaf_fn: Option<&Self::LeafFn>, - ) -> Output; -} +use std::ops::Deref; -pub struct WithLeaf { - _phantom: std::marker::PhantomData, -} +pub use rust_sitter_macro::*; +pub use tree_sitter; -impl Extract for WithLeaf { - type LeafFn = dyn Fn(&str) -> L; +use tree_sitter::Node; - fn extract( - node: Option, - source: &[u8], - _last_idx: usize, - leaf_fn: Option<&Self::LeafFn>, - ) -> L { - node.and_then(|n| n.utf8_text(source).ok()) - .map(|s| leaf_fn.unwrap()(s)) - .unwrap() - } +/// The result of a parse. Parses can return errors and potentially still produce a valid result +/// partial result. +pub struct ParseResult { + /// The parse result, if it managed to get one. This can `Some` even if there are errors. + pub result: Option, + /// All errors that were found during parsing. + pub errors: Vec, } -impl Extract<()> for () { - type LeafFn = (); - fn extract( - _node: Option, - _source: &[u8], - _last_idx: usize, - _leaf_fn: Option<&Self::LeafFn>, - ) { - } -} - -impl, U> Extract> for Option { - type LeafFn = T::LeafFn; - fn extract( - node: Option, - source: &[u8], - last_idx: usize, - leaf_fn: Option<&Self::LeafFn>, - ) -> Option { - node.map(|n| T::extract(Some(n), source, last_idx, leaf_fn)) +impl ParseResult { + /// Only return the result if there are no errors. + pub fn into_result(self) -> Result> { + if self.errors.is_empty() { + // It shouldn't be possible to have an empty result with no parse errors. + self.result.ok_or_else(Vec::new) + } else { + Err(self.errors) + } } } -impl, U> Extract> for Box { - type LeafFn = T::LeafFn; - fn extract( - node: Option, - source: &[u8], - last_idx: usize, - leaf_fn: Option<&Self::LeafFn>, - ) -> Box { - Box::new(T::extract(node, source, last_idx, leaf_fn)) +impl std::fmt::Debug for ParseResult { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ParseResult") + .field("result", &self.result) + .field("errors", &self.errors) + .finish() } } -impl, U> Extract> for Vec { - type LeafFn = T::LeafFn; - fn extract( - node: Option, - source: &[u8], - mut last_idx: usize, - leaf_fn: Option<&Self::LeafFn>, - ) -> Vec { - node.map(|node| { - let mut cursor = node.walk(); - let mut out = vec![]; - if cursor.goto_first_child() { - loop { - let n = cursor.node(); - if cursor.field_name().is_some() { - out.push(T::extract(Some(n), source, last_idx, leaf_fn)); - } - - last_idx = n.end_byte(); - - if !cursor.goto_next_sibling() { - break; - } - } - } - - out - }) - .unwrap_or_default() - } +pub struct NodeParseResult<'a, T> { + pub result: Result>, + pub errors: Vec>, } -#[derive(Clone, Debug)] /// A wrapper around a value that also contains the span of the value in the source. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Spanned { /// The underlying parsed node. pub value: T, - /// The span of the node in the source. The first value is the inclusive start - /// of the span, and the second value is the exclusive end of the span. - pub span: (usize, usize), + /// The position where the node is located. + pub position: Position, } impl Deref for Spanned { @@ -126,98 +68,106 @@ impl Deref for Spanned { } } -impl, U> Extract> for Spanned { - type LeafFn = T::LeafFn; - fn extract( - node: Option, - source: &[u8], - last_idx: usize, - leaf_fn: Option<&Self::LeafFn>, - ) -> Spanned { - Spanned { - value: T::extract(node, source, last_idx, leaf_fn), - span: node - .map(|n| (n.start_byte(), n.end_byte())) - .unwrap_or((last_idx, last_idx)), +/// Position in a file, used by errors and `Spanned`. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Position { + /// Byte range. + pub bytes: core::ops::Range, + /// row + column start point. + pub start: Point, + /// row + column end point. + pub end: Point, +} + +impl Position { + fn new(bytes: core::ops::Range, (start, end): (Point, Point)) -> Self { + Self { bytes, start, end } + } + + pub fn from_node(node: Node<'_>) -> Self { + let bytes = node.byte_range(); + let start = Point::from_tree_sitter(node.start_position()); + let end = Point::from_tree_sitter(node.end_position()); + Self { bytes, start, end } + } + + pub fn point_range(&self) -> (Point, Point) { + (self.start, self.end) + } + + fn extend_from(&mut self, other: Position) { + self.bytes = self.bytes.start..other.bytes.end; + self.end = other.end; + } +} + +impl PartialOrd for Position { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Position { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + (self.bytes.start, self.bytes.end).cmp(&(other.bytes.start, other.bytes.end)) + } +} + +/// A line and column point in a source parse. These are 1 based to correspond with a text editor +/// line and column. Note, this is a divergence from tree-sitter, which uses a zero-based `Point`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct Point { + pub line: usize, + pub column: usize, +} + +impl Point { + pub(crate) fn from_tree_sitter(p: tree_sitter::Point) -> Self { + Self { + line: p.row + 1, + column: p.column + 1, } } } -pub mod errors { - #[cfg(feature = "tree-sitter-standard")] - use tree_sitter_runtime_standard as tree_sitter; - - #[cfg(feature = "tree-sitter-c2rust")] - use tree_sitter_runtime_c2rust as tree_sitter; - - #[derive(Debug)] - /// An explanation for an error that occurred during parsing. - pub enum ParseErrorReason { - /// The parser did not expect to see some token. - UnexpectedToken(String), - /// Tree Sitter failed to parse a specific intermediate node. - /// The underlying failures are in the vector. - FailedNode(Vec), - /// The parser expected a specific token, but it was not found. - MissingToken(String), +impl From for Point { + fn from(value: tree_sitter::Point) -> Self { + Self::from_tree_sitter(value) } +} - #[derive(Debug)] - /// An error that occurred during parsing. - pub struct ParseError { - pub reason: ParseErrorReason, - /// Inclusive start of the error. - pub start: usize, - /// Exclusive end of the error. - pub end: usize, +impl Extract for Spanned { + type LeafFn = T::LeafFn; + type Output = Spanned; + fn extract<'a, 'tree>( + ctx: &mut ExtractContext, + node: Option>, + source: &[u8], + l: Self::LeafFn, + ) -> extract::Result<'tree, Self::Output> { + Ok(Spanned { + value: T::extract(ctx, node, source, l)?, + position: node.map(Position::from_node).unwrap_or_else(|| Position { + bytes: ctx.last_idx..ctx.last_idx, + start: Point::from_tree_sitter(ctx.last_pt), + end: Point::from_tree_sitter(ctx.last_pt), + }), + }) } - /// Given the root node of a Tree Sitter parsing result, accumulates all - /// errors that were emitted. - pub fn collect_parsing_errors( - node: &tree_sitter::Node, + fn extract_field<'cursor, 'tree>( + ctx: &mut ExtractContext, + it: &mut extract::ExtractFieldIterator<'cursor, 'tree>, source: &[u8], - errors: &mut Vec, - ) { - if node.is_error() { - if node.child(0).is_some() { - // we managed to parse some children, so collect underlying errors for this node - let mut inner_errors = vec![]; - let mut cursor = node.walk(); - node.children(&mut cursor) - .for_each(|c| collect_parsing_errors(&c, source, &mut inner_errors)); - - errors.push(ParseError { - reason: ParseErrorReason::FailedNode(inner_errors), - start: node.start_byte(), - end: node.end_byte(), - }) - } else { - let contents = node.utf8_text(source).unwrap(); - if !contents.is_empty() { - errors.push(ParseError { - reason: ParseErrorReason::UnexpectedToken(contents.to_string()), - start: node.start_byte(), - end: node.end_byte(), - }) - } else { - errors.push(ParseError { - reason: ParseErrorReason::FailedNode(vec![]), - start: node.start_byte(), - end: node.end_byte(), - }) - } - } - } else if node.is_missing() { - errors.push(ParseError { - reason: ParseErrorReason::MissingToken(node.kind().to_string()), - start: node.start_byte(), - end: node.end_byte(), - }) - } else if node.has_error() { - let mut cursor = node.walk(); - node.children(&mut cursor) - .for_each(|c| collect_parsing_errors(&c, source, errors)); - } + l: Self::LeafFn, + ) -> extract::Result<'tree, Self::Output> { + let mut start = it.position(); + let value = T::extract_field(ctx, it, source, l)?; + let end = it.final_position(); + start.extend_from(end); + Ok(Spanned { + value, + position: start, + }) } } diff --git a/runtime/src/rule.rs b/runtime/src/rule.rs new file mode 100644 index 0000000..d0db17b --- /dev/null +++ b/runtime/src/rule.rs @@ -0,0 +1,44 @@ +use tree_sitter::Node; + +use crate::{Extract, NodeParseResult, ParseResult, extract::ExtractContext}; + +pub trait Rule: Extract { + const RULE_NAME: &'static str; + // TODO: Use the grammar::RuleDef and grammar::Grammar + // For this to work as expected we need a #[derive(Language)], or at least a `Language` trait + // which then has the `parse` function and the `generate_grammar() -> grammar::Grammar` + // implementation instead of just producing an ast. + // Since we aren't using any of this yet though, we will leave this alone. + fn produce_ast() -> String; + // Maybe Cow instead. + fn rule_name() -> &'static str { + Self::RULE_NAME + } + + /// Extracts directly from a node. + fn extract_node<'a>(n: Node<'a>, source: &[u8]) -> NodeParseResult<'a, Self> + where + Self: Sized, + { + let mut ctx = ExtractContext { + last_pt: n.start_position(), + last_idx: n.start_byte(), + field_name: "", + struct_name: Self::rule_name(), + }; + // Extract the errors, and try to parse anyway. + let mut errors = vec![]; + if n.has_error() { + crate::error::collect_node_errors(n, |e| errors.push(e)); + } + let result = Self::extract(&mut ctx, Some(n), source, ()); + NodeParseResult { result, errors } + } +} + +pub trait Language: Rule { + fn produce_grammar() -> String; + + fn language() -> tree_sitter::Language; + fn parse(input: &str) -> ParseResult; +} diff --git a/tool/Cargo.toml b/tool/Cargo.toml index 4335c8f..dd0008f 100644 --- a/tool/Cargo.toml +++ b/tool/Cargo.toml @@ -3,33 +3,24 @@ name = "rust-sitter-tool" description = "The external tool for Rust Sitter that extracts grammars from Rust definitions" readme = "../README.md" repository = "https://github.com/hydro-project/rust-sitter" -version = "0.4.5" -authors = ["Shadaj Laddad "] +version.workspace = true +authors.workspace = true license = "MIT" -edition = "2021" +edition = "2024" keywords = ["parsing", "codegen"] categories = ["development-tools"] -[features] -default = ["build_parsers"] -build_parsers = [ - "dep:tempfile", - "dep:tree-sitter", - "dep:tree-sitter-generate", - "dep:cc", -] - [dependencies] syn = { version = "2", features = ["full", "extra-traits"] } syn-inline-mod = "0.6" serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } -rust-sitter-common = { version = "0.4.5", path = "../common" } +rust-sitter-common = { path = "../common" } -tempfile = { version = "3", optional = true } -tree-sitter = { version = "0.25.2", optional = true } -tree-sitter-generate = { version = "0.25.1", optional = true } -cc = { version = "1", optional = true } +tempfile = "3" +tree-sitter.workspace = true +tree-sitter-generate.workspace = true +cc = "1" [dev-dependencies] -insta = "1.39" +insta = "1" diff --git a/tool/src/expansion.rs b/tool/src/expansion.rs deleted file mode 100644 index 8aa9cf8..0000000 --- a/tool/src/expansion.rs +++ /dev/null @@ -1,511 +0,0 @@ -use std::collections::HashSet; - -use rust_sitter_common::*; -use serde_json::{json, Map, Value}; -use syn::{parse::Parse, punctuated::Punctuated, *}; - -fn gen_field( - path: String, - leaf_type: Type, - leaf_attrs: Vec, - word_rule: &mut Option, - out: &mut Map, -) -> (Value, bool) { - let leaf_attr = leaf_attrs - .iter() - .find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::leaf)); - - if leaf_attrs - .iter() - .any(|attr| attr.path() == &syn::parse_quote!(rust_sitter::word)) - { - if word_rule.is_some() { - panic!("Multiple `word` rules specified"); - } - - *word_rule = Some(path.clone()); - } - - let leaf_params = leaf_attr.and_then(|a| { - a.parse_args_with(Punctuated::::parse_terminated) - .ok() - }); - - let pattern_param = leaf_params.as_ref().and_then(|p| { - p.iter() - .find(|param| param.path == "pattern") - .map(|p| p.expr.clone()) - }); - - let text_param = leaf_params.as_ref().and_then(|p| { - p.iter() - .find(|param| param.path == "text") - .map(|p| p.expr.clone()) - }); - - let mut skip_over = HashSet::new(); - skip_over.insert("Spanned"); - skip_over.insert("Box"); - - let (inner_type_vec, is_vec) = try_extract_inner_type(&leaf_type, "Vec", &skip_over); - let (inner_type_option, is_option) = try_extract_inner_type(&leaf_type, "Option", &skip_over); - - if !is_vec && !is_option { - if let Some(Expr::Lit(lit)) = pattern_param { - if let Lit::Str(s) = &lit.lit { - out.insert( - path.clone(), - json!({ - "type": "PATTERN", - "value": s.value(), - }), - ); - - ( - json!({ - "type": "SYMBOL", - "name": path - }), - is_option, - ) - } else { - panic!("Expected string literal for pattern"); - } - } else if let Some(Expr::Lit(lit)) = text_param { - if let Lit::Str(s) = &lit.lit { - out.insert( - path.clone(), - json!({ - "type": "STRING", - "value": s.value(), - }), - ); - - ( - json!({ - "type": "SYMBOL", - "name": path - }), - is_option, - ) - } else { - panic!("Expected string literal for text"); - } - } else { - let symbol_name = if let Type::Path(p) = filter_inner_type(&leaf_type, &skip_over) { - if p.path.segments.len() == 1 { - p.path.segments[0].ident.to_string() - } else { - panic!("Expected a single segment path"); - } - } else { - panic!("Expected a path"); - }; - - ( - json!({ - "type": "SYMBOL", - "name": symbol_name, - }), - false, - ) - } - } else if is_vec { - let (field_json, field_optional) = gen_field( - path.clone(), - inner_type_vec, - leaf_attr.iter().cloned().cloned().collect(), - word_rule, - out, - ); - - let delimited_attr = leaf_attrs - .iter() - .find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::delimited)); - - let delimited_params = - delimited_attr.and_then(|a| a.parse_args_with(FieldThenParams::parse).ok()); - - let delimiter_json = delimited_params.map(|p| { - gen_field( - format!("{path}_vec_delimiter"), - p.field.ty, - p.field.attrs, - word_rule, - out, - ) - }); - - let repeat_attr = leaf_attrs - .iter() - .find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::repeat)); - - let repeat_params = repeat_attr.and_then(|a| { - a.parse_args_with(Punctuated::::parse_terminated) - .ok() - }); - - let repeat_non_empty = repeat_params - .and_then(|p| { - p.iter() - .find(|param| param.path == "non_empty") - .map(|p| p.expr.clone()) - }) - .map(|e| e == syn::parse_quote!(true)) - .unwrap_or(false); - - let field_rule_non_optional = json!({ - "type": "FIELD", - "name": format!("{path}_vec_element"), - "content": field_json - }); - - let field_rule = if field_optional { - json!({ - "type": "CHOICE", - "members": [ - { - "type": "BLANK" - }, - field_rule_non_optional - ] - }) - } else { - field_rule_non_optional - }; - - let vec_contents = if let Some((delimiter_json, delimiter_optional)) = delimiter_json { - let delim_made_optional = if delimiter_optional { - json!({ - "type": "CHOICE", - "members": [ - { - "type": "BLANK" - }, - delimiter_json - ] - }) - } else { - delimiter_json - }; - - json!({ - "type": "SEQ", - "members": [ - field_rule, - { - "type": if field_optional { - "REPEAT1" - } else { - "REPEAT" - }, - "content": { - "type": "SEQ", - "members": [ - delim_made_optional, - field_rule, - ] - } - } - ] - }) - } else { - json!({ - "type": "REPEAT1", - "content": field_rule - }) - }; - - let contents_ident = format!("{path}_vec_contents"); - out.insert(contents_ident.clone(), vec_contents); - - ( - json!({ - "type": "SYMBOL", - "name": contents_ident, - }), - !repeat_non_empty, - ) - } else { - // is_option - let (field_json, field_optional) = - gen_field(path, inner_type_option, leaf_attrs, word_rule, out); - - if field_optional { - panic!("Option> is not supported"); - } - - (field_json, true) - } -} - -fn gen_struct_or_variant( - path: String, - attrs: Vec, - fields: Fields, - out: &mut Map, - word_rule: &mut Option, -) { - fn gen_field_optional( - path: &str, - field: &Field, - word_rule: &mut Option, - out: &mut Map, - ident_str: String, - ) -> Value { - let (field_contents, is_option) = gen_field( - format!("{path}_{ident_str}"), - field.ty.clone(), - field.attrs.clone(), - word_rule, - out, - ); - - let core = json!({ - "type": "FIELD", - "name": ident_str, - "content": field_contents - }); - - if is_option { - json!({ - "type": "CHOICE", - "members": [ - { - "type": "BLANK" - }, - core - ] - }) - } else { - core - } - } - - let children = fields - .iter() - .enumerate() - .filter_map(|(i, field)| { - if field - .attrs - .iter() - .any(|attr| attr.path() == &syn::parse_quote!(rust_sitter::skip)) - { - None - } else { - let ident_str = field - .ident - .as_ref() - .map(|v| v.to_string()) - .unwrap_or(format!("{i}")); - - Some(gen_field_optional(&path, field, word_rule, out, ident_str)) - } - }) - .collect::>(); - - let prec_attr = attrs - .iter() - .find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::prec)); - - let prec_param = prec_attr.and_then(|a| a.parse_args_with(Expr::parse).ok()); - - let prec_left_attr = attrs - .iter() - .find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::prec_left)); - - let prec_left_param = prec_left_attr.and_then(|a| a.parse_args_with(Expr::parse).ok()); - - let prec_right_attr = attrs - .iter() - .find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::prec_right)); - - let prec_right_param = prec_right_attr.and_then(|a| a.parse_args_with(Expr::parse).ok()); - - let base_rule = match fields { - Fields::Unit => { - let dummy_field = Field { - attrs: attrs.clone(), - vis: Visibility::Inherited, - mutability: FieldMutability::None, - ident: None, - colon_token: None, - ty: Type::Tuple(TypeTuple { - paren_token: Default::default(), - elems: Punctuated::new(), - }), - }; - gen_field_optional(&path, &dummy_field, word_rule, out, "unit".to_owned()) - } - _ => json!({ - "type": "SEQ", - "members": children - }), - }; - - let rule = if let Some(Expr::Lit(lit)) = prec_param { - if prec_left_attr.is_some() || prec_right_attr.is_some() { - panic!("only one of prec, prec_left, and prec_right can be specified"); - } - - if let Lit::Int(i) = &lit.lit { - json!({ - "type": "PREC", - "value": i.base10_parse::().unwrap(), - "content": base_rule - }) - } else { - panic!("Expected integer literal for precedence"); - } - } else if let Some(Expr::Lit(lit)) = prec_left_param { - if prec_right_attr.is_some() { - panic!("only one of prec, prec_left, and prec_right can be specified"); - } - - if let Lit::Int(i) = &lit.lit { - json!({ - "type": "PREC_LEFT", - "value": i.base10_parse::().unwrap(), - "content": base_rule - }) - } else { - panic!("Expected integer literal for precedence"); - } - } else if let Some(Expr::Lit(lit)) = prec_right_param { - if let Lit::Int(i) = &lit.lit { - json!({ - "type": "PREC_RIGHT", - "value": i.base10_parse::().unwrap(), - "content": base_rule - }) - } else { - panic!("Expected integer literal for precedence"); - } - } else { - base_rule - }; - - out.insert(path, rule); -} - -pub fn generate_grammar(module: &ItemMod) -> Value { - let mut rules_map = Map::new(); - // for some reason, source_file must be the first key for things to work - rules_map.insert("source_file".to_string(), json!({})); - - let mut extras_list = vec![]; - - let grammar_name = module - .attrs - .iter() - .find_map(|a| { - if a.path() == &syn::parse_quote!(rust_sitter::grammar) { - let grammar_name_expr = a.parse_args_with(Expr::parse).ok(); - if let Some(Expr::Lit(ExprLit { - attrs: _, - lit: Lit::Str(s), - })) = grammar_name_expr - { - Some(s.value()) - } else { - panic!("Expected string literal for grammar name"); - } - } else { - None - } - }) - .expect("Each grammar must have a name"); - - let (_, contents) = module.content.as_ref().unwrap(); - - let root_type = contents - .iter() - .find_map(|item| match item { - Item::Enum(ItemEnum { ident, attrs, .. }) - | Item::Struct(ItemStruct { ident, attrs, .. }) => { - if attrs - .iter() - .any(|attr| attr.path() == &syn::parse_quote!(rust_sitter::language)) - { - Some(ident.clone()) - } else { - None - } - } - _ => None, - }) - .expect("Each parser must have the root type annotated with `#[rust_sitter::language]`") - .to_string(); - - // Optionally locate the rule annotated with `#[rust_sitter::word]`. - let mut word_rule = None; - contents.iter().for_each(|c| { - let (symbol, attrs) = match c { - Item::Enum(e) => { - e.variants.iter().for_each(|v| { - gen_struct_or_variant( - format!("{}_{}", e.ident, v.ident), - v.attrs.clone(), - v.fields.clone(), - &mut rules_map, - &mut word_rule, - ) - }); - - let mut members: Vec = vec![]; - e.variants.iter().for_each(|v| { - let variant_path = format!("{}_{}", e.ident.clone(), v.ident); - members.push(json!({ - "type": "SYMBOL", - "name": variant_path - })) - }); - - let rule = json!({ - "type": "CHOICE", - "members": members - }); - - rules_map.insert(e.ident.to_string(), rule); - - (e.ident.to_string(), e.attrs.clone()) - } - - Item::Struct(s) => { - gen_struct_or_variant( - s.ident.to_string(), - s.attrs.clone(), - s.fields.clone(), - &mut rules_map, - &mut word_rule, - ); - - (s.ident.to_string(), s.attrs.clone()) - } - - _ => return, - }; - - if attrs - .iter() - .any(|a| a.path() == &syn::parse_quote!(rust_sitter::extra)) - { - extras_list.push(json!({ - "type": "SYMBOL", - "name": symbol - })); - } - }); - - rules_map.insert( - "source_file".to_string(), - rules_map.get(&root_type).unwrap().clone(), - ); - - json!({ - "name": grammar_name, - "word": word_rule, - "rules": rules_map, - "extras": extras_list - }) -} diff --git a/tool/src/lib.rs b/tool/src/lib.rs index e857f9a..595a803 100644 --- a/tool/src/lib.rs +++ b/tool/src/lib.rs @@ -1,160 +1,185 @@ -use serde_json::Value; -use syn::{parse_quote, Item}; - -mod expansion; -use expansion::*; - -const GENERATED_SEMANTIC_VERSION: Option<(u8, u8, u8)> = Some((0, 25, 2)); - -/// Generates JSON strings defining Tree Sitter grammars for every Rust Sitter -/// grammar found in the given module and recursive submodules. -pub fn generate_grammars(root_file: &Path) -> Vec { - let root_file = syn_inline_mod::parse_and_inline_modules(root_file).items; - let mut out = vec![]; - root_file - .iter() - .for_each(|i| generate_all_grammars(i, &mut out)); - out -} - -fn generate_all_grammars(item: &Item, out: &mut Vec) { - if let Item::Mod(m) = item { - m.content - .iter() - .for_each(|(_, items)| items.iter().for_each(|i| generate_all_grammars(i, out))); - - if m.attrs - .iter() - .any(|a| a.path() == &parse_quote!(rust_sitter::grammar)) - { - out.push(generate_grammar(m)) - } - } -} +// TODO: Switch on which version we are using specifically. +const GENERATED_SEMANTIC_VERSION: Option<(u8, u8, u8)> = Some((0, 26, 0)); -#[cfg(feature = "build_parsers")] use std::io::Write; -use std::path::Path; +use std::path::{Path, PathBuf}; -#[cfg(feature = "build_parsers")] use tree_sitter_generate::generate_parser_for_grammar; -#[cfg(feature = "build_parsers")] /// Using the `cc` crate, generates and compiles a C parser with Tree Sitter /// for every Rust Sitter grammar found in the given module and recursive /// submodules. -pub fn build_parsers(root_file: &Path) { - use std::env; - let out_dir = env::var("OUT_DIR").unwrap(); - let emit_artifacts: bool = env::var("RUST_SITTER_EMIT_ARTIFACTS") - .map(|s| s.parse().unwrap_or(false)) - .unwrap_or(false); - generate_grammars(root_file).iter().for_each(|grammar| { - let (grammar_name, grammar_c) = - generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); - let tempfile = tempfile::Builder::new() - .prefix("grammar") - .tempdir() - .unwrap(); - - let dir = if emit_artifacts { - let grammar_dir = Path::new(out_dir.as_str()).join(format!("grammar_{grammar_name}",)); - if grammar_dir.is_dir() { - std::fs::remove_dir_all(&grammar_dir).expect("Couldn't clear old artifacts"); +pub fn build_parser

(root_file: &P) +where + P: AsRef + ?Sized, +{ + ParserBuilder::default().build(root_file); +} + +#[derive(Default)] +pub struct ParserBuilder { + pub output: Option, +} + +impl ParserBuilder { + pub fn output(mut self, output: impl Into) -> Self { + self.output = Some(output.into()); + self + } + + pub fn build

(self, root_file: &P) + where + P: AsRef + ?Sized, + { + let root_file = syn_inline_mod::parse_and_inline_modules(root_file.as_ref()); + match rust_sitter_common::expansion::generate_grammar(root_file.items) { + Err(e) => panic!("{e}"), + Ok(None) => {} + Ok(Some(grammar)) => { + let grammar = serde_json::to_value(grammar).unwrap(); + // TODO: We want to generate better errors here as well. However, it isn't really + // possible to generate it until we can produce a full grammar, which we also can't do + // if we derive on Rule. + if let Err(e) = generate_parser(&grammar, self.output.as_deref()) { + panic!("{e}"); + } + } + } + } +} + +// TODO: Rewrite this function to support specifying the out dir and target manually, to allow +// generating the parser to a local folder for easier integration with external text editors. +fn generate_parser(grammar: &serde_json::Value, out_dir: Option<&Path>) -> Result<(), String> { + let grammar_string = grammar.to_string(); + let (grammar_name, grammar_c) = + match generate_parser_for_grammar(&grammar_string, GENERATED_SEMANTIC_VERSION) { + Ok(o) => o, + Err(e) => { + // Doing it this way produces a clean error from tree-sitter on failure. + return Err(format!("generation error: {e}")); } - std::fs::DirBuilder::new() - .recursive(true) - .create(grammar_dir.clone()) - .expect("Couldn't create grammar JSON directory"); - grammar_dir - } else { - tempfile.path().into() }; + let tempfile = tempfile::Builder::new() + .prefix("grammar") + .tempdir() + .unwrap(); + + let dir = if let Some(out) = out_dir { + out + } else { + tempfile.path() + }; + let _sysroot_dir = write_grammar_and_c_to_dir(&grammar_name, grammar, &grammar_c, dir); + // Check if we have an additional output directory. + if let Ok(output) = std::env::var("RUST_SITTER_PARSER_OUTPUT") { + let output: &Path = output.as_ref(); + write_grammar_and_c_to_dir(&grammar_name, grammar, &grammar_c, output); + } - let grammar_file = dir.join("parser.c"); - let mut f = std::fs::File::create(grammar_file).unwrap(); - - f.write_all(grammar_c.as_bytes()).unwrap(); - drop(f); - - // emit grammar into the build out_dir - let mut grammar_json_file = - std::fs::File::create(dir.join(format!("{grammar_name}.json"))).unwrap(); - grammar_json_file - .write_all(serde_json::to_string_pretty(grammar).unwrap().as_bytes()) - .unwrap(); - drop(grammar_json_file); - - let header_dir = dir.join("tree_sitter"); - std::fs::create_dir(&header_dir).unwrap(); - let mut parser_file = std::fs::File::create(header_dir.join("parser.h")).unwrap(); - parser_file - .write_all(tree_sitter::PARSER_HEADER.as_bytes()) - .unwrap(); - drop(parser_file); - - let sysroot_dir = dir.join("sysroot"); - if env::var("TARGET").unwrap().starts_with("wasm32") { - std::fs::create_dir(&sysroot_dir).unwrap(); - let mut stdint = std::fs::File::create(sysroot_dir.join("stdint.h")).unwrap(); - stdint - .write_all(include_bytes!("wasm-sysroot/stdint.h")) - .unwrap(); - drop(stdint); - - let mut stdlib = std::fs::File::create(sysroot_dir.join("stdlib.h")).unwrap(); - stdlib - .write_all(include_bytes!("wasm-sysroot/stdlib.h")) - .unwrap(); - drop(stdlib); - - let mut stdio = std::fs::File::create(sysroot_dir.join("stdio.h")).unwrap(); - stdio - .write_all(include_bytes!("wasm-sysroot/stdio.h")) - .unwrap(); - drop(stdio); - - let mut stdbool = std::fs::File::create(sysroot_dir.join("stdbool.h")).unwrap(); - stdbool - .write_all(include_bytes!("wasm-sysroot/stdbool.h")) - .unwrap(); - drop(stdbool); - } + let mut c_config = cc::Build::new(); + c_config.std("c11").include(dir); + c_config + .flag_if_supported("-Wno-unused-label") + .flag_if_supported("-Wno-unused-parameter") + .flag_if_supported("-Wno-unused-but-set-variable") + .flag_if_supported("-Wno-trigraphs") + .flag_if_supported("-Wno-everything"); + c_config.file(dir.join("parser.c")); + + c_config.compile(&grammar_name); + Ok(()) +} - let mut c_config = cc::Build::new(); - c_config.std("c11").include(&dir).include(&sysroot_dir); - c_config - .flag_if_supported("-Wno-unused-label") - .flag_if_supported("-Wno-unused-parameter") - .flag_if_supported("-Wno-unused-but-set-variable") - .flag_if_supported("-Wno-trigraphs") - .flag_if_supported("-Wno-everything"); - c_config.file(dir.join("parser.c")); - - c_config.compile(&grammar_name); - }); +fn write_grammar_and_c_to_dir( + grammar_name: &str, + grammar: &serde_json::Value, + grammar_c: &str, + dir: &Path, +) -> PathBuf { + let grammar_file = dir.join("parser.c"); + let mut f = std::fs::File::create(grammar_file).unwrap(); + + f.write_all(grammar_c.as_bytes()).unwrap(); + drop(f); + + // emit grammar into the build out_dir + let mut grammar_json_file = + std::fs::File::create(dir.join(format!("{grammar_name}.json"))).unwrap(); + grammar_json_file + .write_all(serde_json::to_string_pretty(grammar).unwrap().as_bytes()) + .unwrap(); + drop(grammar_json_file); + + let header_dir = dir.join("tree_sitter"); + std::fs::create_dir_all(&header_dir).unwrap(); + let mut parser_file = std::fs::File::create(header_dir.join("parser.h")).unwrap(); + parser_file + .write_all(tree_sitter::PARSER_HEADER.as_bytes()) + .unwrap(); + drop(parser_file); + + let sysroot_dir = dir.join("sysroot"); + // if std::env::var("TARGET").unwrap().starts_with("wasm32") { + // std::fs::create_dir(&sysroot_dir).unwrap(); + // let mut stdint = std::fs::File::create(sysroot_dir.join("stdint.h")).unwrap(); + // stdint + // .write_all(include_bytes!("wasm-sysroot/stdint.h")) + // .unwrap(); + // drop(stdint); + + // let mut stdlib = std::fs::File::create(sysroot_dir.join("stdlib.h")).unwrap(); + // stdlib + // .write_all(include_bytes!("wasm-sysroot/stdlib.h")) + // .unwrap(); + // drop(stdlib); + + // let mut stdio = std::fs::File::create(sysroot_dir.join("stdio.h")).unwrap(); + // stdio + // .write_all(include_bytes!("wasm-sysroot/stdio.h")) + // .unwrap(); + // drop(stdio); + + // let mut stdbool = std::fs::File::create(sysroot_dir.join("stdbool.h")).unwrap(); + // stdbool + // .write_all(include_bytes!("wasm-sysroot/stdbool.h")) + // .unwrap(); + // drop(stdbool); + // } + + sysroot_dir } #[cfg(test)] mod tests { - use syn::parse_quote; + use syn::{ItemMod, parse_quote}; - use super::{generate_grammar, GENERATED_SEMANTIC_VERSION}; + use super::GENERATED_SEMANTIC_VERSION; + // use rust_sitter_common::expansion::generate_grammar; use tree_sitter_generate::generate_parser_for_grammar; + fn generate_grammar(item: ItemMod) -> serde_json::Value { + let (_, items) = item.content.unwrap(); + serde_json::to_value( + rust_sitter_common::expansion::generate_grammar(items) + .unwrap() + .unwrap(), + ) + .unwrap() + } #[test] fn enum_with_named_field() { let m = if let syn::Item::Mod(m) = parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub enum Expr { Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(pattern(r"\d+"))] u32 ), Neg { - #[rust_sitter::leaf(text = "!")] + #[leaf("!")] _bang: (), value: Box, } @@ -166,7 +191,7 @@ mod tests { panic!() }; - let grammar = generate_grammar(&m); + let grammar = generate_grammar(m); insta::assert_snapshot!(grammar); generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); } @@ -174,12 +199,13 @@ mod tests { #[test] fn enum_transformed_fields() { let m = if let syn::Item::Mod(m) = parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub enum Expression { Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v: &str| v.parse::().unwrap())] + #[leaf(pattern(r"\d+"))] + #[transform(|v: &str| v.parse::().unwrap())] i32 ), } @@ -190,7 +216,7 @@ mod tests { panic!() }; - let grammar = generate_grammar(&m); + let grammar = generate_grammar(m); insta::assert_snapshot!(grammar); generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); } @@ -198,16 +224,16 @@ mod tests { #[test] fn enum_recursive() { let m = if let syn::Item::Mod(m) = parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub enum Expression { Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v: &str| v.parse::().unwrap())] + #[leaf(pattern(r"\d+"))] i32 ), Neg( - #[rust_sitter::leaf(text = "-", transform = |v| ())] + #[leaf("-")] (), Box ), @@ -219,7 +245,7 @@ mod tests { panic!() }; - let grammar = generate_grammar(&m); + let grammar = generate_grammar(m); insta::assert_snapshot!(grammar); generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); } @@ -227,18 +253,18 @@ mod tests { #[test] fn enum_prec_left() { let m = if let syn::Item::Mod(m) = parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub enum Expression { Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v: &str| v.parse::().unwrap())] + #[leaf(pattern(r"\d+"))] i32 ), - #[rust_sitter::prec_left(1)] + #[prec_left(1)] Sub( Box, - #[rust_sitter::leaf(text = "-", transform = |v| ())] + #[leaf("-")] (), Box ), @@ -250,7 +276,99 @@ mod tests { panic!() }; - let grammar = generate_grammar(&m); + let grammar = generate_grammar(m); + insta::assert_snapshot!(grammar); + generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); + } + + #[test] + fn enum_conflicts_prec_dynamic() { + let m = if let syn::Item::Mod(m) = parse_quote! { + mod grammar { + #[derive(rust_sitter::Rule)] + #[language] + #[word(Identifier)] + pub struct Program(pub Vec); + + #[derive(rust_sitter::Rule)] + pub enum Statement { + ExpressionStatement(ExpressionStatement), + IfStatement(Box), + } + + #[derive(rust_sitter::Rule)] + pub enum Expression { + Identifier(Identifier), + Number(Number), + BinaryExpression(Box), + } + + #[derive(rust_sitter::Rule)] + #[prec_left(1)] + pub struct BinaryExpression { + pub expression: Expression, + pub binary_expression_inner: BinaryExpressionInner, + pub expression2: Expression, + } + + #[derive(rust_sitter::Rule)] + pub enum BinaryExpressionInner { + String(#[leaf("+")] ()), + String2(#[leaf("-")] ()), + String3(#[leaf("*")] ()), + String4(#[leaf("/")] ()), + } + + #[derive(rust_sitter::Rule)] + pub struct ExpressionStatement { + pub expression: Expression, + #[leaf(";")] + pub _semicolon: (), + } + + #[derive(rust_sitter::Rule)] + #[prec_dynamic(1)] + pub struct IfStatement { + #[leaf("if")] + pub _if: (), + #[leaf("(")] + pub _lparen: (), + pub expression: Expression, + #[leaf(")")] + pub _rparen: (), + #[leaf("{")] + pub _lbrace: (), + pub statement: Statement, + #[leaf("}")] + pub _rbrace: (), + pub if_statement_inner: Option, + } + + #[derive(rust_sitter::Rule)] + pub struct IfStatementElse { + #[leaf("else")] + pub _else: (), + #[leaf("{")] + pub _lbrace: (), + pub statement: Statement, + #[leaf("}")] + pub _rbrace: (), + } + + #[derive(rust_sitter::Rule)] + #[leaf(pattern("[a-zA-Z_][a-zA-Z0-9_]*"))] + pub struct Identifier; + + #[derive(rust_sitter::Rule)] + pub struct Number(#[leaf(pattern("\\d+"))] ()); + } + } { + m + } else { + panic!() + }; + + let grammar = generate_grammar(m); insta::assert_snapshot!(grammar); generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); } @@ -258,21 +376,18 @@ mod tests { #[test] fn grammar_with_extras() { let m = if let syn::Item::Mod(m) = parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] + #[extras( + re(r"\s") + )] pub enum Expression { Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v: &str| v.parse::().unwrap())] + #[leaf(re(r"\d+"))] i32 ), } - - #[rust_sitter::extra] - struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s", transform = |_v| ())] - _whitespace: (), - } } } { m @@ -280,7 +395,7 @@ mod tests { panic!() }; - let grammar = generate_grammar(&m); + let grammar = generate_grammar(m); insta::assert_snapshot!(grammar); generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); } @@ -288,16 +403,17 @@ mod tests { #[test] fn grammar_unboxed_field() { let m = if let syn::Item::Mod(m) = parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub struct Language { e: Expression, } + #[derive(rust_sitter::Rule)] pub enum Expression { Number( - #[rust_sitter::leaf(pattern = r"\d+", transform = |v: &str| v.parse::().unwrap())] + #[leaf(re(r"\d+"))] i32 ), } @@ -308,7 +424,7 @@ mod tests { panic!() }; - let grammar = generate_grammar(&m); + let grammar = generate_grammar(m); insta::assert_snapshot!(grammar); generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); } @@ -316,27 +432,22 @@ mod tests { #[test] fn grammar_repeat() { let m = if let syn::Item::Mod(m) = parse_quote! { - #[rust_sitter::grammar("test")] pub mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] + #[extras( + re(r"\s") + )] pub struct NumberList { - #[rust_sitter::delimited( - #[rust_sitter::leaf(text = ",")] - () - )] + #[sep_by(",")] numbers: Vec, } + #[derive(Rule)] pub struct Number { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] v: i32, } - - #[rust_sitter::extra] - struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s")] - _whitespace: (), - } } } { m @@ -344,7 +455,7 @@ mod tests { panic!() }; - let grammar = generate_grammar(&m); + let grammar = generate_grammar(m); insta::assert_snapshot!(grammar); generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); } @@ -352,23 +463,21 @@ mod tests { #[test] fn grammar_repeat_no_delimiter() { let m = if let syn::Item::Mod(m) = parse_quote! { - #[rust_sitter::grammar("test")] pub mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] + #[extras( + re(r"\s") + )] pub struct NumberList { numbers: Vec, } + #[derive(rust_sitter::Rule)] pub struct Number { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] v: i32, } - - #[rust_sitter::extra] - struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s")] - _whitespace: (), - } } } { m @@ -376,7 +485,7 @@ mod tests { panic!() }; - let grammar = generate_grammar(&m); + let grammar = generate_grammar(m); insta::assert_snapshot!(grammar); generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); } @@ -384,28 +493,23 @@ mod tests { #[test] fn grammar_repeat1() { let m = if let syn::Item::Mod(m) = parse_quote! { - #[rust_sitter::grammar("test")] pub mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] + #[extras( + re(r"\s") + )] pub struct NumberList { - #[rust_sitter::repeat(non_empty = true)] - #[rust_sitter::delimited( - #[rust_sitter::leaf(text = ",")] - () - )] + #[repeat(non_empty = true)] + #[delimited(",")] numbers: Vec, } + #[derive(rust_sitter::Rule)] pub struct Number { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] v: i32, } - - #[rust_sitter::extra] - struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s")] - _whitespace: (), - } } } { m @@ -413,7 +517,7 @@ mod tests { panic!() }; - let grammar = generate_grammar(&m); + let grammar = generate_grammar(m); insta::assert_snapshot!(grammar); generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); } @@ -421,19 +525,20 @@ mod tests { #[test] fn struct_optional() { let m = if let syn::Item::Mod(m) = parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub struct Language { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] v: Option, - #[rust_sitter::leaf(pattern = r" ", transform = |v| ())] + #[leaf(re(r" "))] space: (), t: Option, } + #[derive(rust_sitter::Rule)] pub struct Number { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] v: i32 } } @@ -443,7 +548,7 @@ mod tests { panic!() }; - let grammar = generate_grammar(&m); + let grammar = generate_grammar(m); insta::assert_snapshot!(grammar); generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); } @@ -451,17 +556,18 @@ mod tests { #[test] fn enum_with_unamed_vector() { let m = if let syn::Item::Mod(m) = parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { + #[derive(rust_sitter::Rule)] pub struct Number { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] value: u32 } - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] pub enum Expr { Numbers( - #[rust_sitter::repeat(non_empty = true)] + #[repeat1] Vec ) } @@ -472,7 +578,7 @@ mod tests { panic!() }; - let grammar = generate_grammar(&m); + let grammar = generate_grammar(m); insta::assert_snapshot!(grammar); generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); } @@ -480,21 +586,45 @@ mod tests { #[test] fn spanned_in_vec() { let m = if let syn::Item::Mod(m) = parse_quote! { - #[rust_sitter::grammar("test")] mod grammar { use rust_sitter::Spanned; - #[rust_sitter::language] + #[derive(rust_sitter::Rule)] + #[language] + #[extras( + re(r"\s") + )] pub struct NumberList { - #[rust_sitter::leaf(pattern = r"\d+", transform = |v| v.parse().unwrap())] + #[leaf(re(r"\d+"))] numbers: Vec>, } + } + } { + m + } else { + panic!() + }; - #[rust_sitter::extra] - struct Whitespace { - #[rust_sitter::leaf(pattern = r"\s")] - _whitespace: (), - } + let grammar = generate_grammar(m); + insta::assert_snapshot!(grammar); + generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); + } + + #[test] + fn immediate() { + let m = if let syn::Item::Mod(m) = parse_quote! { + mod grammar { + #[derive(rust_sitter::Rule)] + #[language] + #[extras( + re(r"\s") + )] + pub struct StringFragment( + #[immediate] + #[prec(1)] + #[leaf(pattern(r#"[^"\\]+"#))] + () + ); } } { m @@ -502,7 +632,7 @@ mod tests { panic!() }; - let grammar = generate_grammar(&m); + let grammar = generate_grammar(m); insta::assert_snapshot!(grammar); generate_parser_for_grammar(&grammar.to_string(), GENERATED_SEMANTIC_VERSION).unwrap(); } diff --git a/tool/src/snapshots/rust_sitter_tool__tests__enum_conflicts_prec_dynamic.snap b/tool/src/snapshots/rust_sitter_tool__tests__enum_conflicts_prec_dynamic.snap new file mode 100644 index 0000000..1fd0c09 --- /dev/null +++ b/tool/src/snapshots/rust_sitter_tool__tests__enum_conflicts_prec_dynamic.snap @@ -0,0 +1,5 @@ +--- +source: tool/src/lib.rs +expression: grammar +--- +{"name":"Program","word":"Identifier","rules":{"source_file":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"0","content":{"type":"REPEAT1","content":{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Statement"}}}}]}]},"Program":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"0","content":{"type":"REPEAT1","content":{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Statement"}}}}]}]},"Statement_ExpressionStatement":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"ExpressionStatement"}}]},"Statement_IfStatement":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"IfStatement"}}]},"Statement":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Statement_ExpressionStatement"},{"type":"SYMBOL","name":"Statement_IfStatement"}]},"Expression_Identifier":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Identifier"}}]},"Expression_Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Number"}}]},"Expression_BinaryExpression":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"BinaryExpression"}}]},"Expression":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Identifier"},{"type":"SYMBOL","name":"Expression_Number"},{"type":"SYMBOL","name":"Expression_BinaryExpression"}]},"BinaryExpression":{"type":"PREC_LEFT","value":1,"content":{"type":"SEQ","members":[{"type":"FIELD","name":"expression","content":{"type":"SYMBOL","name":"Expression"}},{"type":"FIELD","name":"binary_expression_inner","content":{"type":"SYMBOL","name":"BinaryExpressionInner"}},{"type":"FIELD","name":"expression2","content":{"type":"SYMBOL","name":"Expression"}}]}},"BinaryExpressionInner_String":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"STRING","value":"+"}}]},"BinaryExpressionInner_String2":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"STRING","value":"-"}}]},"BinaryExpressionInner_String3":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"STRING","value":"*"}}]},"BinaryExpressionInner_String4":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"STRING","value":"/"}}]},"BinaryExpressionInner":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"BinaryExpressionInner_String"},{"type":"SYMBOL","name":"BinaryExpressionInner_String2"},{"type":"SYMBOL","name":"BinaryExpressionInner_String3"},{"type":"SYMBOL","name":"BinaryExpressionInner_String4"}]},"ExpressionStatement":{"type":"SEQ","members":[{"type":"FIELD","name":"expression","content":{"type":"SYMBOL","name":"Expression"}},{"type":"FIELD","name":"_semicolon","content":{"type":"STRING","value":";"}}]},"IfStatement":{"type":"PREC_DYNAMIC","value":1,"content":{"type":"SEQ","members":[{"type":"FIELD","name":"_if","content":{"type":"STRING","value":"if"}},{"type":"FIELD","name":"_lparen","content":{"type":"STRING","value":"("}},{"type":"FIELD","name":"expression","content":{"type":"SYMBOL","name":"Expression"}},{"type":"FIELD","name":"_rparen","content":{"type":"STRING","value":")"}},{"type":"FIELD","name":"_lbrace","content":{"type":"STRING","value":"{"}},{"type":"FIELD","name":"statement","content":{"type":"SYMBOL","name":"Statement"}},{"type":"FIELD","name":"_rbrace","content":{"type":"STRING","value":"}"}},{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"if_statement_inner","content":{"type":"SYMBOL","name":"IfStatementElse"}}]}]}},"IfStatementElse":{"type":"SEQ","members":[{"type":"FIELD","name":"_else","content":{"type":"STRING","value":"else"}},{"type":"FIELD","name":"_lbrace","content":{"type":"STRING","value":"{"}},{"type":"FIELD","name":"statement","content":{"type":"SYMBOL","name":"Statement"}},{"type":"FIELD","name":"_rbrace","content":{"type":"STRING","value":"}"}}]},"Identifier":{"type":"PATTERN","value":"[a-zA-Z_][a-zA-Z0-9_]*"},"Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"PATTERN","value":"\\d+"}}]}},"extras":[]} diff --git a/tool/src/snapshots/rust_sitter_tool__tests__enum_prec_left.snap b/tool/src/snapshots/rust_sitter_tool__tests__enum_prec_left.snap index c2c0aec..04e00a9 100644 --- a/tool/src/snapshots/rust_sitter_tool__tests__enum_prec_left.snap +++ b/tool/src/snapshots/rust_sitter_tool__tests__enum_prec_left.snap @@ -1,5 +1,5 @@ --- source: tool/src/lib.rs -expression: generate_grammar(&m) +expression: grammar --- -{"name":"test","word":null,"rules":{"source_file":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"},{"type":"SYMBOL","name":"Expression_Sub"}]},"Expression_Number_0":{"type":"PATTERN","value":"\\d+"},"Expression_Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Expression_Number_0"}}]},"Expression_Sub_1":{"type":"STRING","value":"-"},"Expression_Sub":{"type":"PREC_LEFT","value":1,"content":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Expression"}},{"type":"FIELD","name":"1","content":{"type":"SYMBOL","name":"Expression_Sub_1"}},{"type":"FIELD","name":"2","content":{"type":"SYMBOL","name":"Expression"}}]}},"Expression":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"},{"type":"SYMBOL","name":"Expression_Sub"}]}},"extras":[]} +{"name":"Expression","word":null,"rules":{"source_file":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"},{"type":"SYMBOL","name":"Expression_Sub"}]},"Expression_Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"PATTERN","value":"\\d+"}}]},"Expression_Sub":{"type":"PREC_LEFT","value":1,"content":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Expression"}},{"type":"FIELD","name":"1","content":{"type":"STRING","value":"-"}},{"type":"FIELD","name":"2","content":{"type":"SYMBOL","name":"Expression"}}]}},"Expression":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"},{"type":"SYMBOL","name":"Expression_Sub"}]}},"extras":[]} diff --git a/tool/src/snapshots/rust_sitter_tool__tests__enum_recursive.snap b/tool/src/snapshots/rust_sitter_tool__tests__enum_recursive.snap index 14afcf2..9568dc1 100644 --- a/tool/src/snapshots/rust_sitter_tool__tests__enum_recursive.snap +++ b/tool/src/snapshots/rust_sitter_tool__tests__enum_recursive.snap @@ -1,5 +1,5 @@ --- source: tool/src/lib.rs -expression: generate_grammar(&m) +expression: grammar --- -{"name":"test","word":null,"rules":{"source_file":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"},{"type":"SYMBOL","name":"Expression_Neg"}]},"Expression_Number_0":{"type":"PATTERN","value":"\\d+"},"Expression_Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Expression_Number_0"}}]},"Expression_Neg_0":{"type":"STRING","value":"-"},"Expression_Neg":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Expression_Neg_0"}},{"type":"FIELD","name":"1","content":{"type":"SYMBOL","name":"Expression"}}]},"Expression":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"},{"type":"SYMBOL","name":"Expression_Neg"}]}},"extras":[]} +{"name":"Expression","word":null,"rules":{"source_file":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"},{"type":"SYMBOL","name":"Expression_Neg"}]},"Expression_Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"PATTERN","value":"\\d+"}}]},"Expression_Neg":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"STRING","value":"-"}},{"type":"FIELD","name":"1","content":{"type":"SYMBOL","name":"Expression"}}]},"Expression":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"},{"type":"SYMBOL","name":"Expression_Neg"}]}},"extras":[]} diff --git a/tool/src/snapshots/rust_sitter_tool__tests__enum_transformed_fields.snap b/tool/src/snapshots/rust_sitter_tool__tests__enum_transformed_fields.snap index 016e818..d87eca9 100644 --- a/tool/src/snapshots/rust_sitter_tool__tests__enum_transformed_fields.snap +++ b/tool/src/snapshots/rust_sitter_tool__tests__enum_transformed_fields.snap @@ -1,5 +1,5 @@ --- source: tool/src/lib.rs -expression: generate_grammar(&m) +expression: grammar --- -{"name":"test","word":null,"rules":{"source_file":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"}]},"Expression_Number_0":{"type":"PATTERN","value":"\\d+"},"Expression_Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Expression_Number_0"}}]},"Expression":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"}]}},"extras":[]} +{"name":"Expression","word":null,"rules":{"source_file":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"}]},"Expression_Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"PATTERN","value":"\\d+"}}]},"Expression":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"}]}},"extras":[]} diff --git a/tool/src/snapshots/rust_sitter_tool__tests__enum_with_named_field.snap b/tool/src/snapshots/rust_sitter_tool__tests__enum_with_named_field.snap index b02e30d..bfb7539 100644 --- a/tool/src/snapshots/rust_sitter_tool__tests__enum_with_named_field.snap +++ b/tool/src/snapshots/rust_sitter_tool__tests__enum_with_named_field.snap @@ -1,5 +1,5 @@ --- source: tool/src/lib.rs -expression: generate_grammar(&m) +expression: grammar --- -{"name":"test","word":null,"rules":{"source_file":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expr_Number"},{"type":"SYMBOL","name":"Expr_Neg"}]},"Expr_Number_0":{"type":"PATTERN","value":"\\d+"},"Expr_Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Expr_Number_0"}}]},"Expr_Neg__bang":{"type":"STRING","value":"!"},"Expr_Neg":{"type":"SEQ","members":[{"type":"FIELD","name":"_bang","content":{"type":"SYMBOL","name":"Expr_Neg__bang"}},{"type":"FIELD","name":"value","content":{"type":"SYMBOL","name":"Expr"}}]},"Expr":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expr_Number"},{"type":"SYMBOL","name":"Expr_Neg"}]}},"extras":[]} +{"name":"Expr","word":null,"rules":{"source_file":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expr_Number"},{"type":"SYMBOL","name":"Expr_Neg"}]},"Expr_Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"PATTERN","value":"\\d+"}}]},"Expr_Neg":{"type":"SEQ","members":[{"type":"FIELD","name":"_bang","content":{"type":"STRING","value":"!"}},{"type":"FIELD","name":"value","content":{"type":"SYMBOL","name":"Expr"}}]},"Expr":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expr_Number"},{"type":"SYMBOL","name":"Expr_Neg"}]}},"extras":[]} diff --git a/tool/src/snapshots/rust_sitter_tool__tests__enum_with_unamed_vector.snap b/tool/src/snapshots/rust_sitter_tool__tests__enum_with_unamed_vector.snap index 3dee28f..b8e218c 100644 --- a/tool/src/snapshots/rust_sitter_tool__tests__enum_with_unamed_vector.snap +++ b/tool/src/snapshots/rust_sitter_tool__tests__enum_with_unamed_vector.snap @@ -2,4 +2,4 @@ source: tool/src/lib.rs expression: grammar --- -{"name":"test","word":null,"rules":{"source_file":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expr_Numbers"}]},"Number_value":{"type":"PATTERN","value":"\\d+"},"Number":{"type":"SEQ","members":[{"type":"FIELD","name":"value","content":{"type":"SYMBOL","name":"Number_value"}}]},"Expr_Numbers_0_vec_contents":{"type":"REPEAT1","content":{"type":"FIELD","name":"Expr_Numbers_0_vec_element","content":{"type":"SYMBOL","name":"Number"}}},"Expr_Numbers":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Expr_Numbers_0_vec_contents"}}]},"Expr":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expr_Numbers"}]}},"extras":[]} +{"name":"Expr","word":null,"rules":{"source_file":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expr_Numbers"}]},"Number":{"type":"SEQ","members":[{"type":"FIELD","name":"value","content":{"type":"PATTERN","value":"\\d+"}}]},"Expr_Numbers":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"REPEAT1","content":{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Number"}}}}]},"Expr":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expr_Numbers"}]}},"extras":[]} diff --git a/tool/src/snapshots/rust_sitter_tool__tests__grammar_repeat.snap b/tool/src/snapshots/rust_sitter_tool__tests__grammar_repeat.snap index dc3025e..5038607 100644 --- a/tool/src/snapshots/rust_sitter_tool__tests__grammar_repeat.snap +++ b/tool/src/snapshots/rust_sitter_tool__tests__grammar_repeat.snap @@ -2,4 +2,4 @@ source: tool/src/lib.rs expression: grammar --- -{"name":"test","word":null,"rules":{"source_file":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"SYMBOL","name":"NumberList_numbers_vec_contents"}}]}]},"NumberList_numbers_vec_delimiter":{"type":"STRING","value":","},"NumberList_numbers_vec_contents":{"type":"SEQ","members":[{"type":"FIELD","name":"NumberList_numbers_vec_element","content":{"type":"SYMBOL","name":"Number"}},{"type":"REPEAT","content":{"type":"SEQ","members":[{"type":"SYMBOL","name":"NumberList_numbers_vec_delimiter"},{"type":"FIELD","name":"NumberList_numbers_vec_element","content":{"type":"SYMBOL","name":"Number"}}]}}]},"NumberList":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"SYMBOL","name":"NumberList_numbers_vec_contents"}}]}]},"Number_v":{"type":"PATTERN","value":"\\d+"},"Number":{"type":"SEQ","members":[{"type":"FIELD","name":"v","content":{"type":"SYMBOL","name":"Number_v"}}]},"Whitespace__whitespace":{"type":"PATTERN","value":"\\s"},"Whitespace":{"type":"SEQ","members":[{"type":"FIELD","name":"_whitespace","content":{"type":"SYMBOL","name":"Whitespace__whitespace"}}]}},"extras":[{"type":"SYMBOL","name":"Whitespace"}]} +{"name":"NumberList","word":null,"rules":{"source_file":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"SEQ","members":[{"type":"SYMBOL","name":"Number"},{"type":"REPEAT","content":{"type":"SEQ","members":[{"type":"STRING","value":","},{"type":"SYMBOL","name":"Number"}]}}]}}]}]},"NumberList":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"SEQ","members":[{"type":"SYMBOL","name":"Number"},{"type":"REPEAT","content":{"type":"SEQ","members":[{"type":"STRING","value":","},{"type":"SYMBOL","name":"Number"}]}}]}}]}]},"Number":{"type":"SEQ","members":[{"type":"FIELD","name":"v","content":{"type":"PATTERN","value":"\\d+"}}]}},"extras":[{"type":"PATTERN","value":"\\s"}]} diff --git a/tool/src/snapshots/rust_sitter_tool__tests__grammar_repeat1.snap b/tool/src/snapshots/rust_sitter_tool__tests__grammar_repeat1.snap index c14068e..2fcba85 100644 --- a/tool/src/snapshots/rust_sitter_tool__tests__grammar_repeat1.snap +++ b/tool/src/snapshots/rust_sitter_tool__tests__grammar_repeat1.snap @@ -2,4 +2,4 @@ source: tool/src/lib.rs expression: grammar --- -{"name":"test","word":null,"rules":{"source_file":{"type":"SEQ","members":[{"type":"FIELD","name":"numbers","content":{"type":"SYMBOL","name":"NumberList_numbers_vec_contents"}}]},"NumberList_numbers_vec_delimiter":{"type":"STRING","value":","},"NumberList_numbers_vec_contents":{"type":"SEQ","members":[{"type":"FIELD","name":"NumberList_numbers_vec_element","content":{"type":"SYMBOL","name":"Number"}},{"type":"REPEAT","content":{"type":"SEQ","members":[{"type":"SYMBOL","name":"NumberList_numbers_vec_delimiter"},{"type":"FIELD","name":"NumberList_numbers_vec_element","content":{"type":"SYMBOL","name":"Number"}}]}}]},"NumberList":{"type":"SEQ","members":[{"type":"FIELD","name":"numbers","content":{"type":"SYMBOL","name":"NumberList_numbers_vec_contents"}}]},"Number_v":{"type":"PATTERN","value":"\\d+"},"Number":{"type":"SEQ","members":[{"type":"FIELD","name":"v","content":{"type":"SYMBOL","name":"Number_v"}}]},"Whitespace__whitespace":{"type":"PATTERN","value":"\\s"},"Whitespace":{"type":"SEQ","members":[{"type":"FIELD","name":"_whitespace","content":{"type":"SYMBOL","name":"Whitespace__whitespace"}}]}},"extras":[{"type":"SYMBOL","name":"Whitespace"}]} +{"name":"NumberList","word":null,"rules":{"source_file":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"REPEAT1","content":{"type":"FIELD","name":"numbers","content":{"type":"SYMBOL","name":"Number"}}}}]}]},"NumberList":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"REPEAT1","content":{"type":"FIELD","name":"numbers","content":{"type":"SYMBOL","name":"Number"}}}}]}]},"Number":{"type":"SEQ","members":[{"type":"FIELD","name":"v","content":{"type":"PATTERN","value":"\\d+"}}]}},"extras":[{"type":"PATTERN","value":"\\s"}]} diff --git a/tool/src/snapshots/rust_sitter_tool__tests__grammar_repeat_no_delimiter.snap b/tool/src/snapshots/rust_sitter_tool__tests__grammar_repeat_no_delimiter.snap index c5f6753..2fcba85 100644 --- a/tool/src/snapshots/rust_sitter_tool__tests__grammar_repeat_no_delimiter.snap +++ b/tool/src/snapshots/rust_sitter_tool__tests__grammar_repeat_no_delimiter.snap @@ -2,4 +2,4 @@ source: tool/src/lib.rs expression: grammar --- -{"name":"test","word":null,"rules":{"source_file":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"SYMBOL","name":"NumberList_numbers_vec_contents"}}]}]},"NumberList_numbers_vec_contents":{"type":"REPEAT1","content":{"type":"FIELD","name":"NumberList_numbers_vec_element","content":{"type":"SYMBOL","name":"Number"}}},"NumberList":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"SYMBOL","name":"NumberList_numbers_vec_contents"}}]}]},"Number_v":{"type":"PATTERN","value":"\\d+"},"Number":{"type":"SEQ","members":[{"type":"FIELD","name":"v","content":{"type":"SYMBOL","name":"Number_v"}}]},"Whitespace__whitespace":{"type":"PATTERN","value":"\\s"},"Whitespace":{"type":"SEQ","members":[{"type":"FIELD","name":"_whitespace","content":{"type":"SYMBOL","name":"Whitespace__whitespace"}}]}},"extras":[{"type":"SYMBOL","name":"Whitespace"}]} +{"name":"NumberList","word":null,"rules":{"source_file":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"REPEAT1","content":{"type":"FIELD","name":"numbers","content":{"type":"SYMBOL","name":"Number"}}}}]}]},"NumberList":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"REPEAT1","content":{"type":"FIELD","name":"numbers","content":{"type":"SYMBOL","name":"Number"}}}}]}]},"Number":{"type":"SEQ","members":[{"type":"FIELD","name":"v","content":{"type":"PATTERN","value":"\\d+"}}]}},"extras":[{"type":"PATTERN","value":"\\s"}]} diff --git a/tool/src/snapshots/rust_sitter_tool__tests__grammar_unboxed_field.snap b/tool/src/snapshots/rust_sitter_tool__tests__grammar_unboxed_field.snap index 246cfe3..1c3bfcc 100644 --- a/tool/src/snapshots/rust_sitter_tool__tests__grammar_unboxed_field.snap +++ b/tool/src/snapshots/rust_sitter_tool__tests__grammar_unboxed_field.snap @@ -1,5 +1,5 @@ --- source: tool/src/lib.rs -expression: generate_grammar(&m) +expression: grammar --- -{"name":"test","word":null,"rules":{"source_file":{"type":"SEQ","members":[{"type":"FIELD","name":"e","content":{"type":"SYMBOL","name":"Expression"}}]},"Language":{"type":"SEQ","members":[{"type":"FIELD","name":"e","content":{"type":"SYMBOL","name":"Expression"}}]},"Expression_Number_0":{"type":"PATTERN","value":"\\d+"},"Expression_Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Expression_Number_0"}}]},"Expression":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"}]}},"extras":[]} +{"name":"Language","word":null,"rules":{"source_file":{"type":"SEQ","members":[{"type":"FIELD","name":"e","content":{"type":"SYMBOL","name":"Expression"}}]},"Language":{"type":"SEQ","members":[{"type":"FIELD","name":"e","content":{"type":"SYMBOL","name":"Expression"}}]},"Expression_Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"PATTERN","value":"\\d+"}}]},"Expression":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"}]}},"extras":[]} diff --git a/tool/src/snapshots/rust_sitter_tool__tests__grammar_with_extras.snap b/tool/src/snapshots/rust_sitter_tool__tests__grammar_with_extras.snap index abc38d0..4269765 100644 --- a/tool/src/snapshots/rust_sitter_tool__tests__grammar_with_extras.snap +++ b/tool/src/snapshots/rust_sitter_tool__tests__grammar_with_extras.snap @@ -1,5 +1,5 @@ --- source: tool/src/lib.rs -expression: generate_grammar(&m) +expression: grammar --- -{"name":"test","word":null,"rules":{"source_file":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"}]},"Expression_Number_0":{"type":"PATTERN","value":"\\d+"},"Expression_Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"SYMBOL","name":"Expression_Number_0"}}]},"Expression":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"}]},"Whitespace__whitespace":{"type":"PATTERN","value":"\\s"},"Whitespace":{"type":"SEQ","members":[{"type":"FIELD","name":"_whitespace","content":{"type":"SYMBOL","name":"Whitespace__whitespace"}}]}},"extras":[{"type":"SYMBOL","name":"Whitespace"}]} +{"name":"Expression","word":null,"rules":{"source_file":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"}]},"Expression_Number":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"PATTERN","value":"\\d+"}}]},"Expression":{"type":"CHOICE","members":[{"type":"SYMBOL","name":"Expression_Number"}]}},"extras":[{"type":"PATTERN","value":"\\s"}]} diff --git a/tool/src/snapshots/rust_sitter_tool__tests__immediate.snap b/tool/src/snapshots/rust_sitter_tool__tests__immediate.snap new file mode 100644 index 0000000..a047b29 --- /dev/null +++ b/tool/src/snapshots/rust_sitter_tool__tests__immediate.snap @@ -0,0 +1,5 @@ +--- +source: tool/src/lib.rs +expression: grammar +--- +{"name":"StringFragment","word":null,"rules":{"source_file":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"PREC","value":1,"content":{"type":"PATTERN","value":"[^\"\\\\]+"}}}]},"StringFragment":{"type":"SEQ","members":[{"type":"FIELD","name":"0","content":{"type":"PREC","value":1,"content":{"type":"PATTERN","value":"[^\"\\\\]+"}}}]}},"extras":[{"type":"PATTERN","value":"\\s"}]} diff --git a/tool/src/snapshots/rust_sitter_tool__tests__spanned_in_vec.snap b/tool/src/snapshots/rust_sitter_tool__tests__spanned_in_vec.snap index c6342da..6c39108 100644 --- a/tool/src/snapshots/rust_sitter_tool__tests__spanned_in_vec.snap +++ b/tool/src/snapshots/rust_sitter_tool__tests__spanned_in_vec.snap @@ -2,4 +2,4 @@ source: tool/src/lib.rs expression: grammar --- -{"name":"test","word":null,"rules":{"source_file":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"SYMBOL","name":"NumberList_numbers_vec_contents"}}]}]},"NumberList_numbers":{"type":"PATTERN","value":"\\d+"},"NumberList_numbers_vec_contents":{"type":"REPEAT1","content":{"type":"FIELD","name":"NumberList_numbers_vec_element","content":{"type":"SYMBOL","name":"NumberList_numbers"}}},"NumberList":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"SYMBOL","name":"NumberList_numbers_vec_contents"}}]}]},"Whitespace__whitespace":{"type":"PATTERN","value":"\\s"},"Whitespace":{"type":"SEQ","members":[{"type":"FIELD","name":"_whitespace","content":{"type":"SYMBOL","name":"Whitespace__whitespace"}}]}},"extras":[{"type":"SYMBOL","name":"Whitespace"}]} +{"name":"NumberList","word":null,"rules":{"source_file":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"REPEAT1","content":{"type":"FIELD","name":"numbers","content":{"type":"PATTERN","value":"\\d+"}}}}]}]},"NumberList":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"numbers","content":{"type":"REPEAT1","content":{"type":"FIELD","name":"numbers","content":{"type":"PATTERN","value":"\\d+"}}}}]}]}},"extras":[{"type":"PATTERN","value":"\\s"}]} diff --git a/tool/src/snapshots/rust_sitter_tool__tests__struct_optional.snap b/tool/src/snapshots/rust_sitter_tool__tests__struct_optional.snap index 9f2629e..d3bd089 100644 --- a/tool/src/snapshots/rust_sitter_tool__tests__struct_optional.snap +++ b/tool/src/snapshots/rust_sitter_tool__tests__struct_optional.snap @@ -2,4 +2,4 @@ source: tool/src/lib.rs expression: grammar --- -{"name":"test","word":null,"rules":{"source_file":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"v","content":{"type":"SYMBOL","name":"Language_v"}}]},{"type":"FIELD","name":"space","content":{"type":"SYMBOL","name":"Language_space"}},{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"t","content":{"type":"SYMBOL","name":"Number"}}]}]},"Language_v":{"type":"PATTERN","value":"\\d+"},"Language_space":{"type":"PATTERN","value":" "},"Language":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"v","content":{"type":"SYMBOL","name":"Language_v"}}]},{"type":"FIELD","name":"space","content":{"type":"SYMBOL","name":"Language_space"}},{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"t","content":{"type":"SYMBOL","name":"Number"}}]}]},"Number_v":{"type":"PATTERN","value":"\\d+"},"Number":{"type":"SEQ","members":[{"type":"FIELD","name":"v","content":{"type":"SYMBOL","name":"Number_v"}}]}},"extras":[]} +{"name":"Language","word":null,"rules":{"source_file":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"v","content":{"type":"PATTERN","value":"\\d+"}}]},{"type":"FIELD","name":"space","content":{"type":"PATTERN","value":" "}},{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"t","content":{"type":"SYMBOL","name":"Number"}}]}]},"Language":{"type":"SEQ","members":[{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"v","content":{"type":"PATTERN","value":"\\d+"}}]},{"type":"FIELD","name":"space","content":{"type":"PATTERN","value":" "}},{"type":"CHOICE","members":[{"type":"BLANK"},{"type":"FIELD","name":"t","content":{"type":"SYMBOL","name":"Number"}}]}]},"Number":{"type":"SEQ","members":[{"type":"FIELD","name":"v","content":{"type":"PATTERN","value":"\\d+"}}]}},"extras":[]} diff --git a/tool/target/rust-analyzer/metadata/sysroot/Cargo.lock b/tool/target/rust-analyzer/metadata/sysroot/Cargo.lock new file mode 100644 index 0000000..b125f01 --- /dev/null +++ b/tool/target/rust-analyzer/metadata/sysroot/Cargo.lock @@ -0,0 +1,511 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "compiler_builtins", + "gimli", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-core", +] + +[[package]] +name = "alloc" +version = "0.0.0" +dependencies = [ + "compiler_builtins", + "core", +] + +[[package]] +name = "alloctests" +version = "0.0.0" +dependencies = [ + "rand", + "rand_xorshift", +] + +[[package]] +name = "cc" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aeb932158bd710538c73702db6945cb68a8fb08c519e6e12706b94263b36db8" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-core", +] + +[[package]] +name = "compiler_builtins" +version = "0.1.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "164cdc689e4c6d69417f77a5f48be240c291e84fbef0b1281755dc754b19c809" +dependencies = [ + "cc", + "rustc-std-workspace-core", +] + +[[package]] +name = "core" +version = "0.0.0" + +[[package]] +name = "coretests" +version = "0.0.0" +dependencies = [ + "rand", + "rand_xorshift", +] + +[[package]] +name = "dlmalloc" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cff88b751e7a276c4ab0e222c3f355190adc6dde9ce39c851db39da34990df7" +dependencies = [ + "cfg-if", + "compiler_builtins", + "libc", + "rustc-std-workspace-core", + "windows-sys", +] + +[[package]] +name = "fortanix-sgx-abi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57cafc2274c10fab234f176b25903ce17e690fca7597090d50880e047a0389c5" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-core", +] + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "rustc-std-workspace-core", + "rustc-std-workspace-std", + "unicode-width", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + +[[package]] +name = "hashbrown" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + +[[package]] +name = "hermit-abi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + +[[package]] +name = "libc" +version = "0.2.172" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +dependencies = [ + "rustc-std-workspace-core", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-core", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" +dependencies = [ + "adler2", + "compiler_builtins", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "compiler_builtins", + "memchr", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + +[[package]] +name = "panic_abort" +version = "0.0.0" +dependencies = [ + "alloc", + "cfg-if", + "compiler_builtins", + "core", + "libc", +] + +[[package]] +name = "panic_unwind" +version = "0.0.0" +dependencies = [ + "alloc", + "cfg-if", + "compiler_builtins", + "core", + "libc", + "unwind", +] + +[[package]] +name = "proc_macro" +version = "0.0.0" +dependencies = [ + "core", + "rustc-literal-escaper", + "std", +] + +[[package]] +name = "profiler_builtins" +version = "0.0.0" +dependencies = [ + "cc", +] + +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-core", +] + +[[package]] +name = "r-efi-alloc" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e43c53ff1a01d423d1cb762fd991de07d32965ff0ca2e4f80444ac7804198203" +dependencies = [ + "compiler_builtins", + "r-efi", + "rustc-std-workspace-core", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-core", +] + +[[package]] +name = "rustc-literal-escaper" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0041b6238913c41fe704213a4a9329e2f685a156d1781998128b4149c230ad04" +dependencies = [ + "rustc-std-workspace-std", +] + +[[package]] +name = "rustc-std-workspace-alloc" +version = "1.99.0" +dependencies = [ + "alloc", +] + +[[package]] +name = "rustc-std-workspace-core" +version = "1.99.0" +dependencies = [ + "core", +] + +[[package]] +name = "rustc-std-workspace-std" +version = "1.99.0" +dependencies = [ + "std", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "std" +version = "0.0.0" +dependencies = [ + "addr2line", + "alloc", + "cfg-if", + "compiler_builtins", + "core", + "dlmalloc", + "fortanix-sgx-abi", + "hashbrown", + "hermit-abi", + "libc", + "miniz_oxide", + "object", + "panic_abort", + "panic_unwind", + "r-efi", + "r-efi-alloc", + "rand", + "rand_xorshift", + "rustc-demangle", + "std_detect", + "unwind", + "wasi", + "windows-targets 0.0.0", +] + +[[package]] +name = "std_detect" +version = "0.1.5" +dependencies = [ + "cfg-if", + "compiler_builtins", + "libc", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + +[[package]] +name = "sysroot" +version = "0.0.0" +dependencies = [ + "proc_macro", + "profiler_builtins", + "std", + "test", +] + +[[package]] +name = "test" +version = "0.0.0" +dependencies = [ + "core", + "getopts", + "libc", + "std", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-core", + "rustc-std-workspace-std", +] + +[[package]] +name = "unwind" +version = "0.0.0" +dependencies = [ + "cfg-if", + "compiler_builtins", + "core", + "libc", + "unwinding", +] + +[[package]] +name = "unwinding" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8393f2782b6060a807337ff353780c1ca15206f9ba2424df18cb6e733bd7b345" +dependencies = [ + "compiler_builtins", + "gimli", + "rustc-std-workspace-core", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.0.0" + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[patch.unused]] +name = "tree-sitter" +version = "0.26.0" + +[[patch.unused]] +name = "tree-sitter-generate" +version = "0.26.0" diff --git a/tool/target/rust-analyzer/metadata/workspace/Cargo.lock b/tool/target/rust-analyzer/metadata/workspace/Cargo.lock new file mode 100644 index 0000000..94d3e25 --- /dev/null +++ b/tool/target/rust-analyzer/metadata/workspace/Cargo.lock @@ -0,0 +1,1129 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "cc" +version = "1.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" + +[[package]] +name = "codemap" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e769b5c8c8283982a987c6e948e540254f1058d5a74b8794914d4ef5fc2a24" + +[[package]] +name = "codemap-diagnostic" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc20770be05b566a963bf91505e60412c4a2d016d1ef95c5512823bb085a8122" +dependencies = [ + "codemap", + "termcolor", +] + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "windows-sys 0.59.0", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" + +[[package]] +name = "insta" +version = "1.43.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "154934ea70c58054b556dd430b99a98c2a7ff5309ac9891597e339b5c28f4371" +dependencies = [ + "console", + "once_cell", + "similar", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.175" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "minicov" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27fe9f1cc3c22e1687f9446c2083c4c5fc7f0bcf1c7a86bdbded14985895b4b" +dependencies = [ + "cc", + "walkdir", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61789d7719defeb74ea5fe81f2fdfdbd28a803847077cecce2ff14e1472f6f1" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rust-sitter" +version = "0.5.0" +dependencies = [ + "insta", + "rust-sitter-macro", + "serde", + "serde_json", + "tempfile", + "tree-sitter", +] + +[[package]] +name = "rust-sitter-common" +version = "0.5.0" +dependencies = [ + "itertools", + "proc-macro2", + "quote", + "serde_json", + "syn", +] + +[[package]] +name = "rust-sitter-example" +version = "0.5.0" +dependencies = [ + "codemap", + "codemap-diagnostic", + "insta", + "rust-sitter", + "rust-sitter-tool", + "wasm-bindgen-test", +] + +[[package]] +name = "rust-sitter-macro" +version = "0.5.0" +dependencies = [ + "insta", + "proc-macro2", + "quote", + "rust-sitter-common", + "syn", + "tempfile", +] + +[[package]] +name = "rust-sitter-tool" +version = "0.5.0" +dependencies = [ + "cc", + "insta", + "rust-sitter-common", + "serde", + "serde_json", + "syn", + "syn-inline-mod", + "tempfile", + "tree-sitter", + "tree-sitter-generate", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustix" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.60.2", +] + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +dependencies = [ + "serde", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.142" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + +[[package]] +name = "smallbitvec" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d31d263dd118560e1a492922182ab6ca6dc1d03a3bf54e7699993f31a4150e3f" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + +[[package]] +name = "syn" +version = "2.0.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc3fcb250e53458e712715cf74285c1f889686520d79294a9ef3bd7aa1fc619" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn-inline-mod" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fa6dca1fdb7b2ed46dd534a326725419d4fb10f23d8c85a8b2860e5eb25d0f9" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b0949c3a6c842cbde3f1686d6eea5a010516deb7085f79db747562d4102f41e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5b44b4ab9c2fdd0e0512e6bece8388e214c0749f5862b114cc5b7a25daf227" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "topological-sort" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d" + +[[package]] +name = "tree-sitter" +version = "0.26.0" +dependencies = [ + "cc", + "regex", + "regex-syntax", + "serde_json", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-generate" +version = "0.26.0" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "indoc", + "log", + "regex", + "regex-syntax", + "rustc-hash", + "semver", + "serde", + "serde_json", + "smallbitvec", + "thiserror", + "topological-sort", + "tree-sitter", + "url", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.4" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c8d5e33ca3b6d9fa3b4676d774c5778031d27a578c2b007f905acf816152c3" +dependencies = [ + "js-sys", + "minicov", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17d5042cc5fa009658f9a7333ef24291b1291a25b6382dd68862a7f3b969f69b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.3", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/types/Cargo.toml b/types/Cargo.toml new file mode 100644 index 0000000..948004d --- /dev/null +++ b/types/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "rust-sitter-types" +edition = "2024" +version.workspace = true +authors.workspace = true + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +indexmap = { version = "2", features = ["serde"] } diff --git a/types/src/grammar.rs b/types/src/grammar.rs new file mode 100644 index 0000000..729d913 --- /dev/null +++ b/types/src/grammar.rs @@ -0,0 +1,204 @@ +//! Grammar related functions. +use std::collections::HashSet; + +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; + +// NOTE: This could be useful for generating the grammar in the first place instead of just +// building json! values directly. + +/// Type for the JSON representation of a grammar, mostly copied from `tree_sitter_generate`. +#[derive(Deserialize, Serialize)] +pub struct Grammar { + pub name: String, + pub word: Option, + pub rules: IndexMap, + pub extras: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(tag = "type")] +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +pub enum RuleDef { + ALIAS { + content: Box, + named: bool, + value: String, + }, + BLANK, + STRING { + value: String, + }, + PATTERN { + value: String, + #[serde(skip_serializing_if = "Option::is_none")] + flags: Option, + }, + SYMBOL { + name: String, + }, + CHOICE { + members: Vec, + }, + FIELD { + name: String, + content: Box, + }, + SEQ { + members: Vec, + }, + REPEAT { + content: Box, + }, + REPEAT1 { + content: Box, + }, + PREC_DYNAMIC { + value: i32, + content: Box, + }, + PREC_LEFT { + value: PrecedenceValue, + content: Box, + }, + PREC_RIGHT { + value: PrecedenceValue, + content: Box, + }, + PREC { + value: PrecedenceValue, + content: Box, + }, + TOKEN { + content: Box, + }, + IMMEDIATE_TOKEN { + content: Box, + }, + RESERVED { + context_name: String, + content: Box, + }, +} + +impl RuleDef { + pub fn is_symbol(&self) -> bool { + matches!(self, RuleDef::SYMBOL { .. }) + } + + pub fn is_blank(&self) -> bool { + matches!(self, RuleDef::BLANK) + } + + pub fn optional(rule: RuleDef) -> RuleDef { + RuleDef::CHOICE { + members: vec![RuleDef::BLANK, rule], + } + } + + pub fn as_optional(&self) -> Option<&RuleDef> { + match self { + Self::CHOICE { members } => match members.as_slice() { + &[ref rule, RuleDef::BLANK] | &[RuleDef::BLANK, ref rule] => Some(rule), + _ => None, + }, + Self::PREC { value: _, content } + | Self::PREC_LEFT { value: _, content } + | Self::PREC_RIGHT { value: _, content } + | Self::PREC_DYNAMIC { value: _, content } => content.as_optional(), + _ => None, + } + } + + /// Pull out a sequence, including through precedence unwrapping. + pub fn as_seq(&self) -> Option<&[RuleDef]> { + match self { + Self::SEQ { members } => Some(members), + Self::PREC { value: _, content } + | Self::PREC_LEFT { value: _, content } + | Self::PREC_RIGHT { value: _, content } + | Self::PREC_DYNAMIC { value: _, content } => content.as_seq(), + _ => None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(untagged)] +pub enum PrecedenceValue { + Integer(i32), + Name(String), +} + +impl From for PrecedenceValue { + fn from(value: i32) -> Self { + Self::Integer(value) + } +} + +impl Grammar { + /// Starting from `rule_name`, find all symbols (named or anonymous) which can be reached. + pub fn reachable_set<'a>(&'a self, rule_name: &str) -> Option> { + let mut set = HashSet::new(); + let (name, rule) = self.rules.get_key_value(rule_name)?; + set.insert(name.as_str()); + self.compute_reachable(rule, &mut set)?; + Some(set) + } + + fn compute_reachable<'a>( + &'a self, + rule: &'a RuleDef, + set: &mut HashSet<&'a str>, + ) -> Option<()> { + match rule { + RuleDef::ALIAS { + content, + named: _, + value, + } => { + if set.insert(value) { + self.compute_reachable(content, set)?; + } + } + RuleDef::BLANK => {} + RuleDef::STRING { value } => { + set.insert(value.as_str()); + } + RuleDef::PATTERN { value: _, flags: _ } => {} + RuleDef::SYMBOL { name } => { + // Don't repeat if we have already seen it before. + if set.insert(name.as_str()) { + let rule = self.rules.get(name)?; + self.compute_reachable(rule, set)?; + } + } + RuleDef::CHOICE { members } => { + for member in members { + self.compute_reachable(member, set)?; + } + } + RuleDef::FIELD { name: _, content } => self.compute_reachable(content, set)?, + RuleDef::SEQ { members } => { + for member in members { + self.compute_reachable(member, set)?; + } + } + RuleDef::REPEAT { content } => self.compute_reachable(content, set)?, + RuleDef::REPEAT1 { content } => self.compute_reachable(content, set)?, + RuleDef::PREC_DYNAMIC { value: _, content } => self.compute_reachable(content, set)?, + RuleDef::PREC_LEFT { value: _, content } => self.compute_reachable(content, set)?, + RuleDef::PREC_RIGHT { value: _, content } => self.compute_reachable(content, set)?, + RuleDef::PREC { value: _, content } => self.compute_reachable(content, set)?, + RuleDef::TOKEN { content } => self.compute_reachable(content, set)?, + RuleDef::IMMEDIATE_TOKEN { content } => self.compute_reachable(content, set)?, + RuleDef::RESERVED { + context_name: _, + content, + } => self.compute_reachable(content, set)?, + } + + Some(()) + } +} diff --git a/types/src/lib.rs b/types/src/lib.rs new file mode 100644 index 0000000..38f3afc --- /dev/null +++ b/types/src/lib.rs @@ -0,0 +1 @@ +pub mod grammar;