From febe075f53586ca3c8686e611bf18b4c2e421453 Mon Sep 17 00:00:00 2001 From: Iktahana <171251543+Iktahana@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:30:13 +0900 Subject: [PATCH] feat: resolve MDI source spans --- docs/src/content/docs/bindings/javascript.md | 14 +- docs/src/content/docs/bindings/rust.md | 9 + docs/src/content/docs/core/rust-api.md | 13 +- .../content/docs/ja/bindings/javascript.md | 13 +- docs/src/content/docs/ja/bindings/rust.md | 8 + docs/src/content/docs/ja/core/rust-api.md | 7 +- .../content/docs/zh-tw/bindings/javascript.md | 12 +- docs/src/content/docs/zh-tw/bindings/rust.md | 7 + docs/src/content/docs/zh-tw/core/rust-api.md | 7 +- mdi-core/README.md | 5 + mdi-core/src/lib.rs | 32 ++- mdi-core/src/text_projection.rs | 271 ++++++++++++++++++ mdi-core/tests/source_span_resolution.rs | 218 ++++++++++++++ nodejs/browser-test/src.ts | 6 +- nodejs/packages/mdi-core/README.md | 6 + nodejs/packages/mdi-core/src/browser.d.ts | 1 + nodejs/packages/mdi-core/src/browser.js | 1 + nodejs/packages/mdi-core/src/node.cjs | 1 + nodejs/packages/mdi/README.md | 15 +- nodejs/packages/mdi/src/index.test.ts | 44 ++- nodejs/packages/mdi/src/index.ts | 84 ++++++ .../mdi/src/source-span-version.test.ts | 24 ++ nodejs/scripts/test-browser-wasm.mjs | 15 +- nodejs/scripts/test-safari-wasm.mjs | 2 + 24 files changed, 795 insertions(+), 20 deletions(-) create mode 100644 mdi-core/tests/source_span_resolution.rs create mode 100644 nodejs/packages/mdi/src/source-span-version.test.ts diff --git a/docs/src/content/docs/bindings/javascript.md b/docs/src/content/docs/bindings/javascript.md index 65e5666..c86c96c 100644 --- a/docs/src/content/docs/bindings/javascript.md +++ b/docs/src/content/docs/bindings/javascript.md @@ -60,7 +60,7 @@ Ruby readings remain searchable annotations whose `anchor` points back to the base-text range. ```ts -import { getMdiTextBlocks, sourceSpansForTextRange } from "@illusions-lab/mdi"; +import { getMdiTextBlocks, resolveMdiSourceSpan, sourceSpansForTextRange } from "@illusions-lab/mdi"; const result = getMdiTextBlocks("# 題\n\n{東京|とうきょう}"); const paragraph = result.blocks[1]; @@ -69,6 +69,7 @@ const match = { start: "2:1", end: "2:3" } as const; console.log(paragraph.text); // 東京 console.log(paragraph.annotations[0].text); // とうきょう console.log(sourceSpansForTextRange(paragraph, match)); // UTF-8 source spans +console.log(resolveMdiSourceSpan("# 題\n\n{東京|とうきょう}", { startByte: 8, endByte: 14 })); ``` `sourceMap.synthetic` identifies separators added by the projection, such as @@ -76,6 +77,17 @@ table tabs and row newlines; these deliberately produce no source span. `parseMdiTextPosition`, `formatMdiTextPosition`, and `formatMdiTextRange` are stateless helpers for the canonical coordinate spelling. +`resolveMdiSourceSpan(source, span)` performs the inverse lookup in Rust. The +span is half-open UTF-8 bytes and must use uint32 values in source order, stay +within the source, and end on code-point boundaries. It returns ordered +`blockText` and zero-based `annotation` matches plus `complete`, `partial`, or +`none` coverage. Ruby base/readings are independent channels. A match is +`exact` only when its full forward coverage equals the input; otherwise it is +`overlap`. Empty spans return no caret-like neighbor. Structural delimiters, +synthetic separators, and unmapped text produce no invented canonical range, +so forward/reverse mapping is not promised to be bijective across annotations, +multi-to-one tokens, partial graphemes, discontinuities, or unmapped text. + ## Choose the export level The one-argument EPUB and DOCX calls are synchronous Rust baseline exports: diff --git a/docs/src/content/docs/bindings/rust.md b/docs/src/content/docs/bindings/rust.md index f75941d..d8d49f6 100644 --- a/docs/src/content/docs/bindings/rust.md +++ b/docs/src/content/docs/bindings/rust.md @@ -63,6 +63,15 @@ match render_pdf(source, &PdfOptions::default()) { `MDI_IR_VERSION` and `MDI_SPEC_VERSION` are `&'static str` constants exported directly — check them if you're persisting a `ParseOutput` and reloading it later, the same way any other binding must. `SourceSpan { start_byte: u32, end_byte: u32 }` is a half-open UTF-8 byte range, exactly as described in [Diagnostics and UTF-8 source spans](/core/diagnostics/) — being in Rust doesn't change the unit; it's still bytes, not `char` indices, because `str` in Rust is itself UTF-8 bytes and indexing by anything else would require an extra pass every binding would have to pay for. +For searchable canonical text, use `get_mdi_text_blocks(source)`. Its inverse, +`resolve_mdi_source_span(source, span)`, validates ordering, bounds, and UTF-8 +boundaries, then returns all maximal block-text and annotation grapheme ranges. +Coverage is `Complete`, `Partial`, or `None`; matches are `Exact` only when the +complete forward coverage equals the input. Empty spans return no matches, and +structural, synthetic, or unmapped bytes do not gain invented ranges. Ruby's +separate channels and multi-to-one/discontinuous mappings mean round trips are +not generally bijective. + ## Current implementation status Parsing (`parse_document`/`parse_output`), serialization (`serialize_mdi`), and every renderer (`render_html`, `render_text_format`, `render_epub`, `render_docx`, `render_pdf`) are implemented today, at the "baseline" level described on [Rust Core API status](/core/rust-api/#not-yet-implemented). There is no separate `validate`/`normalize` API distinct from `parse_output`/`serialize_mdi` — see that same page for exactly what's missing. diff --git a/docs/src/content/docs/core/rust-api.md b/docs/src/content/docs/core/rust-api.md index fe14309..70510b6 100644 --- a/docs/src/content/docs/core/rust-api.md +++ b/docs/src/content/docs/core/rust-api.md @@ -46,13 +46,24 @@ This page lists only symbols present in [`mdi-core/src/lib.rs`](https://github.c ## Public data types -`ParseOutput`, `ParserCapabilities`, `Diagnostic`, `DiagnosticSeverity`, `SourceSpan`, `Document`, `Frontmatter`, `FrontmatterEntry`, `MdiTextBlocksResult`, `MdiTextBlock`, `MdiTextPosition`, `MdiTextRange`, `MdiTextSourceMap`, `MdiTextSourceRun`, `MdiTextAnnotation`, `PdfOptions`, `EpubCover`, `ResolvedExportProfile` and its nested profile/Chromium print types (current-generation API); `MdiSyntaxDocument`, `MdiBlock`, `PagebreakVariant`, `Inline`, `RubyReading` (the older, `parse_mdi_syntax`-only shape — `Inline`/`RubyReading` are also reused internally to build the current-generation `Document`'s MDI nodes, but their `serde` output is what appears inside `Document.children`, not `MdiSyntaxDocument`). +`ParseOutput`, `ParserCapabilities`, `Diagnostic`, `DiagnosticSeverity`, `SourceSpan`, `Document`, `Frontmatter`, `FrontmatterEntry`, `MdiTextBlocksResult`, `MdiTextBlock`, `MdiTextPosition`, `MdiTextRange`, `MdiTextSourceMap`, `MdiTextSourceRun`, `MdiTextAnnotation`, `MdiSourceSpanTextResolution`, `MdiSourceSpanTextMatch`, `MdiSourceSpanCoverage`, `MdiSourceSpanRelation`, `MdiSourceSpanResolutionError`, `PdfOptions`, `EpubCover`, `ResolvedExportProfile` and its nested profile/Chromium print types (current-generation API); `MdiSyntaxDocument`, `MdiBlock`, `PagebreakVariant`, `Inline`, `RubyReading` (the older, `parse_mdi_syntax`-only shape — `Inline`/`RubyReading` are also reused internally to build the current-generation `Document`'s MDI nodes, but their `serde` output is what appears inside `Document.children`, not `MdiSyntaxDocument`). Use `get_mdi_text_blocks(source)` or `get_mdi_text_blocks_json(source)` for the Rust-owned plaintext search projection. It returns source-order blocks with one-based Unicode-grapheme positions, UTF-8 source-map boundaries, ruby reading annotations, and the same document/diagnostic envelope as `parse_output`. +Use `resolve_mdi_source_span(source, span)` to map a validated half-open UTF-8 +`SourceSpan` back to maximal canonical grapheme ranges. It returns block text +before zero-based annotation channels in deterministic block order. Coverage +is `Complete`, `Partial`, or `None`, and a range is `Exact` only when its full +forward source coverage equals the requested span. Reversed, out-of-bounds, or +non-code-point-boundary inputs return `MdiSourceSpanResolutionError`; an empty +span is valid and has no matches. Pure structural, synthetic, and unmapped +bytes do not create ranges. Ruby's two channels, multi-to-one tokens, partial +graphemes, discontinuous mappings, and unmapped text mean this is not a general +inverse of every forward lookup. + ## Not yet implemented These exist as concepts in `ARCHITECTURE.md`/`SYNTAX.md` but have **no corresponding function in `mdi-core` today** — don't assume they exist because the architecture diagram mentions the concept: diff --git a/docs/src/content/docs/ja/bindings/javascript.md b/docs/src/content/docs/ja/bindings/javascript.md index f35cb2d..8f649ab 100644 --- a/docs/src/content/docs/ja/bindings/javascript.md +++ b/docs/src/content/docs/ja/bindings/javascript.md @@ -58,7 +58,7 @@ block、完全な document IR、diagnostics を返します。`3:18` は三つ annotation として検索でき、`anchor` は base text の range を指します。 ```ts -import { getMdiTextBlocks, sourceSpansForTextRange } from "@illusions-lab/mdi"; +import { getMdiTextBlocks, resolveMdiSourceSpan, sourceSpansForTextRange } from "@illusions-lab/mdi"; const result = getMdiTextBlocks("# 題\n\n{東京|とうきょう}"); const paragraph = result.blocks[1]; @@ -67,6 +67,7 @@ const match = { start: "2:1", end: "2:3" } as const; console.log(paragraph.text); // 東京 console.log(paragraph.annotations[0].text); // とうきょう console.log(sourceSpansForTextRange(paragraph, match)); // UTF-8 source span +console.log(resolveMdiSourceSpan("# 題\n\n{東京|とうきょう}", { startByte: 8, endByte: 14 })); ``` `sourceMap.synthetic` は table の tab や row newline など projection が追加した @@ -74,6 +75,16 @@ separator を示し、source span は作りません。`parseMdiTextPosition`、 `formatMdiTextPosition`、`formatMdiTextRange` は canonical な座標表記用の stateless helper です。 +`resolveMdiSourceSpan(source, span)` は Rust で逆引きします。入力は half-open +UTF-8 byte の uint32 で、source 内・昇順・code-point boundary でなければ +なりません。結果は block 順、本文優先、zero-based annotation 順の canonical +range と `complete | partial | none` coverage です。forward coverage 全体が入力と +一致する match だけが `exact`、それ以外は `overlap` です。ruby base と reading +は別 channel です。空 span は caret と解釈せず match を返しません。純粋な構造 +delimiter、synthetic、unmapped byte に range は作られないため、annotation、 +multi-to-one token、partial grapheme、discontinuous mapping を含む round trip は +一般には bijection ではありません。 + ## baseline と設定付き EPUB/DOCX 一引数の API は synchronous Rust baseline export です。 diff --git a/docs/src/content/docs/ja/bindings/rust.md b/docs/src/content/docs/ja/bindings/rust.md index e960644..2166e16 100644 --- a/docs/src/content/docs/ja/bindings/rust.md +++ b/docs/src/content/docs/ja/bindings/rust.md @@ -49,6 +49,14 @@ match mdi_core::render_pdf(source, &mdi_core::PdfOptions::default()) { `MDI_IR_VERSION` と `MDI_SPEC_VERSION` は exported constant です。永続化した `ParseOutput` を読み直すなら version を確認してください。`SourceSpan { start_byte, end_byte }` は UTF-8 byte の半開 range で、`char` index ではありません。 +検索用 canonical text は `get_mdi_text_blocks(source)`、逆引きは +`resolve_mdi_source_span(source, span)` を使います。後者は順序、範囲、UTF-8 +boundary を検証し、本文と annotation の maximal grapheme range、 +`Complete | Partial | None` coverage、`Exact | Overlap` relation を返します。 +空 span、純構造 delimiter、synthetic、unmapped source は range を作りません。 +Ruby の別 channel や multi-to-one/discontinuous mapping により、round trip は +一般に bijection ではありません。 + ## 現在の実装状況 parse、`serialize_mdi`、HTML/TXT/EPUB/DOCX/PDF renderer はすべて実装済みです。baseline の正確な範囲は [Rust Core API](/ja/core/rust-api/#not-yet-implemented) を参照してください。 diff --git a/docs/src/content/docs/ja/core/rust-api.md b/docs/src/content/docs/ja/core/rust-api.md index d015b6b..579c8ad 100644 --- a/docs/src/content/docs/ja/core/rust-api.md +++ b/docs/src/content/docs/ja/core/rust-api.md @@ -44,7 +44,12 @@ description: 現在 mdi-core/src/lib.rs に実在する public symbol の一覧 ## Public data types -current-generation API は `ParseOutput`、`ParserCapabilities`、`Diagnostic`、`SourceSpan`、`Document`、`Frontmatter`、`PdfOptions`、`EpubCover`、`ResolvedExportProfile` とその nested profile/Chromium print types です。旧 shape は `MdiSyntaxDocument`、`MdiBlock`、`PagebreakVariant`、`Inline`、`RubyReading` です。 +current-generation API は `ParseOutput`、`ParserCapabilities`、`Diagnostic`、`SourceSpan`、`Document`、`Frontmatter`、`MdiTextBlocksResult`、`MdiSourceSpanTextResolution`、`MdiSourceSpanTextMatch`、`MdiSourceSpanCoverage`、`MdiSourceSpanRelation`、`MdiSourceSpanResolutionError`、`PdfOptions`、`EpubCover`、`ResolvedExportProfile` とその nested profile/Chromium print types です。旧 shape は `MdiSyntaxDocument`、`MdiBlock`、`PagebreakVariant`、`Inline`、`RubyReading` です。 + +`get_mdi_text_blocks(source)` は grapheme 単位の canonical text projection を返し、 +`resolve_mdi_source_span(source, span)` は half-open UTF-8 source span を本文と ruby +annotation の range に逆引きします。coverage、relation、boundary、delimiter、 +synthetic/unmapped、round-trip の制約は [Rust binding](/ja/bindings/rust/) を参照してください。 ## Not yet implemented diff --git a/docs/src/content/docs/zh-tw/bindings/javascript.md b/docs/src/content/docs/zh-tw/bindings/javascript.md index 7d31782..16a8ce5 100644 --- a/docs/src/content/docs/zh-tw/bindings/javascript.md +++ b/docs/src/content/docs/zh-tw/bindings/javascript.md @@ -56,7 +56,7 @@ Unicode grapheme。Ruby 讀音會作為可搜尋的獨立 annotation channel， 仍指回正文 base text 的 range。 ```ts -import { getMdiTextBlocks, sourceSpansForTextRange } from "@illusions-lab/mdi"; +import { getMdiTextBlocks, resolveMdiSourceSpan, sourceSpansForTextRange } from "@illusions-lab/mdi"; const result = getMdiTextBlocks("# 題\n\n{東京|とうきょう}"); const paragraph = result.blocks[1]; @@ -65,12 +65,22 @@ const match = { start: "2:1", end: "2:3" } as const; console.log(paragraph.text); // 東京 console.log(paragraph.annotations[0].text); // とうきょう console.log(sourceSpansForTextRange(paragraph, match)); // UTF-8 source spans +console.log(resolveMdiSourceSpan("# 題\n\n{東京|とうきょう}", { startByte: 8, endByte: 14 })); ``` `sourceMap.synthetic` 指出 projection 額外加入的 separator，例如 table 的 tab 與 row newline；它們不會偽造 source span。`parseMdiTextPosition`、 `formatMdiTextPosition`、`formatMdiTextRange` 是 canonical 座標格式的無狀態 helper。 +`resolveMdiSourceSpan(source, span)` 由 Rust 執行反向解析。輸入是 half-open UTF-8 +byte uint32，必須有序、位於 source 範圍內且落在 code-point boundaries。結果依 +block、正文優先、零基底 annotation index 排序，coverage 為 +`complete | partial | none`。只有 match 的完整 forward coverage 恰等於輸入時 +才是 `exact`，其餘交集為 `overlap`。Ruby base 與 reading 是獨立 channel；空 +span 不視為 caret，也不回傳鄰近 range。純結構 delimiter、synthetic 與 unmapped +byte 不會取得虛構 range，因此 annotation、多對一 token、partial grapheme、 +discontinuous mapping 等情況不保證 round trip 是雙射。 + ## baseline 與可設定 EPUB/DOCX 一個參數的 API 是 synchronous Rust baseline export： diff --git a/docs/src/content/docs/zh-tw/bindings/rust.md b/docs/src/content/docs/zh-tw/bindings/rust.md index bbb8d1d..e459238 100644 --- a/docs/src/content/docs/zh-tw/bindings/rust.md +++ b/docs/src/content/docs/zh-tw/bindings/rust.md @@ -50,6 +50,13 @@ match render_pdf(source, &PdfOptions::default()) { `MDI_IR_VERSION` 與 `MDI_SPEC_VERSION` 為 exported `&'static str` constants；儲存 `ParseOutput` 後再載入時應檢查。`SourceSpan { start_byte: u32, end_byte: u32 }` 是 half-open UTF-8 byte range，詳見[診斷](/zh-tw/core/diagnostics/)。 +搜尋用 canonical text 可用 `get_mdi_text_blocks(source)`；反向查詢使用 +`resolve_mdi_source_span(source, span)`。它會驗證順序、範圍與 UTF-8 boundaries， +回傳正文及 annotation 的最大 grapheme ranges、`Complete | Partial | None` +coverage 和 `Exact | Overlap` relation。空 span、純結構 delimiter、synthetic 與 +unmapped source 都不會產生 range。Ruby 雙 channel 與多對一／不連續 mapping +代表 round trip 通常不是雙射。 + ## 目前實作狀態 Parsing、`serialize_mdi` 及所有 renderer（`render_html`、`render_text_format`、`render_epub`、`render_docx`、`render_pdf`）皆已實作，限制見 [Rust Core API 尚未實作項目](/zh-tw/core/rust-api/#尚未實作)。沒有獨立 `validate`/`normalize` API，分別由 `parse_output`/`serialize_mdi` 擔任。 diff --git a/docs/src/content/docs/zh-tw/core/rust-api.md b/docs/src/content/docs/zh-tw/core/rust-api.md index 8f3698e..21490df 100644 --- a/docs/src/content/docs/zh-tw/core/rust-api.md +++ b/docs/src/content/docs/zh-tw/core/rust-api.md @@ -46,7 +46,12 @@ description: "`mdi-core/src/lib.rs` 現在實際公開的所有 symbol，包含 ## Public data types -`ParseOutput`、`ParserCapabilities`、`Diagnostic`、`DiagnosticSeverity`、`SourceSpan`、`Document`、`Frontmatter`、`FrontmatterEntry`、`PdfOptions`、`EpubCover`、`ResolvedExportProfile` 與其 nested profile/Chromium print types（目前 API）；`MdiSyntaxDocument`、`MdiBlock`、`PagebreakVariant`、`Inline`、`RubyReading`（較舊、只供 `parse_mdi_syntax` 的 shape；`Inline`/`RubyReading` 也在內部用來建立目前 `Document` 的 MDI nodes）。 +`ParseOutput`、`ParserCapabilities`、`Diagnostic`、`DiagnosticSeverity`、`SourceSpan`、`Document`、`Frontmatter`、`FrontmatterEntry`、`MdiTextBlocksResult`、`MdiSourceSpanTextResolution`、`MdiSourceSpanTextMatch`、`MdiSourceSpanCoverage`、`MdiSourceSpanRelation`、`MdiSourceSpanResolutionError`、`PdfOptions`、`EpubCover`、`ResolvedExportProfile` 與其 nested profile/Chromium print types（目前 API）；`MdiSyntaxDocument`、`MdiBlock`、`PagebreakVariant`、`Inline`、`RubyReading`（較舊、只供 `parse_mdi_syntax` 的 shape；`Inline`/`RubyReading` 也在內部用來建立目前 `Document` 的 MDI nodes）。 + +`get_mdi_text_blocks(source)` 提供 grapheme canonical text projection； +`resolve_mdi_source_span(source, span)` 把 half-open UTF-8 source span 反解為正文與 +ruby annotation ranges。Coverage、relation、boundary、delimiter、synthetic/unmapped +及 round-trip 限制請參閱 [Rust 綁定](/zh-tw/bindings/rust/)。 ## 尚未實作 diff --git a/mdi-core/README.md b/mdi-core/README.md index f3e5d1d..15cd87e 100644 --- a/mdi-core/README.md +++ b/mdi-core/README.md @@ -30,6 +30,11 @@ Use `get_mdi_text_blocks` when a search or annotation index needs source-order plaintext blocks with one-based Unicode-grapheme coordinates and exact UTF-8 source maps. The result includes the same document IR and diagnostics, and the source is parsed only once. +Use `resolve_mdi_source_span` for the inverse lookup: a validated half-open +UTF-8 source span becomes ordered canonical block-text and ruby-annotation +ranges. Its `complete`, `partial`, or `none` coverage reports whether every +requested source byte belongs to at least one mapped grapheme; synthetic and +unmapped projection text never creates a match. When rendering one parsed document in multiple formats, use the `*_document` functions, such as `render_html_document`, to avoid parsing it again. diff --git a/mdi-core/src/lib.rs b/mdi-core/src/lib.rs index 72ea848..67ccfcf 100644 --- a/mdi-core/src/lib.rs +++ b/mdi-core/src/lib.rs @@ -26,9 +26,12 @@ pub use publication_profile::{ prepare_chromium_print_profile_resolved, resolve_export_profile, resolve_export_profile_json, }; pub use text_projection::{ - MDI_TEXT_PROJECTION_VERSION, MdiAnnotationSourceMap, MdiTextAnnotation, MdiTextBlock, - MdiTextBlockKind, MdiTextBlocksResult, MdiTextPosition, MdiTextRange, MdiTextSourceMap, - MdiTextSourceRun, get_mdi_text_blocks, get_mdi_text_blocks_json, + MDI_TEXT_PROJECTION_VERSION, MdiAnnotationSourceMap, MdiSourceSpanCoverage, + MdiSourceSpanRelation, MdiSourceSpanResolutionError, MdiSourceSpanTextMatch, + MdiSourceSpanTextResolution, MdiTextAnnotation, MdiTextBlock, MdiTextBlockKind, + MdiTextBlocksResult, MdiTextPosition, MdiTextRange, MdiTextSourceMap, MdiTextSourceRun, + get_mdi_text_blocks, get_mdi_text_blocks_json, resolve_mdi_source_span, + resolve_mdi_source_span_json, }; /// MDI syntax version implemented by this crate. @@ -3984,12 +3987,12 @@ fn classify_block_macro(source: &str) -> BlockMacroClass { #[cfg(feature = "wasm")] mod wasm { use super::{ - BlockMacroClass, EpubCover, PagebreakVariant, RubyReading, TextFormat, + BlockMacroClass, EpubCover, PagebreakVariant, RubyReading, SourceSpan, TextFormat, apply_pdf_profile_json, classify_block_macro, get_mdi_text_blocks_json, page_size_catalog_json, parse_json, prepare_chromium_print_profile_json, render_docx, render_docx_with_profile, render_epub, render_epub_with_profile, render_html, render_text, - render_text_format, resolve_export_profile_json, serialize_mdi, split_ruby, unescape_mdi, - unescape_ruby, + render_text_format, resolve_export_profile_json, resolve_mdi_source_span_json, + serialize_mdi, split_ruby, unescape_mdi, unescape_ruby, }; use wasm_bindgen::prelude::*; @@ -4007,6 +4010,23 @@ mod wasm { get_mdi_text_blocks_json(source) } + /// Resolve a half-open UTF-8 source span to canonical text ranges in Rust. + #[wasm_bindgen(js_name = resolveMdiSourceSpanJson)] + pub fn wasm_resolve_mdi_source_span_json( + source: &str, + start_byte: u32, + end_byte: u32, + ) -> Result { + resolve_mdi_source_span_json( + source, + SourceSpan { + start_byte, + end_byte, + }, + ) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + /// Render source through the Rust parser and Rust HTML renderer. #[wasm_bindgen(js_name = renderHtml)] pub fn wasm_render_html(source: &str) -> String { diff --git a/mdi-core/src/text_projection.rs b/mdi-core/src/text_projection.rs index bb676a9..95d6b87 100644 --- a/mdi-core/src/text_projection.rs +++ b/mdi-core/src/text_projection.rs @@ -3,6 +3,7 @@ use crate::{ SourceSpan, diagnostics, parse_document, }; use serde::Serialize; +use std::fmt; use unicode_segmentation::UnicodeSegmentation; pub(crate) enum PlainInline<'a> { @@ -135,6 +136,75 @@ pub struct MdiTextAnnotation { pub source_map: MdiAnnotationSourceMap, } +/// Result of resolving one half-open UTF-8 source span back to canonical text. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MdiSourceSpanTextResolution { + pub projection_version: &'static str, + pub source_span: SourceSpan, + pub coverage: MdiSourceSpanCoverage, + pub matches: Vec, +} + +/// How much of a non-empty source span belongs to mapped graphemes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum MdiSourceSpanCoverage { + Complete, + Partial, + None, +} + +/// Relationship between a canonical match's forward source coverage and the +/// requested source span. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum MdiSourceSpanRelation { + Exact, + Overlap, +} + +/// A maximal adjacent canonical range in either block text or one annotation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] +pub enum MdiSourceSpanTextMatch { + BlockText { + block_index: u32, + range: MdiTextRange, + relation: MdiSourceSpanRelation, + }, + Annotation { + block_index: u32, + annotation_index: u32, + range: MdiTextRange, + relation: MdiSourceSpanRelation, + }, +} + +/// Validation error returned by [`resolve_mdi_source_span`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MdiSourceSpanResolutionError { + Reversed, + OutOfBounds, + NotUtf8Boundary, +} + +impl fmt::Display for MdiSourceSpanResolutionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Reversed => "source span startByte must not exceed endByte", + Self::OutOfBounds => "source span falls outside the UTF-8 source length", + Self::NotUtf8Boundary => "source span endpoints must be UTF-8 code-point boundaries", + }) + } +} + +impl std::error::Error for MdiSourceSpanResolutionError {} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum UnitMap { Mapped(SourceSpan), @@ -263,6 +333,207 @@ pub fn get_mdi_text_blocks_json(source: &str) -> String { .expect("serializing the MDI text projection cannot fail") } +/// Resolve a half-open UTF-8 source span to every mapped canonical grapheme +/// range. Block text and annotation text are independent channels. +pub fn resolve_mdi_source_span( + source: &str, + span: SourceSpan, +) -> Result { + validate_source_span(source, span)?; + if span.start_byte == span.end_byte { + return Ok(MdiSourceSpanTextResolution { + projection_version: MDI_TEXT_PROJECTION_VERSION, + source_span: span, + coverage: MdiSourceSpanCoverage::None, + matches: Vec::new(), + }); + } + + let projection = get_mdi_text_blocks(source); + let mut matches = Vec::new(); + let mut covered = Vec::new(); + for block in &projection.blocks { + resolve_source_map_channel( + block.index, + None, + &block.source_map, + span, + &mut matches, + &mut covered, + ); + for (annotation_index, annotation) in block.annotations.iter().enumerate() { + resolve_source_map_channel( + block.index, + Some(annotation_index as u32), + &annotation.source_map, + span, + &mut matches, + &mut covered, + ); + } + } + + let covered = merged_intervals(covered); + let coverage = if covered.is_empty() { + MdiSourceSpanCoverage::None + } else if covered.len() == 1 + && covered[0].start_byte == span.start_byte + && covered[0].end_byte == span.end_byte + { + MdiSourceSpanCoverage::Complete + } else { + MdiSourceSpanCoverage::Partial + }; + Ok(MdiSourceSpanTextResolution { + projection_version: MDI_TEXT_PROJECTION_VERSION, + source_span: span, + coverage, + matches, + }) +} + +/// JSON boundary for language bindings. +pub fn resolve_mdi_source_span_json( + source: &str, + span: SourceSpan, +) -> Result { + let resolution = resolve_mdi_source_span(source, span)?; + Ok(serde_json::to_string(&resolution) + .expect("serializing an MDI source-span resolution cannot fail")) +} + +fn validate_source_span( + source: &str, + span: SourceSpan, +) -> Result<(), MdiSourceSpanResolutionError> { + if span.start_byte > span.end_byte { + return Err(MdiSourceSpanResolutionError::Reversed); + } + let start = span.start_byte as usize; + let end = span.end_byte as usize; + if end > source.len() { + return Err(MdiSourceSpanResolutionError::OutOfBounds); + } + if !source.is_char_boundary(start) || !source.is_char_boundary(end) { + return Err(MdiSourceSpanResolutionError::NotUtf8Boundary); + } + Ok(()) +} + +fn resolve_source_map_channel( + block_index: u32, + annotation_index: Option, + map: &MdiTextSourceMap, + requested: SourceSpan, + matches: &mut Vec, + covered: &mut Vec, +) { + let mut current_start = None; + let mut current_end = 0; + let mut current_spans = Vec::new(); + + let flush = |start: &mut Option, + end: &mut u32, + spans: &mut Vec, + matches: &mut Vec| { + let Some(start_character) = start.take() else { + return; + }; + let relation = if intervals_equal_span(spans, requested) { + MdiSourceSpanRelation::Exact + } else { + MdiSourceSpanRelation::Overlap + }; + let range = MdiTextRange { + start: MdiTextPosition { + block: block_index, + character: start_character, + }, + end: MdiTextPosition { + block: block_index, + character: *end, + }, + }; + matches.push(match annotation_index { + Some(annotation_index) => MdiSourceSpanTextMatch::Annotation { + block_index, + annotation_index, + range, + relation, + }, + None => MdiSourceSpanTextMatch::BlockText { + block_index, + range, + relation, + }, + }); + spans.clear(); + }; + + for run in &map.runs { + let run_start = run.range.start.character; + for (offset, boundaries) in run.source_boundaries.windows(2).enumerate() { + let character = run_start + offset as u32; + let unit_span = SourceSpan { + start_byte: boundaries[0], + end_byte: boundaries[1], + }; + if unit_span.start_byte < requested.end_byte + && requested.start_byte < unit_span.end_byte + { + if current_start.is_some() && character != current_end { + flush( + &mut current_start, + &mut current_end, + &mut current_spans, + matches, + ); + } + current_start.get_or_insert(character); + current_end = character + 1; + current_spans.push(unit_span); + covered.push(SourceSpan { + start_byte: unit_span.start_byte.max(requested.start_byte), + end_byte: unit_span.end_byte.min(requested.end_byte), + }); + } else if current_start.is_some() && character == current_end { + flush( + &mut current_start, + &mut current_end, + &mut current_spans, + matches, + ); + } + } + } + flush( + &mut current_start, + &mut current_end, + &mut current_spans, + matches, + ); +} + +fn intervals_equal_span(intervals: &[SourceSpan], span: SourceSpan) -> bool { + let merged = merged_intervals(intervals.to_vec()); + merged.len() == 1 && merged[0] == span +} + +fn merged_intervals(mut intervals: Vec) -> Vec { + intervals.sort_unstable_by_key(|span| (span.start_byte, span.end_byte)); + let mut merged: Vec = Vec::new(); + for interval in intervals { + if let Some(previous) = merged.last_mut() + && interval.start_byte <= previous.end_byte + { + previous.end_byte = previous.end_byte.max(interval.end_byte); + } else { + merged.push(interval); + } + } + merged +} + impl Collector<'_> { fn collect(&mut self, node: &serde_json::Value, quoted: bool) { let kind = node_type(node); diff --git a/mdi-core/tests/source_span_resolution.rs b/mdi-core/tests/source_span_resolution.rs new file mode 100644 index 0000000..6e5eea5 --- /dev/null +++ b/mdi-core/tests/source_span_resolution.rs @@ -0,0 +1,218 @@ +use mdi_core::{ + MdiSourceSpanCoverage, MdiSourceSpanRelation, MdiSourceSpanResolutionError, + MdiSourceSpanTextMatch, SourceSpan, get_mdi_text_blocks, resolve_mdi_source_span, + resolve_mdi_source_span_json, +}; + +fn span(start_byte: u32, end_byte: u32) -> SourceSpan { + SourceSpan { + start_byte, + end_byte, + } +} + +fn byte_offset(source: &str, needle: &str) -> u32 { + source.find(needle).unwrap() as u32 +} + +#[test] +fn resolves_ascii_cjk_combining_and_emoji_graphemes() { + for (source, needle, expected_range) in [ + ("abc", "b", (2, 3)), + ("甲乙丙", "乙", (2, 3)), + ("e\u{301}x", "e\u{301}", (1, 2)), + ("👩🏽‍💻!", "👩🏽‍💻", (1, 2)), + ] { + let start = byte_offset(source, needle); + let resolution = resolve_mdi_source_span( + source, + span(start, start + u32::try_from(needle.len()).unwrap()), + ) + .unwrap(); + assert_eq!(resolution.coverage, MdiSourceSpanCoverage::Complete); + assert_eq!(resolution.matches.len(), 1); + let MdiSourceSpanTextMatch::BlockText { + range, relation, .. + } = &resolution.matches[0] + else { + panic!("expected block text"); + }; + assert_eq!((range.start.character, range.end.character), expected_range); + assert_eq!(*relation, MdiSourceSpanRelation::Exact); + } +} + +#[test] +fn a_code_point_subspan_of_a_grapheme_is_complete_but_overlapping() { + let source = "e\u{301}"; + let resolution = resolve_mdi_source_span(source, span(0, 1)).unwrap(); + assert_eq!(resolution.coverage, MdiSourceSpanCoverage::Complete); + assert!(matches!( + resolution.matches.as_slice(), + [MdiSourceSpanTextMatch::BlockText { + relation: MdiSourceSpanRelation::Overlap, + .. + }] + )); +} + +#[test] +fn resolves_ruby_base_and_annotation_as_ordered_independent_channels() { + let source = "前{東京|とうきょう}後"; + let token_start = byte_offset(source, "{"); + let token_end = byte_offset(source, "}") + 1; + let resolution = resolve_mdi_source_span(source, span(token_start, token_end)).unwrap(); + + assert_eq!(resolution.coverage, MdiSourceSpanCoverage::Partial); + assert_eq!(resolution.matches.len(), 2); + assert!(matches!( + &resolution.matches[0], + MdiSourceSpanTextMatch::BlockText { + block_index: 1, + relation: MdiSourceSpanRelation::Overlap, + range, + } if range.start.character == 2 && range.end.character == 4 + )); + assert!(matches!( + &resolution.matches[1], + MdiSourceSpanTextMatch::Annotation { + block_index: 1, + annotation_index: 0, + relation: MdiSourceSpanRelation::Overlap, + range, + } if range.start.character == 1 && range.end.character == 6 + )); +} + +#[test] +fn merges_adjacent_hits_across_source_map_runs() { + let source = "a*b*c"; + let resolution = resolve_mdi_source_span(source, span(0, source.len() as u32)).unwrap(); + assert_eq!(resolution.matches.len(), 1); + assert!(matches!( + &resolution.matches[0], + MdiSourceSpanTextMatch::BlockText { range, .. } + if range.start.character == 1 && range.end.character == 4 + )); + assert_eq!(resolution.coverage, MdiSourceSpanCoverage::Partial); +} + +#[test] +fn reports_complete_partial_and_none_without_inventing_structural_ranges() { + let complete = resolve_mdi_source_span("abc", span(0, 3)).unwrap(); + assert_eq!(complete.coverage, MdiSourceSpanCoverage::Complete); + + let multi_block = "a\n\nb"; + let partial = resolve_mdi_source_span(multi_block, span(0, 4)).unwrap(); + assert_eq!(partial.coverage, MdiSourceSpanCoverage::Partial); + assert_eq!(partial.matches.len(), 2); + + let none = resolve_mdi_source_span("---", span(0, 3)).unwrap(); + assert_eq!(none.coverage, MdiSourceSpanCoverage::None); + assert!(none.matches.is_empty()); + + let empty = resolve_mdi_source_span("東京", span(3, 3)).unwrap(); + assert_eq!(empty.coverage, MdiSourceSpanCoverage::None); + assert!(empty.matches.is_empty()); +} + +#[test] +fn a_projected_structural_token_only_matches_when_owned_by_a_grapheme() { + let source = "a[[br]]b"; + let start = byte_offset(source, "[[br]]"); + let resolution = resolve_mdi_source_span(source, span(start, start + 6)).unwrap(); + assert_eq!(resolution.coverage, MdiSourceSpanCoverage::Complete); + assert!(matches!( + resolution.matches.as_slice(), + [MdiSourceSpanTextMatch::BlockText { + relation: MdiSourceSpanRelation::Exact, + range, + .. + }] if range.start.character == 2 && range.end.character == 3 + )); +} + +#[test] +fn validates_order_bounds_and_utf8_boundaries() { + assert_eq!( + resolve_mdi_source_span("abc", span(2, 1)), + Err(MdiSourceSpanResolutionError::Reversed) + ); + assert_eq!( + resolve_mdi_source_span("abc", span(0, 4)), + Err(MdiSourceSpanResolutionError::OutOfBounds) + ); + assert_eq!( + resolve_mdi_source_span("東京", span(1, 3)), + Err(MdiSourceSpanResolutionError::NotUtf8Boundary) + ); +} + +#[test] +fn json_is_deterministic_and_uses_the_public_wire_shape() { + let source = "{東京|とうきょう}"; + let source_span = span(0, source.len() as u32); + let first = resolve_mdi_source_span_json(source, source_span).unwrap(); + assert_eq!( + first, + resolve_mdi_source_span_json(source, source_span).unwrap() + ); + let value: serde_json::Value = serde_json::from_str(&first).unwrap(); + assert_eq!(value["projectionVersion"], "1.0"); + assert_eq!(value["sourceSpan"]["startByte"], 0); + assert_eq!(value["matches"][0]["kind"], "blockText"); + assert_eq!(value["matches"][1]["kind"], "annotation"); + assert_eq!(value["matches"][1]["annotationIndex"], 0); +} + +#[test] +fn syntax_matrix_remains_deterministic_and_never_returns_invalid_ranges() { + let samples = [ + "{東京|とう.きょう}", + "^12^ 《《印》》 [[em:x]] [[br]]", + "| a | b |\n| - | - |\n| c | d |", + "> - nested **text**", + "```mdi\n{literal|code}\n```", + "{malformed|ruby", + "---\ntitle: hidden\n---", + ]; + for source in samples { + let requested = span(0, source.len() as u32); + let resolution = resolve_mdi_source_span(source, requested).unwrap(); + assert_eq!(resolution.source_span, requested); + for matched in resolution.matches { + let range = match matched { + MdiSourceSpanTextMatch::BlockText { range, .. } + | MdiSourceSpanTextMatch::Annotation { range, .. } => range, + }; + assert_eq!(range.start.block, range.end.block); + assert!(range.start.character < range.end.character); + } + } +} + +#[test] +fn every_forward_block_grapheme_span_resolves_back_to_that_grapheme() { + let source = + "# 題\n\n前{東京|とうきょう} e\u{301} 👩🏽‍💻[[br]]後\n\n| a | b |\n| - | - |\n| c | d |"; + let projection = get_mdi_text_blocks(source); + for block in projection.blocks { + for run in block.source_map.runs { + for (offset, boundaries) in run.source_boundaries.windows(2).enumerate() { + let character = run.range.start.character + offset as u32; + let resolution = + resolve_mdi_source_span(source, span(boundaries[0], boundaries[1])).unwrap(); + assert!(resolution.matches.iter().any(|matched| matches!( + matched, + MdiSourceSpanTextMatch::BlockText { + block_index, + range, + .. + } if *block_index == block.index + && range.start.character == character + && range.end.character == character + 1 + ))); + } + } + } +} diff --git a/nodejs/browser-test/src.ts b/nodejs/browser-test/src.ts index 9529c6c..a1f66cf 100644 --- a/nodejs/browser-test/src.ts +++ b/nodejs/browser-test/src.ts @@ -1,4 +1,4 @@ -import { getMdiTextBlocks, initializeMdi, parse, serializeMdi } from "@illusions-lab/mdi"; +import { getMdiTextBlocks, initializeMdi, parse, resolveMdiSourceSpan, serializeMdi } from "@illusions-lab/mdi"; import remarkMdi from "@illusions-lab/mdi-remark"; import remarkParse from "remark-parse"; import remarkStringify from "remark-stringify"; @@ -48,6 +48,8 @@ async function run(): Promise { const parsed = parse(source); const projection = getMdiTextBlocks(source); + const sourceResolutionSpan = parsed.document.children[1]!.span!; + const sourceResolution = resolveMdiSourceSpan(source, sourceResolutionSpan); const recoveryProjection = getMdiTextBlocks(recoverySource); const canonical = serializeMdi(source); const large = parse(largeSource); @@ -62,6 +64,8 @@ async function run(): Promise { projectionVersion: projection.projectionVersion, projectionSource: source, projectionJson: JSON.stringify(projection), + sourceResolutionSpan, + sourceResolutionJson: JSON.stringify(sourceResolution), recoverySource, recoveryProjectionJson: JSON.stringify(recoveryProjection), recoveryDiagnostic: recoveryProjection.diagnostics[0], diff --git a/nodejs/packages/mdi-core/README.md b/nodejs/packages/mdi-core/README.md index e35abcb..56cda5e 100644 --- a/nodejs/packages/mdi-core/README.md +++ b/nodejs/packages/mdi-core/README.md @@ -26,6 +26,12 @@ generated WASM interface, such as a custom language binding or an integration that transports the Rust JSON IR directly. It is not a second JavaScript parser and does not contain a JavaScript grammar. +The raw text-projection boundaries are `getMdiTextBlocksJson(source)` and +`resolveMdiSourceSpanJson(source, startByte, endByte)`. The latter returns the +Rust-owned inverse mapping as JSON and rejects reversed, out-of-bounds, or +non-UTF-8-boundary spans. Applications normally use the typed +`resolveMdiSourceSpan` wrapper from `@illusions-lab/mdi`. + Browser bundlers select the web-compatible runtime facade automatically through the package's `browser` export condition. The generated `.wasm` asset remains private implementation detail and is published alongside that facade. Low-level diff --git a/nodejs/packages/mdi-core/src/browser.d.ts b/nodejs/packages/mdi-core/src/browser.d.ts index c520876..c842794 100644 --- a/nodejs/packages/mdi-core/src/browser.d.ts +++ b/nodejs/packages/mdi-core/src/browser.d.ts @@ -1,6 +1,7 @@ export { applyPdfProfileJson, blockMacroAmount, blockMacroKind, blockMacroVariant, getMdiTextBlocksJson, pageSizeCatalogJson, parseMdiSyntaxJson, prepareChromiumPrintProfileJson, + resolveMdiSourceSpanJson, renderDocx, renderDocxWithProfile, renderEpub, renderEpubWithProfile, renderHtml, renderText, renderTextFormat, resolveExportProfileJson, resolveRuby, serializeMdi, unescapeMdi, unescapeRubyText, diff --git a/nodejs/packages/mdi-core/src/browser.js b/nodejs/packages/mdi-core/src/browser.js index f9a40da..f7f158f 100644 --- a/nodejs/packages/mdi-core/src/browser.js +++ b/nodejs/packages/mdi-core/src/browser.js @@ -21,6 +21,7 @@ export const blockMacroKind = (...args) => (requireInitialized(), bindings.block export const blockMacroVariant = (...args) => (requireInitialized(), bindings.blockMacroVariant(...args)); export const pageSizeCatalogJson = (...args) => (requireInitialized(), bindings.pageSizeCatalogJson(...args)); export const getMdiTextBlocksJson = (...args) => (requireInitialized(), bindings.getMdiTextBlocksJson(...args)); +export const resolveMdiSourceSpanJson = (...args) => (requireInitialized(), bindings.resolveMdiSourceSpanJson(...args)); export const parseMdiSyntaxJson = (...args) => (requireInitialized(), bindings.parseMdiSyntaxJson(...args)); export const prepareChromiumPrintProfileJson = (...args) => (requireInitialized(), bindings.prepareChromiumPrintProfileJson(...args)); export const renderDocx = (...args) => (requireInitialized(), bindings.renderDocx(...args)); diff --git a/nodejs/packages/mdi-core/src/node.cjs b/nodejs/packages/mdi-core/src/node.cjs index e712f93..c0f9246 100644 --- a/nodejs/packages/mdi-core/src/node.cjs +++ b/nodejs/packages/mdi-core/src/node.cjs @@ -14,6 +14,7 @@ exports.blockMacroKind = bindings.blockMacroKind; exports.blockMacroVariant = bindings.blockMacroVariant; exports.pageSizeCatalogJson = bindings.pageSizeCatalogJson; exports.getMdiTextBlocksJson = bindings.getMdiTextBlocksJson; +exports.resolveMdiSourceSpanJson = bindings.resolveMdiSourceSpanJson; exports.parseMdiSyntaxJson = bindings.parseMdiSyntaxJson; exports.prepareChromiumPrintProfileJson = bindings.prepareChromiumPrintProfileJson; exports.renderDocx = bindings.renderDocx; diff --git a/nodejs/packages/mdi/README.md b/nodejs/packages/mdi/README.md index 6437539..c992440 100644 --- a/nodejs/packages/mdi/README.md +++ b/nodejs/packages/mdi/README.md @@ -87,15 +87,28 @@ such as `3:18` count one-based Unicode grapheme clusters; ruby readings are a separate annotation channel anchored to the base-text range. ```ts -import { getMdiTextBlocks, sourceSpansForTextRange } from "@illusions-lab/mdi"; +import { getMdiTextBlocks, resolveMdiSourceSpan, sourceSpansForTextRange } from "@illusions-lab/mdi"; const result = getMdiTextBlocks("{東京|とうきょう}"); const block = result.blocks[0]; console.log(block.text); // 東京 console.log(block.annotations[0].anchor); // { start: "1:1", end: "1:3" } console.log(sourceSpansForTextRange(block, { start: "1:1", end: "1:3" })); +console.log(resolveMdiSourceSpan("{東京|とうきょう}", { startByte: 1, endByte: 7 })); ``` +`resolveMdiSourceSpan` accepts half-open UTF-8 byte offsets and returns ordered +`blockText` and `annotation` matches in canonical grapheme coordinates. +`coverage` is `complete`, `partial`, or `none`; each match is `exact` only when +its complete forward source coverage equals the requested span. Ruby base text +and readings are separate channels, and annotation indexes are zero-based. +Zero-width spans are valid and return no matches. Pure Markdown/MDI delimiters, +synthetic separators, and unmapped text do not acquire invented ranges, though +a delimiter token already owned by one projected grapheme (such as an explicit +break) can match. Reverse and forward mapping are therefore not generally +bijective, especially for annotations, multi-byte token mappings, partial +graphemes, discontinuous runs, and synthetic or unmapped text. + Each source-derived grapheme is represented by a `sourceMap.runs` boundary; table tabs/newlines and multi-paragraph joiners appear in `synthetic` and do not receive invented source spans. `parseMdiTextPosition`, diff --git a/nodejs/packages/mdi/src/index.test.ts b/nodejs/packages/mdi/src/index.test.ts index 9f7575d..e96c854 100644 --- a/nodejs/packages/mdi/src/index.test.ts +++ b/nodejs/packages/mdi/src/index.test.ts @@ -1,6 +1,6 @@ import JSZip from "jszip"; import { describe, expect, it } from "vitest"; -import { MDI_IR_VERSION, MDI_SPEC_VERSION, formatMdiTextPosition, formatMdiTextRange, getMdiTextBlocks, initializeMdi, parse, parseMdiTextPosition, prepareRender, renderDocx, renderDocxWithDiagnostics, renderDocxWithProfile, renderEpub, renderEpubWithDiagnostics, renderEpubWithProfile, renderHtml, renderHtmlWithDiagnostics, renderText, renderTextFormat, renderTextFormatWithDiagnostics, renderTextWithDiagnostics, serializeMdi, sourceSpansForTextRange, toPublicationMdast } from "./index.js"; +import { MDI_IR_VERSION, MDI_SPEC_VERSION, formatMdiTextPosition, formatMdiTextRange, getMdiTextBlocks, initializeMdi, parse, parseMdiTextPosition, prepareRender, renderDocx, renderDocxWithDiagnostics, renderDocxWithProfile, renderEpub, renderEpubWithDiagnostics, renderEpubWithProfile, renderHtml, renderHtmlWithDiagnostics, renderText, renderTextFormat, renderTextFormatWithDiagnostics, renderTextWithDiagnostics, resolveMdiSourceSpan, serializeMdi, sourceSpansForTextRange, toPublicationMdast } from "./index.js"; function assertValidSpans(node: { span?: { startByte: number; endByte: number }; children?: unknown[] }, source: string): void { if (node.span) { @@ -74,6 +74,48 @@ describe("Rust MDI JavaScript binding", () => { expect(sourceSpansForTextRange(table, { start: "1:2", end: "1:3" })).toEqual([]); }); + it("resolves UTF-8 source spans to ordered block and annotation ranges", () => { + const source = "前{東京|とうきょう}後"; + const startByte = Buffer.byteLength("前"); + const endByte = Buffer.byteLength("前{東京|とうきょう}"); + expect(resolveMdiSourceSpan(source, { startByte, endByte })).toEqual({ + projectionVersion: "1.0", + sourceSpan: { startByte, endByte }, + coverage: "partial", + matches: [ + { + kind: "blockText", + blockIndex: 1, + range: { start: "1:2", end: "1:4" }, + relation: "overlap", + }, + { + kind: "annotation", + blockIndex: 1, + annotationIndex: 0, + range: { start: "1:1", end: "1:6" }, + relation: "overlap", + }, + ], + }); + expect(resolveMdiSourceSpan(source, { startByte: 0, endByte: 0 })).toMatchObject({ + coverage: "none", + matches: [], + }); + }); + + it("validates source-span types, uint32 values, bounds, order, and UTF-8 boundaries", () => { + expect(() => resolveMdiSourceSpan(null as never, { startByte: 0, endByte: 0 })).toThrow(TypeError); + expect(() => resolveMdiSourceSpan("x", null as never)).toThrow(TypeError); + expect(() => resolveMdiSourceSpan("x", { startByte: "0" as never, endByte: 0 })).toThrow(TypeError); + for (const invalid of [-1, 0.5, Number.NaN, 0x1_0000_0000]) { + expect(() => resolveMdiSourceSpan("x", { startByte: invalid, endByte: 0 })).toThrow(RangeError); + } + expect(() => resolveMdiSourceSpan("x", { startByte: 1, endByte: 0 })).toThrow(RangeError); + expect(() => resolveMdiSourceSpan("x", { startByte: 0, endByte: 2 })).toThrow(RangeError); + expect(() => resolveMdiSourceSpan("東京", { startByte: 1, endByte: 3 })).toThrow(RangeError); + }); + it("exposes nested syntax decisions made by Rust", () => { const result = parse("**第^12^話**\n\n| a | b |\n| - | - |\n| 1 | 2 |"); expect(result.document.children.map((node) => node.type)).toEqual(["paragraph", "table"]); diff --git a/nodejs/packages/mdi/src/index.ts b/nodejs/packages/mdi/src/index.ts index 0e0d63d..2b1ae06 100644 --- a/nodejs/packages/mdi/src/index.ts +++ b/nodejs/packages/mdi/src/index.ts @@ -6,6 +6,7 @@ import { parse as parseYaml } from "yaml"; const { getMdiTextBlocksJson, + resolveMdiSourceSpanJson, parseMdiSyntaxJson, renderHtml: renderHtmlFromRust, renderEpub: renderEpubFromRust, @@ -183,6 +184,36 @@ export interface MdiTextBlocksResult { diagnostics: MdiDiagnostic[]; } +export type MdiSourceSpanCoverage = "complete" | "partial" | "none"; +export type MdiSourceSpanRelation = "exact" | "overlap"; + +export interface MdiSourceSpanBlockTextMatch { + kind: "blockText"; + blockIndex: number; + range: MdiTextRange; + relation: MdiSourceSpanRelation; +} + +export interface MdiSourceSpanAnnotationMatch { + kind: "annotation"; + blockIndex: number; + /** Zero-based index in the containing block's annotations array. */ + annotationIndex: number; + range: MdiTextRange; + relation: MdiSourceSpanRelation; +} + +export type MdiSourceSpanTextMatch = + | MdiSourceSpanBlockTextMatch + | MdiSourceSpanAnnotationMatch; + +export interface MdiSourceSpanTextResolution { + projectionVersion: "1.0"; + sourceSpan: MdiSourceSpan; + coverage: MdiSourceSpanCoverage; + matches: MdiSourceSpanTextMatch[]; +} + export interface MdiDiagnostic { severity: "warning" | "error"; code: string; @@ -296,6 +327,59 @@ export function getMdiTextBlocks(source: string): MdiTextBlocksResult { return result; } +/** + * Resolve a half-open UTF-8 source span to all mapped canonical block and + * annotation ranges. Mapping semantics are implemented exclusively in Rust. + */ +export function resolveMdiSourceSpan( + source: string, + span: MdiSourceSpan, +): MdiSourceSpanTextResolution { + if (typeof source !== "string") throw new TypeError("source must be a string"); + assertSourceSpanInput(span); + const utf8 = utf8SourceBoundaries(source); + if (span.startByte > span.endByte) { + throw new RangeError("span.startByte must not exceed span.endByte"); + } + if (span.endByte > utf8.length) { + throw new RangeError("span falls outside the UTF-8 source length"); + } + if (!utf8.boundaries.has(span.startByte) || !utf8.boundaries.has(span.endByte)) { + throw new RangeError("span endpoints must be UTF-8 code-point boundaries"); + } + const result = JSON.parse( + resolveMdiSourceSpanJson(source, span.startByte, span.endByte), + ) as MdiSourceSpanTextResolution; + if (result.projectionVersion !== MDI_TEXT_PROJECTION_VERSION) { + throw new Error(`Unsupported MDI text projection version: ${String(result.projectionVersion)}`); + } + return result; +} + +function assertSourceSpanInput(span: MdiSourceSpan): void { + if (!span || typeof span !== "object" || Array.isArray(span)) { + throw new TypeError("span must be an object"); + } + if (typeof span.startByte !== "number" || typeof span.endByte !== "number") { + throw new TypeError("span.startByte and span.endByte must be numbers"); + } + const isUint32 = (value: number): boolean => Number.isInteger(value) && value >= 0 && value <= 0xffff_ffff; + if (!isUint32(span.startByte) || !isUint32(span.endByte)) { + throw new RangeError("span.startByte and span.endByte must be uint32 values"); + } +} + +function utf8SourceBoundaries(source: string): { length: number; boundaries: Set } { + let length = 0; + const boundaries = new Set([0]); + for (const character of source) { + const codePoint = character.codePointAt(0)!; + length += codePoint <= 0x7f ? 1 : codePoint <= 0x7ff ? 2 : codePoint <= 0xffff ? 3 : 4; + boundaries.add(length); + } + return { length, boundaries }; +} + /** Parse and validate a canonical one-based `block:character` position. */ export function parseMdiTextPosition(position: string): MdiTextPositionValue { if (typeof position !== "string") throw new TypeError("position must be a string"); diff --git a/nodejs/packages/mdi/src/source-span-version.test.ts b/nodejs/packages/mdi/src/source-span-version.test.ts new file mode 100644 index 0000000..b700e7e --- /dev/null +++ b/nodejs/packages/mdi/src/source-span-version.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@illusions-lab/mdi-core", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + resolveMdiSourceSpanJson: () => JSON.stringify({ + projectionVersion: "9.9", + sourceSpan: { startByte: 0, endByte: 0 }, + coverage: "none", + matches: [], + }), + }; +}); + +import { resolveMdiSourceSpan } from "./index.js"; + +describe("resolveMdiSourceSpan projection-version guard", () => { + it("rejects an unsupported Rust projection version", () => { + expect(() => resolveMdiSourceSpan("", { startByte: 0, endByte: 0 })).toThrow( + "Unsupported MDI text projection version: 9.9", + ); + }); +}); diff --git a/nodejs/scripts/test-browser-wasm.mjs b/nodejs/scripts/test-browser-wasm.mjs index f1c040e..c5a27d3 100644 --- a/nodejs/scripts/test-browser-wasm.mjs +++ b/nodejs/scripts/test-browser-wasm.mjs @@ -106,7 +106,7 @@ try { const url = `http://127.0.0.1:${address.port}/mdi-editor/`; console.log("Running packed browser WASM contract in Chromium, Firefox, and WebKit"); for (const [browserName, browserType] of browserTypes) { - await testBrowser(browserName, browserType, url, packedMdi.getMdiTextBlocks, runRetry && browserName === "Chromium" ? `${url}?retry=1` : undefined); + await testBrowser(browserName, browserType, url, packedMdi.getMdiTextBlocks, packedMdi.resolveMdiSourceSpan, runRetry && browserName === "Chromium" ? `${url}?retry=1` : undefined); } assert.equal(wasmRequests.length, browserTypes.length + (runRetry ? 2 : 0), "failed initialization must retry exactly once without duplicate concurrent loads"); assert(wasmRequests.every(({ contentType }) => contentType === "application/wasm"), "WASM must be served with application/wasm"); @@ -118,21 +118,21 @@ try { await rm(outputDirectory, { recursive: true, force: true }); } -async function testBrowser(browserName, browserType, url, getNodeProjection, retryUrl) { +async function testBrowser(browserName, browserType, url, getNodeProjection, resolveNodeSourceSpan, retryUrl) { console.log(`Launching ${browserName}`); const browser = await browserType.launch({ headless: true }); try { - await testPage(browserName, await browser.newPage(), url, getNodeProjection); + await testPage(browserName, await browser.newPage(), url, getNodeProjection, resolveNodeSourceSpan); if (retryUrl) { retryFailureRemaining = 1; - await testPage(`${browserName} retry`, await browser.newPage(), retryUrl, getNodeProjection); + await testPage(`${browserName} retry`, await browser.newPage(), retryUrl, getNodeProjection, resolveNodeSourceSpan); } } finally { await browser.close(); } } -async function testPage(browserName, page, url, getNodeProjection) { +async function testPage(browserName, page, url, getNodeProjection, resolveNodeSourceSpan) { const pageErrors = []; page.on("pageerror", (error) => pageErrors.push(error)); await page.goto(url, { waitUntil: "networkidle" }); @@ -157,6 +157,11 @@ async function testPage(browserName, page, url, getNodeProjection) { JSON.stringify(getNodeProjection(result.recoverySource)), `${browserName}: malformed recovery must match Node without trapping Wasm`, ); + assert.equal( + result.sourceResolutionJson, + JSON.stringify(resolveNodeSourceSpan(result.projectionSource, result.sourceResolutionSpan)), + `${browserName}: Node and browser must return byte-for-byte identical source resolution JSON`, + ); assert.equal(result.recoveryDiagnostic?.code, "mdi.parser.recovered", browserName); assert.equal(result.projectedBlocks[0]?.kind, "heading", browserName); assert.equal(result.projectedBlocks[0]?.range?.start, "1:1", browserName); diff --git a/nodejs/scripts/test-safari-wasm.mjs b/nodejs/scripts/test-safari-wasm.mjs index b081699..57e1c43 100644 --- a/nodejs/scripts/test-safari-wasm.mjs +++ b/nodejs/scripts/test-safari-wasm.mjs @@ -61,6 +61,8 @@ try { const result = await pollResult(); assert.equal(result.error, undefined, result.error); assert.equal(result.irVersion, "1.0"); + assert.equal(JSON.parse(result.sourceResolutionJson).projectionVersion, "1.0"); + assert(JSON.parse(result.sourceResolutionJson).matches.length > 0); assert.equal(result.hasFrontmatter, true); assert.equal(result.tableType, "table"); assert(result.utf8Span?.endByte > result.utf8Span?.startByte);