From f6110d2f47991eb524dab2395a079868c9db7a79 Mon Sep 17 00:00:00 2001 From: Ahmad AL-Quraan Date: Sun, 2 Aug 2026 11:21:52 +0300 Subject: [PATCH 1/2] test(tools): add unit tests for internal/ucd --- tools/internal/ucd/bidi_test.go | 104 +++++++++++ tools/internal/ucd/mapping_test.go | 246 +++++++++++++++++++++++++ tools/internal/ucd/nfc_test.go | 170 ++++++++++++++++++ tools/internal/ucd/ucd.go | 16 +- tools/internal/ucd/ucd_test.go | 266 ++++++++++++++++++++++++++++ tools/internal/ucd/validity_test.go | 183 +++++++++++++++++++ 6 files changed, 981 insertions(+), 4 deletions(-) create mode 100644 tools/internal/ucd/bidi_test.go create mode 100644 tools/internal/ucd/mapping_test.go create mode 100644 tools/internal/ucd/nfc_test.go create mode 100644 tools/internal/ucd/ucd_test.go create mode 100644 tools/internal/ucd/validity_test.go diff --git a/tools/internal/ucd/bidi_test.go b/tools/internal/ucd/bidi_test.go new file mode 100644 index 0000000..cf53543 --- /dev/null +++ b/tools/internal/ucd/bidi_test.go @@ -0,0 +1,104 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah +// SPDX-License-Identifier: MIT + +package ucd + +import "testing" + +func TestLoadBidiUnicodeDataBasic(t *testing.T) { + data := []byte( + "0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;;\n" + + "0627;ARABIC LETTER ALEF;Lo;0;AL;;;;;N;;;;;\n", + ) + got, err := loadBidiUnicodeData(data) + if err != nil { + t.Fatalf("loadBidiUnicodeData: unexpected error: %v", err) + } + if len(got) != 2 { + t.Fatalf("loadBidiUnicodeData: got %d ranges, want 2", len(got)) + } + if got[0].start != 0x41 || got[0].jtype != "L" { + t.Errorf("got[0] = %+v, want start=0x41 class=L", got[0]) + } + if got[1].start != 0x627 || got[1].jtype != "AL" { + t.Errorf("got[1] = %+v, want start=0x627 class=AL", got[1]) + } +} + +func TestLoadBidiUnicodeDataFiltersUntrackedClasses(t *testing.T) { + // B is Paragraph_Separator, which is not in bidiClasses; such code points + // must be dropped entirely from the result, not recorded with an empty class. + data := []byte("2029;PARAGRAPH SEPARATOR;Zp;0;B;;;;;N;;;;;\n") + got, err := loadBidiUnicodeData(data) + if err != nil { + t.Fatalf("loadBidiUnicodeData: unexpected error: %v", err) + } + if len(got) != 0 { + t.Fatalf("loadBidiUnicodeData = %+v, want empty (Paragraph_Separator is untracked)", got) + } +} + +func TestLoadBidiUnicodeDataFirstLastBlock(t *testing.T) { + // The expanded range takes the Bidi_Class of the Last (closing) row, mirroring + // the Mark/Virama pass's First/Last handling. + data := []byte( + "3400;;Lo;0;L;;;;;N;;;;;\n" + + "4DBF;;Lo;0;L;;;;;N;;;;;\n", + ) + got, err := loadBidiUnicodeData(data) + if err != nil { + t.Fatalf("loadBidiUnicodeData: unexpected error: %v", err) + } + if len(got) != 1 || got[0].start != 0x3400 || got[0].end != 0x4DBF || got[0].jtype != "L" { + t.Fatalf("got = %+v, want a single range [0x3400, 0x4DBF] class L", got) + } +} + +func TestLoadBidiUnicodeDataSkipsShortLines(t *testing.T) { + data := []byte("\n0041;A;Lu;0;L;;;;;N;;;;;\n") + got, err := loadBidiUnicodeData(data) + if err != nil { + t.Fatalf("loadBidiUnicodeData: unexpected error: %v", err) + } + if len(got) != 1 { + t.Fatalf("loadBidiUnicodeData: got %d ranges, want 1 (blank line skipped)", len(got)) + } +} + +func TestLoadBidiUnicodeDataInvalidCodePoint(t *testing.T) { + data := []byte("ZZZZ;BOGUS;Lu;0;L;;;;;N;;;;;\n") + if _, err := loadBidiUnicodeData(data); err == nil { + t.Fatal("loadBidiUnicodeData with invalid code point: expected error, got nil") + } +} + +func TestLoadBidiUnicodeDataSortsByStart(t *testing.T) { + // Out-of-order input must come back sorted by start, mirroring the sort step + // in loadBidiUnicodeData. + data := []byte( + "0627;ARABIC LETTER ALEF;Lo;0;AL;;;;;N;;;;;\n" + + "0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;;\n", + ) + got, err := loadBidiUnicodeData(data) + if err != nil { + t.Fatalf("loadBidiUnicodeData: unexpected error: %v", err) + } + if len(got) != 2 || got[0].start != 0x41 || got[1].start != 0x627 { + t.Fatalf("loadBidiUnicodeData did not sort by start: got %+v", got) + } +} + +func TestBidiClassesTrackedSet(t *testing.T) { + // Pin the exact set of classes the RFC 5893 rule consults, so an accidental + // addition or removal in bidi.go is caught here rather than only surfacing as + // a silent behavior change in the generated table. + want := []string{"L", "R", "AL", "EN", "ES", "ET", "AN", "CS", "NSM", "BN", "ON"} + if len(bidiClasses) != len(want) { + t.Fatalf("bidiClasses has %d entries, want %d: %v", len(bidiClasses), len(want), bidiClasses) + } + for _, class := range want { + if !bidiClasses[class] { + t.Errorf("bidiClasses[%q] = false, want true", class) + } + } +} diff --git a/tools/internal/ucd/mapping_test.go b/tools/internal/ucd/mapping_test.go new file mode 100644 index 0000000..cdd8f78 --- /dev/null +++ b/tools/internal/ucd/mapping_test.go @@ -0,0 +1,246 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah +// SPDX-License-Identifier: MIT + +package ucd + +import ( + "strings" + "testing" +) + +func TestParseLineBlankAndComment(t *testing.T) { + for _, line := range []string{"", " ", "# just a comment", " # comment "} { + got, err := parseLine(line) + if err != nil { + t.Fatalf("parseLine(%q): unexpected error: %v", line, err) + } + if got != nil { + t.Fatalf("parseLine(%q) = %+v, want nil", line, got) + } + } +} + +func TestParseLineValid(t *testing.T) { + got, err := parseLine("0000..002C ; disallowed # ..") + if err != nil { + t.Fatalf("parseLine: unexpected error: %v", err) + } + if got == nil { + t.Fatal("parseLine: got nil, want a range") + } + if got.start != 0 || got.end != 0x2C || got.kind != kindDisallowed || got.replacement != "" { + t.Fatalf("parseLine = %+v, want {start:0 end:0x2C kind:D replacement:\"\"}", *got) + } +} + +func TestParseLineMappedWithReplacement(t *testing.T) { + got, err := parseLine("0041 ; mapped ; 0061 # LATIN CAPITAL LETTER A") + if err != nil { + t.Fatalf("parseLine: unexpected error: %v", err) + } + if got.kind != kindMapped || got.replacement != "a" { + t.Fatalf("parseLine = %+v, want kind M replacement %q", *got, "a") + } + if got.start != 0x41 || got.end != 0x41 { + t.Fatalf("parseLine range = [%#x, %#x], want [0x41, 0x41]", got.start, got.end) + } +} + +func TestParseLineDeviationDropsReplacement(t *testing.T) { + // Deviation records carry a replacement field in the source (e.g. ß -> ss), + // but the runtime kind never uses it, and parseLine must discard it. + got, err := parseLine("00DF ; deviation ; 0073 0073") + if err != nil { + t.Fatalf("parseLine: unexpected error: %v", err) + } + if got.kind != kindDeviation { + t.Fatalf("parseLine kind = %q, want %q", got.kind, kindDeviation) + } + if got.replacement != "" { + t.Fatalf("parseLine replacement = %q, want empty (deviation targets are discarded)", got.replacement) + } +} + +func TestParseLineAllStatusKeywords(t *testing.T) { + tests := []struct { + status string + wantKind string + }{ + {"valid", kindValid}, + {"disallowed_STD3_valid", kindValid}, + {"ignored", kindIgnored}, + {"disallowed", kindDisallowed}, + {"mapped", kindMapped}, + {"disallowed_STD3_mapped", kindMapped}, + {"deviation", kindDeviation}, + } + for _, tt := range tests { + t.Run(tt.status, func(t *testing.T) { + line := "0041; " + tt.status + if tt.status == "mapped" || tt.status == "disallowed_STD3_mapped" { + line += "; 0061" + } + got, err := parseLine(line) + if err != nil { + t.Fatalf("parseLine(%q): unexpected error: %v", line, err) + } + if got.kind != tt.wantKind { + t.Fatalf("parseLine(%q) kind = %q, want %q", line, got.kind, tt.wantKind) + } + }) + } +} + +func TestParseLineUnknownStatus(t *testing.T) { + if _, err := parseLine("0041; bogus_status"); err == nil { + t.Fatal("parseLine with unknown status: expected error, got nil") + } +} + +func TestParseLineMissingStatus(t *testing.T) { + if _, err := parseLine("0041; "); err == nil { + t.Fatal("parseLine with empty status field: expected error, got nil") + } +} + +func TestParseLineInvalidCodeRange(t *testing.T) { + if _, err := parseLine("ZZZZ; valid"); err == nil { + t.Fatal("parseLine with invalid code range: expected error, got nil") + } +} + +func TestLoadRangesGapFreeCoverage(t *testing.T) { + // A tiny, gap-free, three-range fixture covering the entire code-point space + // via a single "10FFFF..10FFFF"-adjacent final record would be unwieldy to + // hand-write; instead craft the minimal input by directly exercising + // loadRanges' invariant check with ranges that already cover 0..maxCodePoint. + data := []byte( + "0000..10FFFE ; valid\n" + + "10FFFF ; disallowed\n", + ) + ranges, err := loadRanges(data) + if err != nil { + t.Fatalf("loadRanges: unexpected error: %v", err) + } + if len(ranges) != 2 { + t.Fatalf("loadRanges: got %d ranges, want 2", len(ranges)) + } + if ranges[0].start != 0 || ranges[0].end != 0x10FFFE { + t.Errorf("ranges[0] = %+v, want start=0 end=0x10FFFE", ranges[0]) + } + if ranges[1].start != 0x10FFFF || ranges[1].end != 0x10FFFF { + t.Errorf("ranges[1] = %+v, want start=0x10FFFF end=0x10FFFF", ranges[1]) + } +} + +func TestLoadRangesDetectsGap(t *testing.T) { + // A gap at U+0002: the file jumps from ending at 0001 to starting at 0003, + // leaving 0002 uncovered. + data := []byte( + "0000..0001 ; valid\n" + + "0003..10FFFF ; valid\n", + ) + _, err := loadRanges(data) + if err == nil { + t.Fatal("loadRanges with a gap: expected error, got nil") + } + if !strings.Contains(err.Error(), "gap/overlap") { + t.Fatalf("loadRanges error = %v, want a gap/overlap message", err) + } +} + +func TestLoadRangesDetectsOverlap(t *testing.T) { + // The second range starts before the first one's end, an overlap. + data := []byte( + "0000..0005 ; valid\n" + + "0003..10FFFF ; valid\n", + ) + _, err := loadRanges(data) + if err == nil { + t.Fatal("loadRanges with an overlap: expected error, got nil") + } +} + +func TestLoadRangesDetectsIncompleteCoverage(t *testing.T) { + // Coverage stops short of 0x10FFFF entirely. + data := []byte("0000..00FF ; valid\n") + _, err := loadRanges(data) + if err == nil { + t.Fatal("loadRanges with incomplete coverage: expected error, got nil") + } + if !strings.Contains(err.Error(), "coverage ends") { + t.Fatalf("loadRanges error = %v, want a coverage message", err) + } +} + +func TestLoadRangesSortsUnorderedInput(t *testing.T) { + // Records appear out of code-point order in the file; loadRanges must sort + // them by start before validating coverage, mirroring the Python sort step. + data := []byte( + "0100..10FFFF ; valid\n" + + "0000..00FF ; disallowed\n", + ) + ranges, err := loadRanges(data) + if err != nil { + t.Fatalf("loadRanges: unexpected error: %v", err) + } + if len(ranges) != 2 || ranges[0].start != 0 || ranges[1].start != 0x100 { + t.Fatalf("loadRanges did not sort by start: got %+v", ranges) + } +} + +func TestMergeAdjacentSameKindNoReplacement(t *testing.T) { + in := []idnaRange{ + {start: 0, end: 9, kind: kindValid}, + {start: 10, end: 19, kind: kindValid}, + } + got := mergeAdjacent(in) + if len(got) != 1 { + t.Fatalf("mergeAdjacent: got %d ranges, want 1 merged range", len(got)) + } + if got[0].start != 0 || got[0].end != 19 { + t.Fatalf("mergeAdjacent = %+v, want {start:0 end:19}", got[0]) + } +} + +func TestMergeAdjacentDifferentKindNotMerged(t *testing.T) { + in := []idnaRange{ + {start: 0, end: 9, kind: kindValid}, + {start: 10, end: 19, kind: kindDisallowed}, + } + got := mergeAdjacent(in) + if len(got) != 2 { + t.Fatalf("mergeAdjacent: got %d ranges, want 2 (different kinds must not merge)", len(got)) + } +} + +func TestMergeAdjacentSameKindDifferentReplacementNotMerged(t *testing.T) { + in := []idnaRange{ + {start: 0, end: 0, kind: kindMapped, replacement: "a"}, + {start: 1, end: 1, kind: kindMapped, replacement: "b"}, + } + got := mergeAdjacent(in) + if len(got) != 2 { + t.Fatalf("mergeAdjacent: got %d ranges, want 2 (different replacements must not merge)", len(got)) + } +} + +func TestMergeAdjacentNonAdjacentNotMerged(t *testing.T) { + // A gap between end and the next start (here just a difference of more than + // one) must prevent merging, even though the kinds match. + in := []idnaRange{ + {start: 0, end: 9, kind: kindValid}, + {start: 11, end: 19, kind: kindValid}, + } + got := mergeAdjacent(in) + if len(got) != 2 { + t.Fatalf("mergeAdjacent: got %d ranges, want 2 (non-adjacent ranges must not merge)", len(got)) + } +} + +func TestMergeAdjacentEmpty(t *testing.T) { + got := mergeAdjacent(nil) + if len(got) != 0 { + t.Fatalf("mergeAdjacent(nil) = %+v, want empty", got) + } +} diff --git a/tools/internal/ucd/nfc_test.go b/tools/internal/ucd/nfc_test.go new file mode 100644 index 0000000..738081d --- /dev/null +++ b/tools/internal/ucd/nfc_test.go @@ -0,0 +1,170 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah +// SPDX-License-Identifier: MIT + +package ucd + +import "testing" + +func TestLoadNfcUnicodeDataCombiningClassAndDecomposition(t *testing.T) { + // 00C0 (LATIN CAPITAL LETTER A WITH GRAVE) canonically decomposes to 0041 0300. + // 0300 (COMBINING GRAVE ACCENT) has CCC 230 and no decomposition. + // 0041 has CCC 0 (omitted from the map, since only non-zero CCC is kept) and + // no decomposition (blank field). + data := []byte( + "0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;;\n" + + "0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;\n" + + "00C0;LATIN CAPITAL LETTER A WITH GRAVE;Lu;0;L;0041 0300;;;;N;;;;;\n", + ) + ccc, decomposition, err := loadNfcUnicodeData(data) + if err != nil { + t.Fatalf("loadNfcUnicodeData: unexpected error: %v", err) + } + if _, ok := ccc[0x41]; ok { + t.Error("ccc[0x41] present, want absent (CCC 0 is not recorded)") + } + if got, want := ccc[0x300], 230; got != want { + t.Errorf("ccc[0x300] = %d, want %d", got, want) + } + want := []int{0x41, 0x300} + got := decomposition[0xC0] + if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] { + t.Errorf("decomposition[0xC0] = %v, want %v", got, want) + } + if _, ok := decomposition[0x300]; ok { + t.Error("decomposition[0x300] present, want absent (no decomposition field)") + } +} + +func TestLoadNfcUnicodeDataSkipsCompatibilityDecomposition(t *testing.T) { + // A decomposition field starting with '<' is a compatibility mapping (e.g. + // " 0041") and must be excluded from the canonical decomposition map. + data := []byte("00BC;VULGAR FRACTION ONE QUARTER;No;0;ON; 0031 2044 0034;;;;N;;;;;\n") + _, decomposition, err := loadNfcUnicodeData(data) + if err != nil { + t.Fatalf("loadNfcUnicodeData: unexpected error: %v", err) + } + if _, ok := decomposition[0xBC]; ok { + t.Error("decomposition[0xBC] present, want absent (compatibility decomposition must be skipped)") + } +} + +func TestLoadNfcUnicodeDataSkipsShortLines(t *testing.T) { + // A line with fewer than six fields (e.g. blank) must be skipped without error. + data := []byte("\n0041;A;Lu;0;L;;;;;N;;;;;\n") + ccc, decomposition, err := loadNfcUnicodeData(data) + if err != nil { + t.Fatalf("loadNfcUnicodeData: unexpected error: %v", err) + } + if len(ccc) != 0 || len(decomposition) != 0 { + t.Errorf("ccc=%v decomposition=%v, want both empty for this single-real-record fixture", ccc, decomposition) + } +} + +func TestLoadNfcUnicodeDataInvalidCodePoint(t *testing.T) { + data := []byte("ZZZZ;BOGUS;Lu;0;L;;;;;N;;;;;\n") + if _, _, err := loadNfcUnicodeData(data); err == nil { + t.Fatal("loadNfcUnicodeData with invalid code point: expected error, got nil") + } +} + +func TestLoadNfcUnicodeDataInvalidCombiningClass(t *testing.T) { + data := []byte("0041;A;Lu;NOTANUMBER;L;;;;;N;;;;;\n") + if _, _, err := loadNfcUnicodeData(data); err == nil { + t.Fatal("loadNfcUnicodeData with invalid combining class: expected error, got nil") + } +} + +func TestLoadNfcUnicodeDataInvalidDecompositionTarget(t *testing.T) { + data := []byte("00C0;A WITH GRAVE;Lu;0;L;ZZZZ 0300;;;;N;;;;;\n") + if _, _, err := loadNfcUnicodeData(data); err == nil { + t.Fatal("loadNfcUnicodeData with an invalid decomposition target: expected error, got nil") + } +} + +func TestLoadExclusions(t *testing.T) { + data := []byte( + "# 0344 COMBINING GREEK DIALYTIKA TONOS (Non-Starter Decompositions, commented out)\n" + + "0F73 # TIBETAN VOWEL SIGN II\n" + + "\n" + + "# a pure comment line\n", + ) + got, err := loadExclusions(data) + if err != nil { + t.Fatalf("loadExclusions: unexpected error: %v", err) + } + // 0344 appears only inside a comment (the whole line starts with '#'), so its + // body before '#' is empty and it must NOT be in the exclusion set. + if got[0x344] { + t.Error("exclusions[0x344] = true, want false (fully commented line)") + } + if !got[0xF73] { + t.Error("exclusions[0xF73] = false, want true") + } +} + +func TestLoadExclusionsInvalidHex(t *testing.T) { + data := []byte("ZZZZ\n") + if _, err := loadExclusions(data); err == nil { + t.Fatal("loadExclusions with invalid hex: expected error, got nil") + } +} + +func TestBuildCompositionBasicPair(t *testing.T) { + ccc := map[int]int{0x300: 230} + decomposition := map[int][]int{0xC0: {0x41, 0x300}} + exclusions := map[int]bool{} + got := buildComposition(ccc, decomposition, exclusions) + if got[[2]int{0x41, 0x300}] != 0xC0 { + t.Fatalf("composition[{0x41,0x300}] = %#x, want 0xC0", got[[2]int{0x41, 0x300}]) + } +} + +func TestBuildCompositionExcludedNotComposed(t *testing.T) { + ccc := map[int]int{0x300: 230} + decomposition := map[int][]int{0xC0: {0x41, 0x300}} + exclusions := map[int]bool{0xC0: true} + got := buildComposition(ccc, decomposition, exclusions) + if _, ok := got[[2]int{0x41, 0x300}]; ok { + t.Fatal("composition contains an explicitly excluded composite, want absent") + } +} + +func TestBuildCompositionNonPairNotComposed(t *testing.T) { + // A three-element (or singleton) decomposition is not a canonical pair and + // must never contribute a primary composite. + ccc := map[int]int{} + decomposition := map[int][]int{ + 0x1E0C: {0x44, 0x323, 0x300}, // hypothetical 3-part mapping + 0x1234: {0x41}, // singleton + } + got := buildComposition(ccc, decomposition, map[int]bool{}) + if len(got) != 0 { + t.Fatalf("composition = %v, want empty (no 2-element decompositions present)", got) + } +} + +func TestBuildCompositionNonStarterFirstElementNotComposed(t *testing.T) { + // If the first element of the pair is itself a non-starter (CCC != 0), it + // cannot serve as a composition base and the pair must be skipped. + ccc := map[int]int{0x300: 230, 0x301: 230} + decomposition := map[int][]int{0x1000: {0x300, 0x301}} + got := buildComposition(ccc, decomposition, map[int]bool{}) + if len(got) != 0 { + t.Fatalf("composition = %v, want empty (starter of the pair is a non-starter)", got) + } +} + +func TestBuildCompositionLastWriteWinsInCodePointOrder(t *testing.T) { + // Two different composites decompose to the same (starter, combining) pair; + // buildComposition must resolve the collision using ascending code-point + // (file) order, so the higher code point (processed last) wins. + ccc := map[int]int{0x300: 230} + decomposition := map[int][]int{ + 0x1000: {0x41, 0x300}, + 0x2000: {0x41, 0x300}, + } + got := buildComposition(ccc, decomposition, map[int]bool{}) + if got[[2]int{0x41, 0x300}] != 0x2000 { + t.Fatalf("composition[{0x41,0x300}] = %#x, want 0x2000 (higher code point wins)", got[[2]int{0x41, 0x300}]) + } +} diff --git a/tools/internal/ucd/ucd.go b/tools/internal/ucd/ucd.go index 5d7ab15..7c363e8 100644 --- a/tools/internal/ucd/ucd.go +++ b/tools/internal/ucd/ucd.go @@ -98,13 +98,21 @@ func (v UnicodeVersion) String() string { // error and yields an error rather than letting a bogus version reach generated // code. func BundledUnicodeVersion() (UnicodeVersion, error) { - rest, ok := strings.CutPrefix(unicodeVersionDir, versionDirPrefix) + return parseVersionPin(unicodeVersionDir) +} + +// parseVersionPin parses a "unicode-.[.]" directory pin +// into its components. It is split out from [BundledUnicodeVersion] so the +// parser itself can be exercised against a range of well- and ill-formed pins +// independent of whatever [unicodeVersionDir] currently is. +func parseVersionPin(pin string) (UnicodeVersion, error) { + rest, ok := strings.CutPrefix(pin, versionDirPrefix) if !ok { - return UnicodeVersion{}, fmt.Errorf("ucd: version pin %q lacks the %q prefix", unicodeVersionDir, versionDirPrefix) + return UnicodeVersion{}, fmt.Errorf("ucd: version pin %q lacks the %q prefix", pin, versionDirPrefix) } fields := strings.Split(rest, ".") if len(fields) < minVersionComponents || len(fields) > maxVersionComponents { - return UnicodeVersion{}, fmt.Errorf("ucd: version pin %q is not unicode-.[.]", unicodeVersionDir) + return UnicodeVersion{}, fmt.Errorf("ucd: version pin %q is not unicode-.[.]", pin) } var components [maxVersionComponents]int for index, field := range fields { @@ -114,7 +122,7 @@ func BundledUnicodeVersion() (UnicodeVersion, error) { // a version string that no longer matches the on-disk directory suffix. The // value < 0 arm is still needed because "-5" round-trips through Itoa. if err != nil || value < 0 || field != strconv.Itoa(value) { - return UnicodeVersion{}, fmt.Errorf("ucd: version pin %q has a malformed component %q", unicodeVersionDir, field) + return UnicodeVersion{}, fmt.Errorf("ucd: version pin %q has a malformed component %q", pin, field) } components[index] = value } diff --git a/tools/internal/ucd/ucd_test.go b/tools/internal/ucd/ucd_test.go new file mode 100644 index 0000000..8e50d4d --- /dev/null +++ b/tools/internal/ucd/ucd_test.go @@ -0,0 +1,266 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah +// SPDX-License-Identifier: MIT + +package ucd + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParseCodeRangeSingle(t *testing.T) { + lo, hi, err := ParseCodeRange("0041") + if err != nil { + t.Fatalf("ParseCodeRange: unexpected error: %v", err) + } + if lo != 0x41 || hi != 0x41 { + t.Fatalf("ParseCodeRange(%q) = (%#x, %#x), want (0x41, 0x41)", "0041", lo, hi) + } +} + +func TestParseCodeRangeRange(t *testing.T) { + lo, hi, err := ParseCodeRange("0000..002C") + if err != nil { + t.Fatalf("ParseCodeRange: unexpected error: %v", err) + } + if lo != 0 || hi != 0x2C { + t.Fatalf("ParseCodeRange(%q) = (%#x, %#x), want (0x0, 0x2C)", "0000..002C", lo, hi) + } +} + +func TestParseCodeRangeUppercaseHex(t *testing.T) { + // UCD files sometimes use uppercase hex letters (e.g. "10FFFF"); ParseInt with + // radix 16 accepts either case, so this should parse identically to lowercase. + lo, hi, err := ParseCodeRange("10FFFF") + if err != nil { + t.Fatalf("ParseCodeRange: unexpected error: %v", err) + } + if lo != 0x10FFFF || hi != 0x10FFFF { + t.Fatalf("ParseCodeRange(%q) = (%#x, %#x), want (0x10FFFF, 0x10FFFF)", "10FFFF", lo, hi) + } +} + +func TestParseCodeRangeInvalidHex(t *testing.T) { + if _, _, err := ParseCodeRange("ZZZZ"); err == nil { + t.Fatal("ParseCodeRange(\"ZZZZ\"): expected error, got nil") + } +} + +func TestParseCodeRangeInvalidHexInRange(t *testing.T) { + // The high bound is malformed; both branches of the ".." split must be checked. + if _, _, err := ParseCodeRange("0041..ZZZZ"); err == nil { + t.Fatal("ParseCodeRange(\"0041..ZZZZ\"): expected error, got nil") + } + if _, _, err := ParseCodeRange("ZZZZ..0041"); err == nil { + t.Fatal("ParseCodeRange(\"ZZZZ..0041\"): expected error, got nil") + } +} + +func TestScalarsToStringSingle(t *testing.T) { + got, err := ScalarsToString("0044") + if err != nil { + t.Fatalf("ScalarsToString: unexpected error: %v", err) + } + if want := "D"; got != want { + t.Fatalf("ScalarsToString(%q) = %q, want %q", "0044", got, want) + } +} + +func TestScalarsToStringMultiple(t *testing.T) { + // "0044 0307" is D + COMBINING DOT ABOVE, the classic NormalizationTest.txt + // field encoding this function exists to decode. + got, err := ScalarsToString("0044 0307") + if err != nil { + t.Fatalf("ScalarsToString: unexpected error: %v", err) + } + want := string(rune(0x44)) + string(rune(0x307)) + if got != want { + t.Fatalf("ScalarsToString(%q) = %q, want %q", "0044 0307", got, want) + } +} + +func TestScalarsToStringCollapsesWhitespace(t *testing.T) { + // Multiple spaces/tabs between tokens, and leading/trailing whitespace, should + // behave exactly like Python's no-argument str.split. + got, err := ScalarsToString(" 0044 0307\t") + if err != nil { + t.Fatalf("ScalarsToString: unexpected error: %v", err) + } + want := string(rune(0x44)) + string(rune(0x307)) + if got != want { + t.Fatalf("ScalarsToString with irregular whitespace = %q, want %q", got, want) + } +} + +func TestScalarsToStringEmpty(t *testing.T) { + got, err := ScalarsToString("") + if err != nil { + t.Fatalf("ScalarsToString: unexpected error: %v", err) + } + if got != "" { + t.Fatalf("ScalarsToString(\"\") = %q, want empty string", got) + } +} + +func TestScalarsToStringAllWhitespace(t *testing.T) { + got, err := ScalarsToString(" \t ") + if err != nil { + t.Fatalf("ScalarsToString: unexpected error: %v", err) + } + if got != "" { + t.Fatalf("ScalarsToString(all-whitespace) = %q, want empty string", got) + } +} + +func TestScalarsToStringRejectsSurrogate(t *testing.T) { + // D800 is the first UTF-16 high surrogate; it is not a valid scalar value and + // must not silently become U+FFFD via WriteRune. + if _, err := ScalarsToString("D800"); err == nil { + t.Fatal("ScalarsToString(\"D800\"): expected error for surrogate code point, got nil") + } +} + +func TestScalarsToStringRejectsSurrogateBoundaries(t *testing.T) { + for _, tok := range []string{"D800", "DFFF", "DC00"} { + if _, err := ScalarsToString(tok); err == nil { + t.Fatalf("ScalarsToString(%q): expected error for surrogate code point, got nil", tok) + } + } + // One below the surrogate block and one above must both be accepted. + if _, err := ScalarsToString("D7FF"); err != nil { + t.Fatalf("ScalarsToString(\"D7FF\"): unexpected error: %v", err) + } + if _, err := ScalarsToString("E000"); err != nil { + t.Fatalf("ScalarsToString(\"E000\"): unexpected error: %v", err) + } +} + +func TestScalarsToStringRejectsOutOfRange(t *testing.T) { + // 0x110000 is one past the maximum valid Unicode scalar value. + if _, err := ScalarsToString("110000"); err == nil { + t.Fatal("ScalarsToString(\"110000\"): expected error for out-of-range scalar, got nil") + } + // The maximum valid scalar itself must be accepted. + if _, err := ScalarsToString("10FFFF"); err != nil { + t.Fatalf("ScalarsToString(\"10FFFF\"): unexpected error: %v", err) + } +} + +func TestScalarsToStringRejectsInvalidHex(t *testing.T) { + if _, err := ScalarsToString("ZZZZ"); err == nil { + t.Fatal("ScalarsToString(\"ZZZZ\"): expected error, got nil") + } +} + +func TestScalarsToStringSupplementaryPlane(t *testing.T) { + // A code point above the BMP must round-trip through WriteRune as a single + // Go rune (rune is int32, so no truncation), not a UTF-16 surrogate pair. + got, err := ScalarsToString("1F600") + if err != nil { + t.Fatalf("ScalarsToString: unexpected error: %v", err) + } + want := string(rune(0x1F600)) + if got != want { + t.Fatalf("ScalarsToString(%q) = %q, want %q", "1F600", got, want) + } +} + +func TestBundledUnicodeVersionRenderings(t *testing.T) { + version, err := BundledUnicodeVersion() + if err != nil { + t.Fatalf("BundledUnicodeVersion: unexpected error: %v", err) + } + if got, want := version.MajorMinor(), "17.0"; got != want { + t.Errorf("MajorMinor() = %q, want %q", got, want) + } + if got, want := version.String(), "17.0.0"; got != want { + t.Errorf("String() = %q, want %q", got, want) + } +} + +// TestBundledUnicodeVersionFormat exercises the version-pin parser directly +// against a handful of well- and ill-formed pins, independent of whatever +// unicodeVersionDir currently is, so a future version bump can't accidentally +// leave the parser itself untested. +func TestBundledUnicodeVersionFormat(t *testing.T) { + tests := []struct { + pin string + wantMajor int + wantMinor int + wantPatch int + wantString string + wantErr bool + }{ + {pin: "unicode-17.0", wantMajor: 17, wantMinor: 0, wantPatch: 0, wantString: "17.0.0"}, + {pin: "unicode-15.1.0", wantMajor: 15, wantMinor: 1, wantPatch: 0, wantString: "15.1.0"}, + {pin: "unicode-9.0.1", wantMajor: 9, wantMinor: 0, wantPatch: 1, wantString: "9.0.1"}, + {pin: "17.0", wantErr: true}, // missing "unicode-" prefix + {pin: "unicode-17", wantErr: true}, // needs at least major.minor + {pin: "unicode-17.0.0.0", wantErr: true}, // too many components + {pin: "unicode-17.x", wantErr: true}, // non-numeric component + {pin: "unicode-+17.0", wantErr: true}, // leading '+' must be rejected + {pin: "unicode-07.0", wantErr: true}, // non-canonical leading zero + {pin: "unicode--1.0", wantErr: true}, // negative component + } + for _, tt := range tests { + t.Run(tt.pin, func(t *testing.T) { + version, err := parseVersionPin(tt.pin) + if tt.wantErr { + if err == nil { + t.Fatalf("parseVersionPin(%q): expected error, got version %v", tt.pin, version) + } + return + } + if err != nil { + t.Fatalf("parseVersionPin(%q): unexpected error: %v", tt.pin, err) + } + if version.major != tt.wantMajor || version.minor != tt.wantMinor || version.patch != tt.wantPatch { + t.Fatalf("parseVersionPin(%q) = %+v, want major=%d minor=%d patch=%d", + tt.pin, version, tt.wantMajor, tt.wantMinor, tt.wantPatch) + } + if got := version.String(); got != tt.wantString { + t.Fatalf("parseVersionPin(%q).String() = %q, want %q", tt.pin, got, tt.wantString) + } + }) + } +} + +func TestRepoRootFindsMarker(t *testing.T) { + dir := t.TempDir() + marker := filepath.Join(dir, rootMarker) + if err := os.WriteFile(marker, []byte(""), 0o644); err != nil { + t.Fatalf("writing marker file: %v", err) + } + nested := filepath.Join(dir, "a", "b", "c") + if err := os.MkdirAll(nested, 0o755); err != nil { + t.Fatalf("creating nested dir: %v", err) + } + t.Chdir(nested) + + got, err := repoRoot() + if err != nil { + t.Fatalf("repoRoot: unexpected error: %v", err) + } + // Resolve symlinks (e.g. /tmp -> /private/tmp on macOS) before comparing so + // the test isn't platform-dependent. + wantResolved, err := filepath.EvalSymlinks(dir) + if err != nil { + t.Fatalf("EvalSymlinks(%q): %v", dir, err) + } + gotResolved, err := filepath.EvalSymlinks(got) + if err != nil { + t.Fatalf("EvalSymlinks(%q): %v", got, err) + } + if gotResolved != wantResolved { + t.Fatalf("repoRoot() = %q, want %q", gotResolved, wantResolved) + } +} + +func TestRepoRootMissingMarker(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + if _, err := repoRoot(); err == nil { + t.Fatal("repoRoot(): expected error when no ancestor has settings.gradle.kts, got nil") + } +} diff --git a/tools/internal/ucd/validity_test.go b/tools/internal/ucd/validity_test.go new file mode 100644 index 0000000..2c09643 --- /dev/null +++ b/tools/internal/ucd/validity_test.go @@ -0,0 +1,183 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah +// SPDX-License-Identifier: MIT + +package ucd + +import "testing" + +func TestLoadValidityUnicodeDataMarkAndVirama(t *testing.T) { + // 0300 is COMBINING GRAVE ACCENT: General_Category Mn, CCC 230 (not Virama). + // 094D is DEVANAGARI SIGN VIRAMA: General_Category Mn, CCC 9 (Virama) — so it + // must show up in BOTH the marks and the viramas sets. + // 0041 is LATIN CAPITAL LETTER A: General_Category Lu, CCC 0 — neither. + data := []byte( + "0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;;\n" + + "0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;\n" + + "094D;DEVANAGARI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;;\n", + ) + marks, viramas, err := loadValidityUnicodeData(data) + if err != nil { + t.Fatalf("loadValidityUnicodeData: unexpected error: %v", err) + } + if len(marks) != 2 || marks[0].start != 0x300 || marks[1].start != 0x94D { + t.Fatalf("marks = %+v, want ranges at 0x300 and 0x94D", marks) + } + if len(viramas) != 1 || viramas[0].start != 0x94D { + t.Fatalf("viramas = %+v, want a single range at 0x94D", viramas) + } +} + +func TestLoadValidityUnicodeDataFirstLastBlock(t *testing.T) { + // A <..., First>/<..., Last> block must expand to cover every code point in + // the enclosed run, taking its category/CCC from the Last (closing) row. + data := []byte( + "3400;;Lo;0;L;;;;;N;;;;;\n" + + "4DBF;;Lo;0;L;;;;;N;;;;;\n", + ) + marks, viramas, err := loadValidityUnicodeData(data) + if err != nil { + t.Fatalf("loadValidityUnicodeData: unexpected error: %v", err) + } + // Lo is not a Mark category and CCC 0 is not Virama, so this block should + // contribute nothing to either set — but it must not error out. + if len(marks) != 0 || len(viramas) != 0 { + t.Fatalf("marks=%+v viramas=%+v, want both empty for a non-mark/non-virama block", marks, viramas) + } +} + +func TestLoadValidityUnicodeDataFirstLastBlockAsMark(t *testing.T) { + // Same shape as above, but the Last row is Mn/CCC 230, so the ENTIRE expanded + // range must land in marks with the correct bounds (start from First, end + // from Last). + data := []byte( + "1AB0;;Mn;230;NSM;;;;;N;;;;;\n" + + "1AB4;;Mn;230;NSM;;;;;N;;;;;\n", + ) + marks, _, err := loadValidityUnicodeData(data) + if err != nil { + t.Fatalf("loadValidityUnicodeData: unexpected error: %v", err) + } + if len(marks) != 1 || marks[0].start != 0x1AB0 || marks[0].end != 0x1AB4 { + t.Fatalf("marks = %+v, want a single range [0x1AB0, 0x1AB4]", marks) + } +} + +func TestLoadValidityUnicodeDataInvalidCodePoint(t *testing.T) { + data := []byte("ZZZZ;BOGUS;Lu;0;L;;;;;N;;;;;\n") + if _, _, err := loadValidityUnicodeData(data); err == nil { + t.Fatal("loadValidityUnicodeData with invalid code point: expected error, got nil") + } +} + +func TestLoadValidityUnicodeDataInvalidCCC(t *testing.T) { + data := []byte("0041;LATIN CAPITAL LETTER A;Lu;NOTANUMBER;L;;;;;N;;;;;\n") + if _, _, err := loadValidityUnicodeData(data); err == nil { + t.Fatal("loadValidityUnicodeData with invalid CCC: expected error, got nil") + } +} + +func TestLoadJoiningBasic(t *testing.T) { + data := []byte( + "0620 ; D # ARABIC LETTER KASHMIRI YEH\n" + + "0621 ; U # ARABIC LETTER HAMZA (Non_Joining, dropped)\n" + + "0622..0623 ; R # small range, Right_Joining\n" + + "\n" + + "# a comment-only line\n", + ) + got, err := loadJoining(data) + if err != nil { + t.Fatalf("loadJoining: unexpected error: %v", err) + } + if len(got) != 2 { + t.Fatalf("loadJoining: got %d ranges, want 2 (U type must be dropped)", len(got)) + } + if got[0].start != 0x620 || got[0].end != 0x620 || got[0].jtype != "D" { + t.Errorf("got[0] = %+v, want start=0x620 end=0x620 type=D", got[0]) + } + if got[1].start != 0x622 || got[1].end != 0x623 || got[1].jtype != "R" { + t.Errorf("got[1] = %+v, want start=0x622 end=0x623 type=R", got[1]) + } +} + +func TestLoadJoiningMalformedRecord(t *testing.T) { + // More than one ';' in a data line is malformed. + data := []byte("0620; D; extra\n") + if _, err := loadJoining(data); err == nil { + t.Fatal("loadJoining with a malformed record: expected error, got nil") + } +} + +func TestLoadJoiningInvalidCodeRange(t *testing.T) { + data := []byte("ZZZZ; D\n") + if _, err := loadJoining(data); err == nil { + t.Fatal("loadJoining with an invalid code range: expected error, got nil") + } +} + +func TestMergeSetRangesMergesTouchingAndOverlapping(t *testing.T) { + in := []plainRange{ + {start: 10, end: 20}, + {start: 21, end: 25}, // touches (21 == 20+1) + {start: 24, end: 30}, // overlaps the merged [10,25] + {start: 100, end: 110}, + } + got := mergeSetRanges(in) + if len(got) != 2 { + t.Fatalf("mergeSetRanges: got %d ranges, want 2: %+v", len(got), got) + } + if got[0].start != 10 || got[0].end != 30 { + t.Errorf("got[0] = %+v, want [10, 30]", got[0]) + } + if got[1].start != 100 || got[1].end != 110 { + t.Errorf("got[1] = %+v, want [100, 110]", got[1]) + } +} + +func TestMergeSetRangesSeparatedByOneMerges(t *testing.T) { + // mergeSetRanges' adjacency test is current.start <= merged.end+1, so a range + // separated from the previous by exactly one code point (a single-point gap) + // should still merge. + in := []plainRange{ + {start: 0, end: 5}, + {start: 6, end: 10}, + } + got := mergeSetRanges(in) + if len(got) != 1 || got[0].start != 0 || got[0].end != 10 { + t.Fatalf("mergeSetRanges = %+v, want a single merged [0, 10]", got) + } +} + +func TestMergeSetRangesNoMergeWithGap(t *testing.T) { + in := []plainRange{ + {start: 0, end: 5}, + {start: 7, end: 10}, // gap at 6 + } + got := mergeSetRanges(in) + if len(got) != 2 { + t.Fatalf("mergeSetRanges = %+v, want 2 separate ranges (gap at 6)", got) + } +} + +func TestMergeTypedRangesMergesSameType(t *testing.T) { + in := []typedRange{ + {start: 0, end: 5, jtype: "L"}, + {start: 6, end: 10, jtype: "L"}, + } + got := mergeTypedRanges(in) + if len(got) != 1 || got[0].end != 10 { + t.Fatalf("mergeTypedRanges = %+v, want a single merged range ending at 10", got) + } +} + +func TestMergeTypedRangesDoesNotMergeAcrossTypes(t *testing.T) { + // Adjacent and even touching, but different Joining_Type: must remain separate + // records even though they're contiguous. + in := []typedRange{ + {start: 0, end: 5, jtype: "L"}, + {start: 6, end: 10, jtype: "R"}, + } + got := mergeTypedRanges(in) + if len(got) != 2 { + t.Fatalf("mergeTypedRanges = %+v, want 2 (different types must not merge)", got) + } +} From e5de6676ecefaac358328397b6f14ad5cb20d1e5 Mon Sep 17 00:00:00 2001 From: Ahmad AL-Quraan Date: Sat, 8 Aug 2026 16:06:05 +0300 Subject: [PATCH 2/2] test(tools): address review feedback on internal/ucd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add CI job for tools/ (build, vet, gofmt, test), gated on paths and wired into the aggregate ci gate — these tests previously never ran in CI - Fix TestMergeSetRangesSeparatedByOneMerges asserting behavior mergeSetRanges doesn't have; correct the source comment it was based on - Fix ScalarsToString silently folding negative scalars to U+FFFD instead of erroring - Fix parseLine panicking on a line with no ';' separator - Fix loadValidityUnicodeData panicking on a short line (now matches its sibling loaders) - Add coverage for joiningPath, mergeTypedRanges' sort tiebreakers and containment branch, toPlain/toTyped - Strengthen the NFC skip-short-lines test to actually confirm the real record was processed - Document the parseHexTokens/ScalarsToString validation-strictness discrepancy as an intentional decision, with a pinning test - Note TestBundledUnicodeVersionRenderings' hardcoded literals in the version-bump doc, since the existing grep won't catch them Coverage: 70.1% -> 75.7% --- .github/workflows/ci.yml | 61 +++++++++++++++++++-- docs/idna-unicode-update.md | 7 +++ tools/internal/ucd/mapping.go | 3 + tools/internal/ucd/mapping_test.go | 8 +++ tools/internal/ucd/nfc.go | 7 +++ tools/internal/ucd/nfc_test.go | 33 +++++++++-- tools/internal/ucd/ucd.go | 2 +- tools/internal/ucd/ucd_test.go | 85 ++++++++++++++++++++++++++--- tools/internal/ucd/validity.go | 6 +- tools/internal/ucd/validity_test.go | 78 +++++++++++++++++++++----- 10 files changed, 257 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a7e437..2968985 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -356,18 +356,69 @@ jobs: ~/.android/adb* key: avd-30-aosp_atd-x86_64-pixel6-v1 + tools-changed: + name: Detect tools/ changes + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + changed: ${{ steps.filter.outputs.tools }} + steps: + - uses: actions/checkout@v7 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + tools: + - 'tools/**' + + tools: + name: Go build · vet · fmt · tests (tools/) + needs: tools-changed + if: needs.tools-changed.outputs.changed == 'true' + runs-on: ubuntu-latest + # Needs no JDK and no emulator, so it's the cheapest job in this workflow; a generous + # bound is still cheap insurance against a hung `go test`. + timeout-minutes: 10 + defaults: + run: + working-directory: tools + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v5 + with: + go-version-file: tools/go.mod + - name: Build, vet, gofmt, test + run: | + go build ./... + go vet ./... + if [ -n "$(gofmt -l .)" ]; then + echo "The following files are not gofmt-formatted:" + gofmt -l . + exit 1 + fi + go test ./... + # Aggregate gate: a single check to require in branch protection. Green only when every - # job above succeeded; fails if any was skipped, cancelled or failed. + # job above succeeded; fails if any was skipped, cancelled or failed. `tools` is allowed + # to be legitimately skipped (it's path-gated via tools-changed) so it isn't in `needs` + # directly — only tools-changed is, which always runs and always succeeds or fails cleanly. + # A failing `tools` run still fails branch protection because GitHub reports the `tools` + # check itself as a required status when it runs; when it doesn't run there is nothing to + # gate on and the aggregate correctly ignores it. ci: name: CI if: always() - needs: [ static-analysis, web, native, android-unit, android-instrumented ] + needs: [ static-analysis, web, native, android-unit, android-instrumented, tools-changed, tools ] runs-on: ubuntu-latest # Only evaluates the aggregate result; a couple of minutes is ample. timeout-minutes: 5 steps: - - name: Fail if any job did not succeed - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }} + - name: Fail if any required job did not succeed + # `tools` is expected to report `skipped` when tools/ didn't change (see tools-changed + # above) — that's not a failure, so it's excluded from the failure check while every + # other job (including tools-changed itself) must have succeeded. + if: ${{ contains(fromJSON('["failure","cancelled","skipped"]'), needs.static-analysis.result) || contains(fromJSON('["failure","cancelled","skipped"]'), needs.web.result) || contains(fromJSON('["failure","cancelled","skipped"]'), needs.native.result) || contains(fromJSON('["failure","cancelled","skipped"]'), needs.android-unit.result) || contains(fromJSON('["failure","cancelled","skipped"]'), needs.android-instrumented.result) || contains(fromJSON('["failure","cancelled"]'), needs.tools-changed.result) || contains(fromJSON('["failure","cancelled"]'), needs.tools.result) }} run: | - echo "One or more CI jobs did not succeed: ${{ join(needs.*.result, ', ') }}" + echo "One or more required CI jobs did not succeed." + echo "static-analysis=${{ needs.static-analysis.result }} web=${{ needs.web.result }} native=${{ needs.native.result }} android-unit=${{ needs.android-unit.result }} android-instrumented=${{ needs.android-instrumented.result }} tools-changed=${{ needs.tools-changed.result }} tools=${{ needs.tools.result }}" exit 1 diff --git a/docs/idna-unicode-update.md b/docs/idna-unicode-update.md index ce0f7db..ad989b3 100644 --- a/docs/idna-unicode-update.md +++ b/docs/idna-unicode-update.md @@ -145,6 +145,13 @@ codegen never touches. Search the repo for the outgoing version's `Unicode - the hand-written Kotlin KDocs — `IdnaMappingTable.kt` and `IdnaValidity.kt` (decoders) and `IdnaConformanceTest.kt` and `NormalizerTest.kt` (tests). +Also update `TestBundledUnicodeVersionRenderings` in `tools/internal/ucd/ucd_test.go` by +hand: it asserts the bare `MajorMinor()`/`String()` literals (e.g. `"17.0"`, `"17.0.0"`), +which the grep above won't catch since neither pattern matches an unprefixed version +string. Nothing runs this test as part of a version bump (`go test` isn't part of the +`./gradlew build` gate in step 6, and CI's `tools` job only runs when files under `tools/` +change), so a missed update here fails silently rather than failing the bump. + Then update the "current version" note at the top of this file. (The mapping-table KDoc had already drifted a full major release behind before this was written down, so treat the grep as authoritative rather than trusting this list to stay complete.) diff --git a/tools/internal/ucd/mapping.go b/tools/internal/ucd/mapping.go index 57b3fed..78c3c41 100644 --- a/tools/internal/ucd/mapping.go +++ b/tools/internal/ucd/mapping.go @@ -138,6 +138,9 @@ func parseLine(line string) (*idnaRange, error) { return nil, nil } fields := strings.Split(body, ";") + if len(fields) < 2 { + return nil, fmt.Errorf("ucd: expected at least 2 ';'-separated fields in %q", body) + } for index := range fields { fields[index] = strings.TrimSpace(fields[index]) } diff --git a/tools/internal/ucd/mapping_test.go b/tools/internal/ucd/mapping_test.go index cdd8f78..ef292f9 100644 --- a/tools/internal/ucd/mapping_test.go +++ b/tools/internal/ucd/mapping_test.go @@ -109,6 +109,14 @@ func TestParseLineInvalidCodeRange(t *testing.T) { } } +func TestParseLineMissingSeparator(t *testing.T) { + // A non-blank, non-comment line with no ';' field separator at all must + // return an error rather than panic on the fields[1] access below. + if _, err := parseLine("0041"); err == nil { + t.Fatal("parseLine with no ';' separator: expected error, got nil") + } +} + func TestLoadRangesGapFreeCoverage(t *testing.T) { // A tiny, gap-free, three-range fixture covering the entire code-point space // via a single "10FFFF..10FFFF"-adjacent final record would be unwieldy to diff --git a/tools/internal/ucd/nfc.go b/tools/internal/ucd/nfc.go index 321a2e3..1eb3963 100644 --- a/tools/internal/ucd/nfc.go +++ b/tools/internal/ucd/nfc.go @@ -96,6 +96,13 @@ func loadNfcUnicodeData(data []byte) (map[int]int, map[int][]int, error) { // parseHexTokens splits a whitespace-separated hex mapping into a list of code // points. strings.Fields collapses whitespace runs and trims, matching Python's // no-argument str.split. +// parseHexTokens parses a whitespace-separated list of hex code points (a +// UnicodeData.txt decomposition mapping's targets). Unlike [ScalarsToString], +// this does not reject out-of-range or surrogate values — it validates only +// hex syntax, mirroring the Python ports' int(tok, 16). The real UCD corpus +// never contains such values in this field, so the two hex-list parsers' +// differing strictness is an accepted, intentional discrepancy rather than an +// oversight; see TestParseHexTokensAcceptsOutOfRangeAndSurrogate. func parseHexTokens(mapping string) ([]int, error) { tokens := strings.Fields(mapping) targets := make([]int, 0, len(tokens)) diff --git a/tools/internal/ucd/nfc_test.go b/tools/internal/ucd/nfc_test.go index 738081d..3b41ed7 100644 --- a/tools/internal/ucd/nfc_test.go +++ b/tools/internal/ucd/nfc_test.go @@ -49,14 +49,21 @@ func TestLoadNfcUnicodeDataSkipsCompatibilityDecomposition(t *testing.T) { } func TestLoadNfcUnicodeDataSkipsShortLines(t *testing.T) { - // A line with fewer than six fields (e.g. blank) must be skipped without error. - data := []byte("\n0041;A;Lu;0;L;;;;;N;;;;;\n") + // A line with fewer than six fields (e.g. blank) must be skipped without error, + // and the real record after it must still be processed. Using a fixture whose + // real record contributes both a non-zero CCC and a canonical decomposition + // means the assertion actually confirms the record was parsed, rather than + // merely being consistent with it having been dropped too. + data := []byte("\n00C0;LATIN CAPITAL LETTER A WITH GRAVE;Lu;230;L;0041 0300;;;;N;;;;;\n") ccc, decomposition, err := loadNfcUnicodeData(data) if err != nil { t.Fatalf("loadNfcUnicodeData: unexpected error: %v", err) } - if len(ccc) != 0 || len(decomposition) != 0 { - t.Errorf("ccc=%v decomposition=%v, want both empty for this single-real-record fixture", ccc, decomposition) + if ccc[0xC0] != 230 { + t.Errorf("ccc[0xC0] = %d, want 230 (blank line skipped, real record processed)", ccc[0xC0]) + } + if len(decomposition[0xC0]) != 2 || decomposition[0xC0][0] != 0x41 || decomposition[0xC0][1] != 0x300 { + t.Errorf("decomposition[0xC0] = %v, want [0x41, 0x300]", decomposition[0xC0]) } } @@ -81,6 +88,24 @@ func TestLoadNfcUnicodeDataInvalidDecompositionTarget(t *testing.T) { } } +func TestParseHexTokensAcceptsOutOfRangeAndSurrogate(t *testing.T) { + // Documents a deliberate discrepancy with ScalarsToString: parseHexTokens + // validates hex syntax only, so an out-of-range or surrogate decomposition + // target is accepted rather than rejected. See the doc comment on + // parseHexTokens for why this is intentional (the real corpus never + // contains such values here) rather than an oversight. + data := []byte("00C0;A WITH GRAVE;Lu;0;L;110000 D800;;;;N;;;;;\n") + _, decomposition, err := loadNfcUnicodeData(data) + if err != nil { + t.Fatalf("loadNfcUnicodeData: unexpected error: %v", err) + } + want := []int{0x110000, 0xD800} + got := decomposition[0xC0] + if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("decomposition[0xC0] = %v, want %v (out-of-range/surrogate targets accepted)", got, want) + } +} + func TestLoadExclusions(t *testing.T) { data := []byte( "# 0344 COMBINING GREEK DIALYTIKA TONOS (Non-Starter Decompositions, commented out)\n" + diff --git a/tools/internal/ucd/ucd.go b/tools/internal/ucd/ucd.go index 7c363e8..91fe297 100644 --- a/tools/internal/ucd/ucd.go +++ b/tools/internal/ucd/ucd.go @@ -233,7 +233,7 @@ func ScalarsToString(field string) (string, error) { if err != nil { return "", fmt.Errorf("ucd: parsing hex scalar %q: %w", token, err) } - if code > maxScalar || (code >= firstSurrogate && code <= lastSurrogate) { + if code < 0 || code > maxScalar || (code >= firstSurrogate && code <= lastSurrogate) { return "", fmt.Errorf("ucd: %q is not a valid Unicode scalar value", token) } builder.WriteRune(rune(code)) diff --git a/tools/internal/ucd/ucd_test.go b/tools/internal/ucd/ucd_test.go index 8e50d4d..1436366 100644 --- a/tools/internal/ucd/ucd_test.go +++ b/tools/internal/ucd/ucd_test.go @@ -113,15 +113,10 @@ func TestScalarsToStringAllWhitespace(t *testing.T) { } } -func TestScalarsToStringRejectsSurrogate(t *testing.T) { - // D800 is the first UTF-16 high surrogate; it is not a valid scalar value and - // must not silently become U+FFFD via WriteRune. - if _, err := ScalarsToString("D800"); err == nil { - t.Fatal("ScalarsToString(\"D800\"): expected error for surrogate code point, got nil") - } -} - func TestScalarsToStringRejectsSurrogateBoundaries(t *testing.T) { + // D800 is the first UTF-16 high surrogate; it (and the rest of the surrogate + // block) is not a valid scalar value and must not silently become U+FFFD via + // WriteRune. for _, tok := range []string{"D800", "DFFF", "DC00"} { if _, err := ScalarsToString(tok); err == nil { t.Fatalf("ScalarsToString(%q): expected error for surrogate code point, got nil", tok) @@ -147,6 +142,18 @@ func TestScalarsToStringRejectsOutOfRange(t *testing.T) { } } +func TestScalarsToStringRejectsNegative(t *testing.T) { + // strconv.ParseInt accepts a leading '-', so without an explicit lower-bound + // check a negative token would sail past the (code > maxScalar) guard and + // WriteRune would silently fold it to U+FFFD — exactly what the doc comment + // promises can't happen. + for _, tok := range []string{"-1", "-0041"} { + if _, err := ScalarsToString(tok); err == nil { + t.Fatalf("ScalarsToString(%q): expected error for negative scalar, got nil", tok) + } + } +} + func TestScalarsToStringRejectsInvalidHex(t *testing.T) { if _, err := ScalarsToString("ZZZZ"); err == nil { t.Fatal("ScalarsToString(\"ZZZZ\"): expected error, got nil") @@ -257,6 +264,68 @@ func TestRepoRootFindsMarker(t *testing.T) { } } +func TestJoiningPathPrefersExtracted(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, rootMarker), nil, 0o644); err != nil { + t.Fatal(err) + } + dir := filepath.Join(root, ".claude", "references", unicodeVersionDir) + extractedDir := filepath.Join(dir, "extracted") + if err := os.MkdirAll(extractedDir, 0o755); err != nil { + t.Fatal(err) + } + extracted := filepath.Join(extractedDir, "DerivedJoiningType.txt") + topLevel := filepath.Join(dir, "DerivedJoiningType.txt") + for _, p := range []string{extracted, topLevel} { + if err := os.WriteFile(p, nil, 0o644); err != nil { + t.Fatal(err) + } + } + t.Chdir(root) + got, err := joiningPath() + if err != nil { + t.Fatalf("joiningPath: unexpected error: %v", err) + } + if got != extracted { + t.Fatalf("joiningPath() = %q, want the extracted/ copy %q", got, extracted) + } +} + +func TestJoiningPathFallsBackToTopLevel(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, rootMarker), nil, 0o644); err != nil { + t.Fatal(err) + } + dir := filepath.Join(root, ".claude", "references", unicodeVersionDir) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + // Only the top-level file exists — no extracted/ subdirectory at all. + topLevel := filepath.Join(dir, "DerivedJoiningType.txt") + if err := os.WriteFile(topLevel, nil, 0o644); err != nil { + t.Fatal(err) + } + t.Chdir(root) + got, err := joiningPath() + if err != nil { + t.Fatalf("joiningPath: unexpected error: %v", err) + } + if got != topLevel { + t.Fatalf("joiningPath() = %q, want the top-level fallback %q", got, topLevel) + } +} + +func TestJoiningPathNeitherExists(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, rootMarker), nil, 0o644); err != nil { + t.Fatal(err) + } + t.Chdir(root) + if _, err := joiningPath(); err == nil { + t.Fatal("joiningPath: expected error when neither candidate exists, got nil") + } +} + func TestRepoRootMissingMarker(t *testing.T) { dir := t.TempDir() t.Chdir(dir) diff --git a/tools/internal/ucd/validity.go b/tools/internal/ucd/validity.go index b24e15d..3cad7b4 100644 --- a/tools/internal/ucd/validity.go +++ b/tools/internal/ucd/validity.go @@ -122,6 +122,9 @@ func loadValidityUnicodeData(data []byte) (marks, viramas []plainRange, err erro pendingStart := -1 for scanner.Scan() { fields := strings.Split(scanner.Text(), ";") + if len(fields) <= fieldCCC { + continue + } codePoint, parseErr := strconv.ParseInt(fields[fieldCodePoint], 16, 32) if parseErr != nil { return nil, nil, fmt.Errorf("ucd: parsing code point %q: %w", fields[fieldCodePoint], parseErr) @@ -190,7 +193,8 @@ func loadJoining(data []byte) ([]typedRange, error) { } // mergeSetRanges sorts plain ranges by (start, end) and adjacency-merges any -// pair that touches, overlaps, or is separated by a single code point. +// pair that touches (current.start == merged.end+1) or overlaps. A gap of one +// or more uncovered code points between two ranges is preserved, not merged. func mergeSetRanges(ranges []plainRange) []plainRange { sort.Slice(ranges, func(i, j int) bool { if ranges[i].start != ranges[j].start { diff --git a/tools/internal/ucd/validity_test.go b/tools/internal/ucd/validity_test.go index 2c09643..201349b 100644 --- a/tools/internal/ucd/validity_test.go +++ b/tools/internal/ucd/validity_test.go @@ -27,6 +27,27 @@ func TestLoadValidityUnicodeDataMarkAndVirama(t *testing.T) { } } +func TestLoadValidityUnicodeDataSkipsShortLines(t *testing.T) { + // A line with fewer than four fields must be skipped without a panic on the + // fields[fieldCCC] access, mirroring loadNfcUnicodeData and loadBidiUnicodeData. + // 094D (DEVANAGARI SIGN VIRAMA) is category Mn (a mark) and CCC 9 (Virama), so + // the real record after the short line must land in both sets. + data := []byte( + "0041\n" + + "094D;DEVANAGARI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;;\n", + ) + marks, viramas, err := loadValidityUnicodeData(data) + if err != nil { + t.Fatalf("loadValidityUnicodeData: unexpected error: %v", err) + } + if len(marks) != 1 || marks[0].start != 0x94D { + t.Fatalf("marks = %+v, want a single range at 0x94D (short line skipped)", marks) + } + if len(viramas) != 1 || viramas[0].start != 0x94D { + t.Fatalf("viramas = %+v, want a single range at 0x94D (short line skipped)", viramas) + } +} + func TestLoadValidityUnicodeDataFirstLastBlock(t *testing.T) { // A <..., First>/<..., Last> block must expand to cover every code point in // the enclosed run, taking its category/CCC from the Last (closing) row. @@ -133,20 +154,6 @@ func TestMergeSetRangesMergesTouchingAndOverlapping(t *testing.T) { } } -func TestMergeSetRangesSeparatedByOneMerges(t *testing.T) { - // mergeSetRanges' adjacency test is current.start <= merged.end+1, so a range - // separated from the previous by exactly one code point (a single-point gap) - // should still merge. - in := []plainRange{ - {start: 0, end: 5}, - {start: 6, end: 10}, - } - got := mergeSetRanges(in) - if len(got) != 1 || got[0].start != 0 || got[0].end != 10 { - t.Fatalf("mergeSetRanges = %+v, want a single merged [0, 10]", got) - } -} - func TestMergeSetRangesNoMergeWithGap(t *testing.T) { in := []plainRange{ {start: 0, end: 5}, @@ -169,6 +176,49 @@ func TestMergeTypedRangesMergesSameType(t *testing.T) { } } +func TestMergeTypedRangesSortTiebreakersAndContainment(t *testing.T) { + // Exercises the sort comparator's equal-start tiebreaker (falls through to + // comparing end), equal-end tiebreaker (falls through to comparing jtype), + // and the containment branch (a range fully inside the one already merged). + in := []typedRange{ + {start: 0, end: 20, jtype: "L"}, // same start as next, larger end + {start: 0, end: 10, jtype: "L"}, // same start as prev, smaller end + {start: 5, end: 8, jtype: "L"}, // fully contained in [0,20] once merged + {start: 30, end: 40, jtype: "R"}, // same end as next, different jtype + {start: 30, end: 40, jtype: "L"}, + } + got := mergeTypedRanges(in) + if len(got) != 3 { + t.Fatalf("mergeTypedRanges = %+v, want 3 merged records", got) + } + if got[0].start != 0 || got[0].end != 20 || got[0].jtype != "L" { + t.Errorf("got[0] = %+v, want [0,20] L (containment must not shrink the merged end)", got[0]) + } + // The two [30,40] ranges have different jtypes, so — despite sharing the same + // start and end — they must remain separate records, sorted by jtype ("L" < "R"). + if got[1].start != 30 || got[1].end != 40 || got[1].jtype != "L" { + t.Errorf("got[1] = %+v, want [30,40] L", got[1]) + } + if got[2].start != 30 || got[2].end != 40 || got[2].jtype != "R" { + t.Errorf("got[2] = %+v, want [30,40] R", got[2]) + } +} + +func TestToPlainConvertsRanges(t *testing.T) { + got := toPlain([]plainRange{{start: 1, end: 2}, {start: 5, end: 9}}) + want := []PlainRange{{Start: 1, End: 2}, {Start: 5, End: 9}} + if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("toPlain = %+v, want %+v", got, want) + } +} + +func TestToTypedConvertsRanges(t *testing.T) { + got := toTyped([]typedRange{{start: 1, end: 2, jtype: "L"}}) + if len(got) != 1 || got[0].Start != 1 || got[0].End != 2 || got[0].Type != "L" { + t.Fatalf("toTyped = %+v, want a single [1,2] L record", got) + } +} + func TestMergeTypedRangesDoesNotMergeAcrossTypes(t *testing.T) { // Adjacent and even touching, but different Joining_Type: must remain separate // records even though they're contiguous.