diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index dfc485d..6b4a23d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -23,7 +23,7 @@ jobs: - name: Run tests run: cargo test --verbose - name: Build docs - run: cargo doc + run: cargo doc && cargo doc --no-default-features - name: Check formatting run: cargo fmt --check - name: Check clippy @@ -32,6 +32,10 @@ jobs: run: cargo build --verbose --no-default-features - name: Run tests (no default features) run: cargo test --verbose --no-default-features + - name: Run tests (ASCII fast path) + run: | + cargo test --verbose --features ascii-fast-path + cargo test --verbose --no-default-features --features ascii-fast-path - name: Check clippy (no default features) run: cargo clippy --verbose --lib --tests --no-default-features @@ -42,6 +46,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' + - name: Test generator + run: python3 -m unittest discover -s scripts - name: Regen run: rm tests/emoji-test.txt && cd scripts && python3 unicode.py - name: Diff @@ -54,4 +60,4 @@ jobs: - uses: dtolnay/rust-toolchain@master with: toolchain: 1.66.0 - - run: cargo check --lib --features core,std + - run: cargo check --lib --features core,std,ascii-fast-path diff --git a/Cargo.toml b/Cargo.toml index 0801bbc..93cb1f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,10 +18,7 @@ categories = [ ] keywords = ["text", "width", "unicode"] readme = "README.md" -description = """ -Determine displayed width of `char` and `str` types -according to Unicode Standard Annex #11 rules. -""" +description = "Estimate terminal-column widths of Unicode characters and strings." edition = "2021" rust-version = "1.66" @@ -42,6 +39,7 @@ core = { version = "1.0", package = "rustc-std-workspace-core", optional = true [features] cjk = [] +ascii-fast-path = [] default = ["cjk"] rustc-dep-of-std = ['std', 'core'] diff --git a/README.md b/README.md index 07b5039..b0680c4 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ [![crates.io version](https://img.shields.io/crates/v/unicode-width)](https://crates.io/crates/unicode-width) [![Docs status](https://img.shields.io/docsrs/unicode-width)](https://docs.rs/unicode-width/) -Determine displayed width of `char` and `str` types according to [Unicode Standard Annex #11][UAX11] -and other portions of the Unicode standard. +Determine terminal-column widths of `char` and `str` types using Unicode properties +and the terminal rendering policies described below. This crate is `#![no_std]`. @@ -24,6 +24,13 @@ fn main() { } ``` +## Unicode support and terminal policies + +The crate uses Unicode 18.0 data, including [UAX #11][UAX11] (East Asian Width), +with terminal rendering policies. The [API documentation](https://docs.rs/unicode-width/latest/unicode_width/) +lists the supported Unicode rules, variation-selector and emoji fallbacks, and +the limits of terminal-width estimates. + **NOTE:** The computed width values may not match the actual rendered column width. For example, many Brahmic scripts like Devanagari have complex rendering rules which this crate does not currently handle (and will never fully handle, because diff --git a/benches/benches.rs b/benches/benches.rs index e4fcee1..93b984d 100644 --- a/benches/benches.rs +++ b/benches/benches.rs @@ -17,6 +17,40 @@ use test::Bencher; use unicode_width::{UnicodeWidthChar, UnicodeWidthStr}; +#[bench] +fn ascii(b: &mut Bencher) { + let string = "The quick brown fox jumps over the lazy dog. ".repeat(128); + b.iter(|| test::black_box(UnicodeWidthStr::width(test::black_box(string.as_str())))); +} + +#[bench] +fn short_unicode(b: &mut Bencher) { + b.iter(|| test::black_box(UnicodeWidthStr::width(test::black_box("漢")))); +} + +#[bench] +fn mixed_unicode(b: &mut Bencher) { + let string = "a".repeat(61) + "é"; + b.iter(|| test::black_box(UnicodeWidthStr::width(test::black_box(string.as_str())))); +} + +#[bench] +fn unicode(b: &mut Bencher) { + let string = "Unicode width: élève 日本語 한글 العربية 👩‍🔬\r\n".repeat(128); + b.iter(|| test::black_box(UnicodeWidthStr::width(test::black_box(string.as_str())))); +} + +#[bench] +fn unicode_scanner(b: &mut Bencher) { + let string = "élève Καλημέρα Привет 漢字カタカナ ".repeat(128); + b.iter(|| unicode_width::char_iter_width(test::black_box(string.as_str()).chars())); +} + +#[bench] +fn short_unicode_scanner(b: &mut Bencher) { + b.iter(|| unicode_width::char_iter_width(test::black_box("helloé").chars())); +} + #[bench] fn cargo(b: &mut Bencher) { let string = iter::repeat('a').take(4096).collect::(); @@ -92,7 +126,7 @@ fn simple_width_match(c: char) -> Option { fn enwik8(b: &mut Bencher) { // To benchmark, download & unzip `enwik8` from https://data.deepai.org/enwik8.zip let data_path = "bench_data/enwik8"; - let string = std::fs::read_to_string(data_path).unwrap_or_default(); + let string = std::fs::read_to_string(data_path).expect("Download and unpack enwik8 first."); b.iter(|| test::black_box(UnicodeWidthStr::width(string.as_str()))); } @@ -101,7 +135,8 @@ fn jawiki(b: &mut Bencher) { // To benchmark, download & extract `jawiki-20220501-pages-articles-multistream-index.txt` from // https://dumps.wikimedia.org/jawiki/20220501/jawiki-20220501-pages-articles-multistream-index.txt.bz2 let data_path = "bench_data/jawiki-20220501-pages-articles-multistream-index.txt"; - let string = std::fs::read_to_string(data_path).unwrap_or_default(); + let string = + std::fs::read_to_string(data_path).expect("Download and extract the jawiki index first."); b.iter(|| test::black_box(UnicodeWidthStr::width(string.as_str()))); } @@ -109,6 +144,6 @@ fn jawiki(b: &mut Bencher) { fn emoji(b: &mut Bencher) { // To benchmark, download emoji-style.txt from https://www.unicode.org/emoji/charts/emoji-style.txt let data_path = "bench_data/emoji-style.txt"; - let string = std::fs::read_to_string(data_path).unwrap_or_default(); + let string = std::fs::read_to_string(data_path).expect("Download emoji-style.txt first."); b.iter(|| test::black_box(UnicodeWidthStr::width(string.as_str()))); } diff --git a/scripts/test_unicode.py b/scripts/test_unicode.py new file mode 100644 index 0000000..2dd7a12 --- /dev/null +++ b/scripts/test_unicode.py @@ -0,0 +1,112 @@ +import io +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import unicode + + +class UnicodeDataTests(unittest.TestCase): + def test_property_records(self): + records = " # comment\n0041;AI\n 0042..0043 ; AL#comment\n0044;AIX\n0045;AI" + with patch.object(unicode, "fetch_open", return_value=io.StringIO(records)): + found = [] + unicode.load_property("LineBreak.txt", "AI|AL", found.append) + self.assertEqual(found, [0x41, 0x42, 0x43, 0x45]) + + def test_east_asian_width_records_and_defaults(self): + records = """# @missing: 0000..10FFFF; N +003D;A +0041;Na +00A1;A#arbitrary comment +00A2;H +00A3;F +00A4..00A5;W +00A6;N +2260..2261;N +F900..FAFF;W +20000..2FFFD;W +30000..3FFFD;W +# @missing: 0041..0043; W +# @missing: 0000..10FFFF; N +# @missing: 0042; W +""" + with patch.object( + unicode, "fetch_open", + side_effect=lambda name: io.StringIO({ + "EastAsianWidth.txt": records, + "UnicodeData.txt": "2260;NOT EQUAL TO;Sm;0;ON;003D 0338;\n2261;COMPATIBILITY FORM;Sm;0;ON; 003D 0338;\n", + }.get(name, "")), + ): + widths = unicode.load_east_asian_widths() + narrow, wide, ambiguous = unicode.EastAsianWidth + for cp, expected in [ + (0, narrow), (0x41, narrow), (0x42, wide), (0x43, narrow), + (0xA1, ambiguous), (0xA2, narrow), (0xA3, wide), (0xA4, wide), + (0xA5, wide), (0xA6, narrow), (0xA7, narrow), + (0x2260, ambiguous), (0x2261, narrow), + (0xF8FF, narrow), (0xF900, wide), (0xFAFF, wide), (0xFB00, narrow), + (0x1FFFF, narrow), (0x20000, wide), (0x2FFFD, wide), (0x2FFFE, narrow), + (0x30000, wide), (0x3FFFD, wide), (0x3FFFE, narrow), (0x10FFFF, narrow), + ]: + with self.subTest(cp=hex(cp)): + self.assertEqual(widths[cp], expected) + + def test_solidus_canonical_decompositions(self): + records = """0342;RECURSIVE;Mn;0;NSM;0341; +0341;CANONICAL;Mn;0;NSM;0301 0301; +0343;COMPATIBILITY;Mn;0;NSM; 0301; +0344;EMPTY;Mn;0;NSM;; +0345;MIXED;Mn;0;NSM;0301 0041; +0346;NUMERAL;No;0;L;;;;300;N;;;;; +""" + with patch.object(unicode, "fetch_open", side_effect=lambda name: io.StringIO( + records if name == "UnicodeData.txt" else "0300..0301;230\n" + )): + widths = [unicode.WidthState.NARROW] * 0x347 + self.assertEqual(unicode.load_solidus_transparent([], widths), [(0x300, 0x301), (0x341, 0x342)]) + + def test_versioned_cache(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "UnicodeData.txt").write_text("stale data", encoding="utf-8") + expected = root / f"{unicode.UNICODE_VERSION}-UnicodeData.txt" + + def download(url, path): + self.assertEqual(url, f"https://www.unicode.org/Public/{unicode.UNICODE_VERSION}/ucd/UnicodeData.txt") + self.assertEqual(Path(path), expected) + expected.write_text("current data", encoding="utf-8") + + with patch.object(unicode.urllib.request, "urlretrieve", side_effect=download) as fetch: + for _ in range(2): + with unicode.fetch_open("UnicodeData.txt", directory) as data: + self.assertEqual(data.read(), "current data") + fetch.assert_called_once() + + def test_version_headers(self): + version = unicode.UNICODE_VERSION + emoji_version = version.rsplit(".", 1)[0] + for filename, emoji, header in [ + ("extracted/DerivedGeneralCategory.txt", False, f"# DerivedGeneralCategory-{version}.txt\n"), + ("ReadMe.txt", False, f"Final data files for version {version} of the\nUnicode Character Database.\n"), + ("emoji/emoji-data.txt", False, f"# emoji-data.txt\n# Version: {version}\n"), + ("emoji-test.txt", True, f"# emoji-test.txt\n# Version: {emoji_version}\n"), + ]: + with self.subTest(filename=filename), tempfile.TemporaryDirectory() as directory: + basename = Path(filename).name + cached = basename if emoji else f"{version}-{basename}" + path = Path(directory) / cached + path.write_text(header, encoding="utf-8") + with patch.object(unicode.urllib.request, "urlretrieve") as fetch: + with unicode.fetch_open(filename, directory, emoji) as data: + self.assertEqual(data.read(), header) + fetch.assert_not_called() + for invalid in [header.replace(version, "0.0.0").replace(emoji_version, "0.0"), "no version\n"]: + path.write_text(invalid, encoding="utf-8") + with self.assertRaisesRegex(ValueError, "expected Unicode"): + unicode.fetch_open(filename, directory, emoji) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/unicode.py b/scripts/unicode.py index 5b6fbb1..7ea66b4 100755 --- a/scripts/unicode.py +++ b/scripts/unicode.py @@ -40,7 +40,7 @@ import sys import urllib.request from collections import defaultdict -from itertools import batched +from itertools import batched, chain from typing import Callable, Iterable, IO UNICODE_VERSION = "18.0.0" @@ -118,7 +118,10 @@ def fetch_open(filename: str, local_prefix: str = "", emoji: bool = False): fetches it from `https://www.unicode.org/Public/`. Exits with code 1 on failure. """ basename = os.path.basename(filename) - localname = os.path.join(local_prefix, basename) + # UnicodeData.txt has no version header, so cache UCD files by version. + localname = os.path.join( + local_prefix, basename if emoji else f"{UNICODE_VERSION}-{basename}" + ) if not os.path.exists(localname): if emoji: prefix = "emoji" @@ -129,10 +132,23 @@ def fetch_open(filename: str, local_prefix: str = "", emoji: bool = False): localname, ) try: - return open(localname, encoding="utf-8") + data = open(localname, encoding="utf-8") except OSError: sys.stderr.write(f"cannot load {localname}") sys.exit(1) + if basename != "UnicodeData.txt": + version = re.search( + r"(?:-\b|Version:\s*|for version )(\d+\.\d+(?:\.\d+)?)", + data.read(1024), + ) + found = version.group(1) if version else "unknown" + if found.count(".") == 1: + found += ".0" + if found != UNICODE_VERSION: + data.close() + raise ValueError(f"{localname}: expected Unicode {UNICODE_VERSION}, found {found}") + data.seek(0) + return data def load_unicode_version() -> tuple[int, int, int]: @@ -144,19 +160,15 @@ def load_unicode_version() -> tuple[int, int, int]: def load_property(filename: str, pattern: str, action: Callable[[int], None]): with fetch_open(filename) as properties: - single = re.compile(rf"^([0-9A-F]+)\s*;\s*{pattern}\s+") - multiple = re.compile(rf"^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*{pattern}\s+") + record = re.compile( + rf"^\s*([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*(?:{pattern})(?=\s|#|$)" + ) - for line in properties.readlines(): - raw_data = None # (low, high) - if match := single.match(line): - raw_data = (match.group(1), match.group(1)) - elif match := multiple.match(line): - raw_data = (match.group(1), match.group(2)) - else: + for line in properties: + if not (match := record.match(line)): continue - low = int(raw_data[0], 16) - high = int(raw_data[1], 16) + low = int(match.group(1), 16) + high = int(match.group(2) or match.group(1), 16) for cp in range(low, high + 1): action(cp) @@ -344,7 +356,7 @@ class WidthState(enum.IntEnum): # Hebrew alef lamed HEBREW_LETTER_LAMED = 0b0011_1000_0000_0000 - "\\u05DC (Alef-ZWJ-Lamed ligature)" + "\\u05DC or \\uFB3C (Alef-ZWJ-Lamed ligature)" ZWJ_HEBREW_LETTER_LAMED = 0b0011_1100_0000_0000 "\\u200D\\u05DC (Alef-ZWJ-Lamed ligature)" @@ -450,10 +462,6 @@ def load_east_asian_widths() -> list[EastAsianWidth]: `Ambiguous` characters are assigned `EffectiveWidth.AMBIGUOUS`.""" with fetch_open("EastAsianWidth.txt") as eaw: - # matches a width assignment for a single codepoint, i.e. "1F336;N # ..." - single = re.compile(r"^([0-9A-F]+)\s*;\s*(\w+) +# (\w+)") - # matches a width assignment for a range of codepoints, i.e. "3001..3003;W # ..." - multiple = re.compile(r"^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*(\w+) +# (\w+)") # map between width category code and condensed width width_codes = { **{c: EastAsianWidth.NARROW for c in ["N", "Na", "H"]}, @@ -461,30 +469,26 @@ def load_east_asian_widths() -> list[EastAsianWidth]: "A": EastAsianWidth.AMBIGUOUS, } - width_map = [] - current = 0 - for line in eaw.readlines(): - raw_data = None # (low, high, width) - if match := single.match(line): - raw_data = (match.group(1), match.group(1), match.group(2)) - elif match := multiple.match(line): - raw_data = (match.group(1), match.group(2), match.group(3)) - else: + defaults = [] + ranges = [] + for line in eaw: + is_default = line.startswith("# @missing:") + fields = line.removeprefix("# @missing:") if is_default else line + fields = fields.partition("#")[0].strip() + if not fields: continue - low = int(raw_data[0], 16) - high = int(raw_data[1], 16) - width = width_codes[raw_data[2]] - - assert current <= high - while current <= high: - # Some codepoints don't fall into any of the ranges in EastAsianWidth.txt. - # All such codepoints are implicitly given Neural width (resolves to narrow) - width_map.append(EastAsianWidth.NARROW if current < low else width) - current += 1 + codepoints, value = fields.split(";") + low, _, high = codepoints.strip().partition("..") + low, high = int(low, 16), int(high or low, 16) + assert 0 <= low <= high < NUM_CODEPOINTS + width = width_codes[value.strip()] + if is_default and not defaults and low == 0 and high == NUM_CODEPOINTS - 1 and value.strip() == "N": + continue + (defaults if is_default else ranges).append((low, high, width)) - while len(width_map) < NUM_CODEPOINTS: - # Catch any leftover codepoints and assign them implicit Neutral/narrow width. - width_map.append(EastAsianWidth.NARROW) + width_map = [EastAsianWidth.NARROW] * NUM_CODEPOINTS + for low, high, width in chain(defaults, ranges): + width_map[low : high + 1] = [width] * (high - low + 1) # Characters with ambiguous line breaking are ambiguous load_property( @@ -509,11 +513,12 @@ def load_east_asian_widths() -> list[EastAsianWidth]: # Canonical equivalence for symbols with stroke with fetch_open("UnicodeData.txt") as udata: - single = re.compile(r"([0-9A-Z]+);.*?;.*?;.*?;.*?;([0-9A-Z]+) 0338;") - for line in udata.readlines(): - if match := single.match(line): - composed = int(match.group(1), 16) - decomposed = int(match.group(2), 16) + for line in udata: + fields = line.split(";", 6) + decomposition = fields[5].split() + if len(decomposition) == 2 and decomposition[1] == "0338": + composed = int(fields[0], 16) + decomposed = int(decomposition[0], 16) if width_map[decomposed] == EastAsianWidth.AMBIGUOUS: width_map[composed] = EastAsianWidth.AMBIGUOUS @@ -670,7 +675,7 @@ def load_width_maps() -> tuple[list[WidthState], list[WidthState]]: for cps, width in [ ([0x0A], WidthState.LINE_FEED), - ([0x05DC], WidthState.HEBREW_LETTER_LAMED), + ([0x05DC, 0xFB3C], WidthState.HEBREW_LETTER_LAMED), (alef_joining, WidthState.JOINING_GROUP_ALEF), (range(0x1780, 0x1783), WidthState.KHMER_COENG_ELIGIBLE_LETTER), (range(0x1784, 0x1788), WidthState.KHMER_COENG_ELIGIBLE_LETTER), @@ -792,13 +797,12 @@ def load_solidus_transparent( # Recursive decompositions while True: with fetch_open("UnicodeData.txt") as udata: - single = re.compile(r"([0-9A-Z]+);.*?;.*?;.*?;.*?;([0-9A-F ]+);") - for line in udata.readlines(): - if match := single.match(line): - composed = int(match.group(1), 16) - decomposed = [int(c, 16) for c in match.group(2).split(" ")] - if all([c in ccc_above_1 for c in decomposed]): - ccc_above_1.add(composed) + for line in udata: + fields = line.split(";", 6) + decomposition = fields[5].split() + if decomposition and not decomposition[0].startswith("<"): + if all(int(c, 16) in ccc_above_1 for c in decomposition): + ccc_above_1.add(int(fields[0], 16)) if len(ccc_above_1) == num_chars: break else: @@ -1256,7 +1260,7 @@ def lookup_fns( /// nothing to worry about if you re-run `unicode.py` (for example, when updating Unicode.) /// However, if you change the *actual structure* of the lookup tables (perhaps by editing the /// `make_tables` function in `unicode.py`) you must ensure that this code reflects those changes. -{cfg}#[inline] +{cfg}#[inline(always)] pub(crate) fn lookup_width{cjk_lo}(c: char) -> (u8, WidthInfo) {{ let cp = c as usize; diff --git a/src/gen/lookup.rs b/src/gen/lookup.rs index 299d134..7267c52 100644 --- a/src/gen/lookup.rs +++ b/src/gen/lookup.rs @@ -20,7 +20,7 @@ use crate::width_info::WidthInfo; /// nothing to worry about if you re-run `unicode.py` (for example, when updating Unicode.) /// However, if you change the *actual structure* of the lookup tables (perhaps by editing the /// `make_tables` function in `unicode.py`) you must ensure that this code reflects those changes. -#[inline] +#[inline(always)] pub(crate) fn lookup_width(c: char) -> (u8, WidthInfo) { let cp = c as usize; @@ -52,6 +52,7 @@ pub(crate) fn lookup_width(c: char) -> (u8, WidthInfo) { '\u{1A10}' => (1, WidthInfo::BUGINESE_LETTER_YA), '\u{2D31}'..='\u{2D6F}' => (1, WidthInfo::TIFINAGH_CONSONANT), '\u{A4FC}'..='\u{A4FD}' => (1, WidthInfo::LISU_TONE_LETTER_MYA_NA_JEU), + '\u{FB3C}' => (1, WidthInfo::HEBREW_LETTER_LAMED), '\u{FE01}' => (0, WidthInfo::VARIATION_SELECTOR_1_2_OR_3), '\u{FE0E}' => (0, WidthInfo::VARIATION_SELECTOR_15), '\u{FE0F}' => (0, WidthInfo::VARIATION_SELECTOR_16), @@ -73,7 +74,7 @@ pub(crate) fn lookup_width(c: char) -> (u8, WidthInfo) { /// However, if you change the *actual structure* of the lookup tables (perhaps by editing the /// `make_tables` function in `unicode.py`) you must ensure that this code reflects those changes. #[cfg(feature = "cjk")] -#[inline] +#[inline(always)] pub(crate) fn lookup_width_cjk(c: char) -> (u8, WidthInfo) { let cp = c as usize; @@ -106,6 +107,7 @@ pub(crate) fn lookup_width_cjk(c: char) -> (u8, WidthInfo) { '\u{1A10}' => (1, WidthInfo::BUGINESE_LETTER_YA), '\u{2D31}'..='\u{2D6F}' => (1, WidthInfo::TIFINAGH_CONSONANT), '\u{A4FC}'..='\u{A4FD}' => (1, WidthInfo::LISU_TONE_LETTER_MYA_NA_JEU), + '\u{FB3C}' => (1, WidthInfo::HEBREW_LETTER_LAMED), '\u{FE00}'..='\u{FE02}' => (0, WidthInfo::VARIATION_SELECTOR_1_2_OR_3), '\u{FE0F}' => (0, WidthInfo::VARIATION_SELECTOR_16), '\u{10C03}' => (1, WidthInfo::OLD_TURKIC_LETTER_ORKHON_I), diff --git a/src/gen/tables.rs b/src/gen/tables.rs index 38d9ed0..dd66ebb 100644 --- a/src/gen/tables.rs +++ b/src/gen/tables.rs @@ -597,7 +597,7 @@ pub(crate) static WIDTH_LEAVES: Align32<[[u8; 32]; WIDTH_LEAVES_LEN]> = Align32( ], [ 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x45, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, - 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x57, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, ], [ @@ -1270,7 +1270,7 @@ pub(crate) static NON_TRANSPARENT_ZERO_WIDTHS: [([u8; 3], [u8; 3]); 74] = [ /// (mostly ccc > 1). /// FIXME: can we get better compression? #[cfg(feature = "cjk")] -pub(crate) static SOLIDUS_TRANSPARENT: [([u8; 3], [u8; 3]); 212] = [ +pub(crate) static SOLIDUS_TRANSPARENT: [([u8; 3], [u8; 3]); 205] = [ ([0x00, 0x03, 0x00], [0x33, 0x03, 0x00]), ([0x39, 0x03, 0x00], [0x6F, 0x03, 0x00]), ([0x83, 0x04, 0x00], [0x87, 0x04, 0x00]), @@ -1393,13 +1393,9 @@ pub(crate) static SOLIDUS_TRANSPARENT: [([u8; 3], [u8; 3]); 212] = [ ([0xED, 0xAB, 0x00], [0xED, 0xAB, 0x00]), ([0x1E, 0xFB, 0x00], [0x1E, 0xFB, 0x00]), ([0x20, 0xFE, 0x00], [0x2F, 0xFE, 0x00]), - ([0x1B, 0x01, 0x01], [0x1B, 0x01, 0x01]), - ([0x6B, 0x01, 0x01], [0x6B, 0x01, 0x01]), ([0xFD, 0x01, 0x01], [0xFD, 0x01, 0x01]), ([0xE0, 0x02, 0x01], [0xE0, 0x02, 0x01]), - ([0xF5, 0x02, 0x01], [0xF5, 0x02, 0x01]), ([0x76, 0x03, 0x01], [0x7A, 0x03, 0x01]), - ([0xD4, 0x09, 0x01], [0xD4, 0x09, 0x01]), ([0x0D, 0x0A, 0x01], [0x0D, 0x0A, 0x01]), ([0x0F, 0x0A, 0x01], [0x0F, 0x0A, 0x01]), ([0x38, 0x0A, 0x01], [0x38, 0x0A, 0x01]), @@ -1408,7 +1404,6 @@ pub(crate) static SOLIDUS_TRANSPARENT: [([u8; 3], [u8; 3]); 212] = [ ([0xE5, 0x0A, 0x01], [0xE6, 0x0A, 0x01]), ([0x24, 0x0D, 0x01], [0x27, 0x0D, 0x01]), ([0x69, 0x0D, 0x01], [0x6D, 0x0D, 0x01]), - ([0x74, 0x0E, 0x01], [0x74, 0x0E, 0x01]), ([0xAB, 0x0E, 0x01], [0xAC, 0x0E, 0x01]), ([0xCB, 0x0E, 0x01], [0xCF, 0x0E, 0x01]), ([0xF0, 0x0E, 0x01], [0xFB, 0x0E, 0x01]), @@ -1481,8 +1476,6 @@ pub(crate) static SOLIDUS_TRANSPARENT: [([u8; 3], [u8; 3]); 212] = [ ([0xF5, 0xE6, 0x01], [0xF5, 0xE6, 0x01]), ([0xD0, 0xE8, 0x01], [0xD6, 0xE8, 0x01]), ([0x44, 0xE9, 0x01], [0x4A, 0xE9, 0x01]), - ([0x85, 0xEC, 0x01], [0x85, 0xEC, 0x01]), - ([0x15, 0xED, 0x01], [0x15, 0xED, 0x01]), ]; /// Array of 1024-bit bitmaps. Index into the correct bitmap with the 10 LSB of your codepoint diff --git a/src/lib.rs b/src/lib.rs index aa9e5d0..1e5dfd9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,9 +8,8 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -//! Determine displayed width of `char` and `str` types according to -//! [Unicode Standard Annex #11](http://www.unicode.org/reports/tr11/) -//! and other portions of the Unicode standard. +//! Determine terminal-column widths of `char` and `str` types using Unicode +//! properties and the terminal rendering policies described below. //! See the [Rules for determining width](#rules-for-determining-width) section //! for the exact rules. //! @@ -27,10 +26,9 @@ //! //! # `"cjk"` feature flag //! -//! This crate has one Cargo feature flag, `"cjk"` -//! (enabled by default). -//! It enables the [`UnicodeWidthChar::width_cjk`] -//! and [`UnicodeWidthStr::width_cjk`], +//! The `"cjk"` Cargo feature is enabled by default. +//! It enables `UnicodeWidthChar::width_cjk` +//! and `UnicodeWidthStr::width_cjk`, //! which perform an alternate width calculation //! more suited to CJK contexts. The flag also unseals the //! [`UnicodeWidthChar`] and [`UnicodeWidthStr`] traits. @@ -48,6 +46,64 @@ //! assert_eq!(teststr.width_cjk(), 6); //! ``` //! +//! # `"ascii-fast-path"` feature flag +//! +//! The `ascii-fast-path` feature is disabled by default. It accelerates ASCII +//! strings without carriage returns, but can slow Unicode strings. Widths stay +//! the same. Enable it for workloads dominated by ASCII. Cargo unifies features +//! across dependencies, so another dependency can enable this path for all callers. +//! The iterator functions `char_iter_width` and `char_iter_width_cjk` bypass the shortcut. +//! +//! # Unicode support and terminal policies +//! +//! [`UNICODE_VERSION`] identifies the Unicode data release. The width rules combine +//! [UAX #11] (East Asian Width), [UAX #44] (Unicode Character Database), [UTS #51] +//! (Unicode Emoji), and the core standard's combining, joining, and variation rules. +//! The crate also uses selected properties from [UAX #29] (Text Segmentation) and +//! [UAX #14] (Line Breaking). +//! +//! Canonically equivalent strings have equal widths, following [UAX #15] (Normalization). +//! Compatibility normalization can change width: for example, `"A"` has width 2 and +//! its NFKC form `"A"` has width 1. Width calculation does not normalize the input. +//! +//! The crate does not implement full grapheme segmentation, line breaking, +//! bidirectional ordering, vertical layout, or font shaping. +//! +//! These widths are estimates for horizontal terminal cells. The `char` methods return +//! `None` for control characters. String methods count each control as width 1, except +//! that `"\r\n"` has width 1 in total. To match layouts with tabs, line breaks, or terminal +//! escape sequences, callers must apply those protocols before measuring the text. +//! +//! [UAX #11]: https://www.unicode.org/reports/tr11/ +//! [UAX #14]: https://www.unicode.org/reports/tr14/ +//! [UAX #15]: https://www.unicode.org/reports/tr15/ +//! [UAX #29]: https://www.unicode.org/reports/tr29/ +//! [UAX #44]: https://www.unicode.org/reports/tr44/ +//! [UTS #51]: https://www.unicode.org/reports/tr51/ +//! +//! Variation selectors have width 0, including unsupported sequences and misplaced selectors. +//! Only the assigned sequences listed below change the width of a preceding character. +//! A selector separated from that character by another selector, mark, or control cannot +//! select a different presentation for it. +//! +//! Unicode 18 recommends a visible display for [misplaced variation selectors]. This crate +//! does not choose a replacement glyph. Callers that display a replacement must measure that text. +//! +//! [misplaced variation selectors]: https://www.unicode.org/versions/Unicode18.0.0/core-spec/chapter-3/#G52452 +//! +//! Fully-qualified emoji in the [RGI set] (recommended for general interchange) and the +//! emoji components in `emoji-test.txt` have width 2 in both width modes. +//! The crate does not insert omitted emoji presentation selectors. Minimally-qualified +//! and unqualified forms follow the rules below and can have different widths. +//! For example, `"*\u{20E3}"` has width 1 and `"*\u{FE0F}\u{20E3}"` has width 2. +//! +//! Recognition uses sequence structure and character properties, rather than an exact RGI +//! list or font support. Recognized non-RGI ZWJ sequences also have width 2. +//! A text presentation selector (`'\u{FE0E}'`) interrupts an emoji ZWJ sequence in both modes, +//! even when it does not narrow its base character in CJK mode. +//! +//! [RGI set]: https://www.unicode.org/reports/tr51/#def_rgi_set +//! //! # Rules for determining width //! //! This crate currently uses the following rules to determine the width of a @@ -204,7 +260,7 @@ pub trait UnicodeWidthChar: private::Sealed { /// Returns the character's displayed width in columns, or `None` if the /// character is a control character. /// - /// This function treats characters in the Ambiguous category according + /// This function generally treats characters in the Ambiguous category according /// to [Unicode Standard Annex #11](http://www.unicode.org/reports/tr11/) /// as 1 column wide. This is consistent with the recommendations for non-CJK /// contexts, or when the context cannot be reliably determined. @@ -213,7 +269,7 @@ pub trait UnicodeWidthChar: private::Sealed { /// Returns the character's displayed width in columns, or `None` if the /// character is a control character. /// - /// This function treats characters in the Ambiguous category according + /// This function generally treats characters in the Ambiguous category according /// to [Unicode Standard Annex #11](http://www.unicode.org/reports/tr11/) /// as 2 columns wide. This is consistent with the recommendations for /// CJK contexts. @@ -238,7 +294,7 @@ impl UnicodeWidthChar for char { pub trait UnicodeWidthStr: private::Sealed { /// Returns the string's displayed width in columns. /// - /// This function treats characters in the Ambiguous category according + /// This function generally treats characters in the Ambiguous category according /// to [Unicode Standard Annex #11](http://www.unicode.org/reports/tr11/) /// as 1 column wide. This is consistent with the recommendations for /// non-CJK contexts, or when the context cannot be reliably determined. @@ -246,9 +302,9 @@ pub trait UnicodeWidthStr: private::Sealed { /// Returns the string's displayed width in columns. /// - /// This function treats characters in the Ambiguous category according + /// This function generally treats characters in the Ambiguous category according /// to [Unicode Standard Annex #11](http://www.unicode.org/reports/tr11/) - /// as 2 column wide. This is consistent with the recommendations for + /// as 2 columns wide. This is consistent with the recommendations for /// CJK contexts. #[cfg(feature = "cjk")] fn width_cjk(&self) -> usize; @@ -257,12 +313,20 @@ pub trait UnicodeWidthStr: private::Sealed { impl UnicodeWidthStr for str { #[inline] fn width(&self) -> usize { + #[cfg(feature = "ascii-fast-path")] + if self.is_ascii() && !self.as_bytes().contains(&b'\r') { + return self.len(); + } lookup::str_width(self.chars()) } #[cfg(feature = "cjk")] #[inline] fn width_cjk(&self) -> usize { + #[cfg(feature = "ascii-fast-path")] + if self.is_ascii() && !self.as_bytes().contains(&b'\r') { + return self.len(); + } lookup::str_width_cjk(self.chars()) } } diff --git a/src/test.rs b/src/test.rs index 433323a..f914861 100644 --- a/src/test.rs +++ b/src/test.rs @@ -16,23 +16,53 @@ mod tables_test; use tables_test::*; -fn str_width_test(s: &str, init: WidthInfo) -> isize { +fn str_width_test(s: &str, init: WidthInfo) -> (isize, WidthInfo) { s.chars() .rfold((0, init), |(sum, next_info), c| -> (isize, WidthInfo) { let (add, info) = width_in_str(c, next_info); (sum.checked_add(isize::from(add)).unwrap(), info) }) - .0 } #[cfg(feature = "cjk")] -fn str_width_test_cjk(s: &str, init: WidthInfo) -> isize { +fn str_width_test_cjk(s: &str, init: WidthInfo) -> (isize, WidthInfo) { s.chars() .rfold((0, init), |(sum, next_info), c| -> (isize, WidthInfo) { let (add, info) = width_in_str_cjk(c, next_info); (sum.checked_add(isize::from(add)).unwrap(), info) }) - .0 +} + +#[test] +fn test_variation_selector_fallback() { + for (s, narrow, _cjk) in [ + ("\u{FE0F}#", 1, 1), + ("\r\u{FE0F}#", 2, 2), + ("#\u{0301}\u{FE0F}", 1, 1), + ("#\u{FE00}\u{FE0F}", 1, 1), + ("#\u{FE0F}\u{FE0F}", 2, 2), + ("\u{2018}\u{0301}\u{FE01}", 1, 2), + ("\u{180B}\u{180F}\u{E0100}\u{E01EF}", 0, 0), + ] { + assert_eq!(str_width(s.chars()), narrow, "{s:?}"); + #[cfg(feature = "cjk")] + assert_eq!(str_width_cjk(s.chars()), _cjk, "{s:?}"); + } +} + +#[test] +fn test_emoji_qualification_policy() { + for (s, narrow, _cjk) in [ + ("\u{1F469}\u{200D}\u{2696}", 3, 3), + ("\u{2764}\u{200D}\u{1F525}", 3, 3), + ("\u{1F469}\u{FE0E}\u{200D}\u{1F680}", 4, 4), + ("\u{2648}\u{FE0E}\u{200D}\u{1F680}", 3, 4), + ("\u{1F469}\u{200D}\u{2696}\u{FE0E}", 3, 3), + ] { + assert_eq!(str_width(s.chars()), narrow, "{s:?}"); + #[cfg(feature = "cjk")] + assert_eq!(str_width_cjk(s.chars()), _cjk, "{s:?}"); + } } #[test] diff --git a/tests/tests.rs b/tests/tests.rs index 0f83c65..6fe31ce 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -31,6 +31,21 @@ fn test_str() { assert_width!("\u{2081}\u{2082}\u{2083}\u{2084}", 4, 8); } +#[test] +fn test_ascii_strings() { + let mut bytes = [b'a'; 64]; + for a in 0..128 { + for b in 0..128 { + bytes[62] = a; + bytes[63] = b; + let s = std::str::from_utf8(&bytes).unwrap(); + let expected = 2 - usize::from([a, b] == *b"\r\n"); + assert_width!(&s[62..], expected, expected); + assert_width!(s, 62 + expected, 62 + expected); + } + } +} + #[test] fn test_emoji() { assert_width!("👩", 2, 2); // Woman @@ -180,6 +195,7 @@ fn test_solidus_overlay() { assert_width!("\u{06B8}\u{FE0F}\u{338}\u{0627}", 1, 1); assert_width!("=\u{338}\u{0627}", 2, 3); + assert_width!("=\u{1011B}\u{338}", 2, 2); } #[test] @@ -273,6 +289,8 @@ fn test_lisu_tones() { #[test] fn test_hebrew_alef_lamed() { + assert_width!("\u{05D0}\u{200D}\u{FB3C}", 1, 1); + assert_width!("\u{05D0}\u{200D}\u{05DC}\u{05BC}", 1, 1); assert_width!("\u{05D0}", 1, 1); assert_width!("\u{05DC}", 1, 1); assert_width!("\u{05D0}\u{05DC}", 2, 2); @@ -586,13 +604,12 @@ fn emoji_test_file() { let (cps, status) = line.split_once(';').unwrap(); let status = status.trim(); if status.starts_with("fully-qualified") || status.starts_with("component") { - let emoji: String = cps - .trim() - .split(' ') - .map(|s| char::try_from(u32::from_str_radix(s, 16).unwrap()).unwrap()) - .collect(); - dbg!(&emoji); - assert_width!(emoji, 2, 2); + let emoji = cps + .split_whitespace() + .map(|s| char::try_from(u32::from_str_radix(s, 16).unwrap()).unwrap()); + assert_eq!(char_iter_width(emoji.clone()), 2, "{cps}"); + #[cfg(feature = "cjk")] + assert_eq!(unicode_width::char_iter_width_cjk(emoji), 2, "{cps}"); } } } @@ -609,6 +626,7 @@ fn test_vs1_vs2_vs3() { assert_width!('\u{FE00}', Some(0), Some(0)); assert_width!('\u{FE01}', Some(0), Some(0)); assert_width!('\u{FE02}', Some(0), Some(0)); + let mut sequence = String::with_capacity(7); for c in '\0'..=char::MAX { if matches!(c, '\u{2018}' | '\u{2019}' | '\u{201C}' | '\u{201D}') { assert_width!(c, Some(1), Some(2)); @@ -619,39 +637,12 @@ fn test_vs1_vs2_vs3() { assert_width!(format!("{c}\u{FE02}"), 1, 1); assert_width!(format!("{c}\u{FE02}\u{FE01}"), 1, 1); } else { - assert_eq!( - format!("{c}\u{FE00}").width(), - c.width().unwrap_or(1), - "{c:?}" - ); - #[cfg(feature = "cjk")] - assert_eq!( - format!("{c}\u{FE00}").width_cjk(), - c.width_cjk().unwrap_or(1), - "{c:?}" - ); - assert_eq!( - format!("{c}\u{FE01}").width(), - c.width().unwrap_or(1), - "{c:?}" - ); - #[cfg(feature = "cjk")] - assert_eq!( - format!("{c}\u{FE01}").width_cjk(), - c.width_cjk().unwrap_or(1), - "{c:?}" - ); - assert_eq!( - format!("{c}\u{FE02}").width(), - c.width().unwrap_or(1), - "{c:?}" - ); - #[cfg(feature = "cjk")] - assert_eq!( - format!("{c}\u{FE02}").width_cjk(), - c.width_cjk().unwrap_or(1), - "{c:?}" - ); + for selector in ['\u{FE00}', '\u{FE01}', '\u{FE02}'] { + sequence.clear(); + sequence.push(c); + sequence.push(selector); + assert_width!(sequence, c.width().unwrap_or(1), c.width_cjk().unwrap_or(1)); + } } } }