Improve Unicode support and terminal-width policies

.github/workflows/rust.yml

@@ -23,7 +23,7 @@ jobs:
     - name: Run tests
       run: cargo test --verbose
     - name: Build docs
-      run: cargo doc
+      run: cargo doc && cargo doc --no-default-features
     - name: Check formatting
       run: cargo fmt --check
     - name: Check clippy
@@ -32,6 +32,10 @@ jobs:
       run: cargo build --verbose --no-default-features
     - name: Run tests (no default features)
       run: cargo test --verbose --no-default-features
+    - name: Run tests (ASCII fast path)
+      run: |
+        cargo test --verbose --features ascii-fast-path
+        cargo test --verbose --no-default-features --features ascii-fast-path
     - name: Check clippy (no default features)
       run: cargo clippy --verbose --lib --tests --no-default-features
 
@@ -42,6 +46,8 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: '3.12'
+    - name: Test generator
+      run: python3 -m unittest discover -s scripts
     - name: Regen
       run: rm tests/emoji-test.txt && cd scripts && python3 unicode.py
     - name: Diff
@@ -54,4 +60,4 @@ jobs:
       - uses: dtolnay/rust-toolchain@master
         with:
           toolchain: 1.66.0
-      - run: cargo check --lib --features core,std
+      - run: cargo check --lib --features core,std,ascii-fast-path

Cargo.toml

@@ -18,10 +18,7 @@ categories = [
 ]
 keywords = ["text", "width", "unicode"]
 readme = "README.md"
-description = """
-Determine displayed width of `char` and `str` types
-according to Unicode Standard Annex #11 rules.
-"""
+description = "Estimate terminal-column widths of Unicode characters and strings."
 edition = "2021"
 rust-version = "1.66"
 
@@ -42,6 +39,7 @@ core = { version = "1.0", package = "rustc-std-workspace-core", optional = true
 
 [features]
 cjk = []
+ascii-fast-path = []
 default = ["cjk"]
 rustc-dep-of-std = ['std', 'core']
 

README.md

@@ -4,8 +4,8 @@
 [![crates.io version](https://img.shields.io/crates/v/unicode-width)](https://crates.io/crates/unicode-width)
 [![Docs status](https://img.shields.io/docsrs/unicode-width)](https://docs.rs/unicode-width/)
 
-Determine displayed width of `char` and `str` types according to [Unicode Standard Annex #11][UAX11]
-and other portions of the Unicode standard.
+Determine terminal-column widths of `char` and `str` types using Unicode properties
+and the terminal rendering policies described below.
 
 This crate is `#![no_std]`.
 
@@ -24,6 +24,13 @@ fn main() {
 }
 ```
 
+## Unicode support and terminal policies
+
+The crate uses Unicode 18.0 data, including [UAX #11][UAX11] (East Asian Width),
+with terminal rendering policies. The [API documentation](https://docs.rs/unicode-width/latest/unicode_width/)
+lists the supported Unicode rules, variation-selector and emoji fallbacks, and
+the limits of terminal-width estimates.
+
 **NOTE:** The computed width values may not match the actual rendered column
 width. For example, many Brahmic scripts like Devanagari have complex rendering rules
 which this crate does not currently handle (and will never fully handle, because

benches/benches.rs

@@ -17,6 +17,40 @@ use test::Bencher;
 
 use unicode_width::{UnicodeWidthChar, UnicodeWidthStr};
 
+#[bench]
+fn ascii(b: &mut Bencher) {
+    let string = "The quick brown fox jumps over the lazy dog. ".repeat(128);
+    b.iter(|| test::black_box(UnicodeWidthStr::width(test::black_box(string.as_str()))));
+}
+
+#[bench]
+fn short_unicode(b: &mut Bencher) {
+    b.iter(|| test::black_box(UnicodeWidthStr::width(test::black_box("漢"))));
+}
+
+#[bench]
+fn mixed_unicode(b: &mut Bencher) {
+    let string = "a".repeat(61) + "é";
+    b.iter(|| test::black_box(UnicodeWidthStr::width(test::black_box(string.as_str()))));
+}
+
+#[bench]
+fn unicode(b: &mut Bencher) {
+    let string = "Unicode width: élève 日本語 한글 العربية 👩‍🔬\r\n".repeat(128);
+    b.iter(|| test::black_box(UnicodeWidthStr::width(test::black_box(string.as_str()))));
+}
+
+#[bench]
+fn unicode_scanner(b: &mut Bencher) {
+    let string = "élève Καλημέρα Привет 漢字カタカナ ".repeat(128);
+    b.iter(|| unicode_width::char_iter_width(test::black_box(string.as_str()).chars()));
+}
+
+#[bench]
+fn short_unicode_scanner(b: &mut Bencher) {
+    b.iter(|| unicode_width::char_iter_width(test::black_box("helloé").chars()));
+}
+
 #[bench]
 fn cargo(b: &mut Bencher) {
     let string = iter::repeat('a').take(4096).collect::<String>();
@@ -92,7 +126,7 @@ fn simple_width_match(c: char) -> Option<usize> {
 fn enwik8(b: &mut Bencher) {
     // To benchmark, download & unzip `enwik8` from https://data.deepai.org/enwik8.zip
     let data_path = "bench_data/enwik8";
-    let string = std::fs::read_to_string(data_path).unwrap_or_default();
+    let string = std::fs::read_to_string(data_path).expect("Download and unpack enwik8 first.");
     b.iter(|| test::black_box(UnicodeWidthStr::width(string.as_str())));
 }
 
@@ -101,14 +135,15 @@ fn jawiki(b: &mut Bencher) {
     // To benchmark, download & extract `jawiki-20220501-pages-articles-multistream-index.txt` from
     // https://dumps.wikimedia.org/jawiki/20220501/jawiki-20220501-pages-articles-multistream-index.txt.bz2
     let data_path = "bench_data/jawiki-20220501-pages-articles-multistream-index.txt";
-    let string = std::fs::read_to_string(data_path).unwrap_or_default();
+    let string =
+        std::fs::read_to_string(data_path).expect("Download and extract the jawiki index first.");
     b.iter(|| test::black_box(UnicodeWidthStr::width(string.as_str())));
 }
 
 #[bench]
 fn emoji(b: &mut Bencher) {
     // To benchmark, download emoji-style.txt from https://www.unicode.org/emoji/charts/emoji-style.txt
     let data_path = "bench_data/emoji-style.txt";
-    let string = std::fs::read_to_string(data_path).unwrap_or_default();
+    let string = std::fs::read_to_string(data_path).expect("Download emoji-style.txt first.");
     b.iter(|| test::black_box(UnicodeWidthStr::width(string.as_str())));
 }

scripts/test_unicode.py

@@ -0,0 +1,112 @@
+import io
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import unicode
+
+
+class UnicodeDataTests(unittest.TestCase):
+    def test_property_records(self):
+        records = " # comment\n0041;AI\n 0042..0043 ; AL#comment\n0044;AIX\n0045;AI"
+        with patch.object(unicode, "fetch_open", return_value=io.StringIO(records)):
+            found = []
+            unicode.load_property("LineBreak.txt", "AI|AL", found.append)
+        self.assertEqual(found, [0x41, 0x42, 0x43, 0x45])
+
+    def test_east_asian_width_records_and_defaults(self):
+        records = """# @missing: 0000..10FFFF; N
+003D;A
+0041;Na
+00A1;A#arbitrary comment
+00A2;H
+00A3;F
+00A4..00A5;W
+00A6;N
+2260..2261;N
+F900..FAFF;W
+20000..2FFFD;W
+30000..3FFFD;W
+# @missing: 0041..0043; W
+# @missing: 0000..10FFFF; N
+# @missing: 0042; W
+"""
+        with patch.object(
+            unicode, "fetch_open",
+            side_effect=lambda name: io.StringIO({
+                "EastAsianWidth.txt": records,
+                "UnicodeData.txt": "2260;NOT EQUAL TO;Sm;0;ON;003D 0338;\n2261;COMPATIBILITY FORM;Sm;0;ON;<compat> 003D 0338;\n",
+            }.get(name, "")),
+        ):
+            widths = unicode.load_east_asian_widths()
+        narrow, wide, ambiguous = unicode.EastAsianWidth
+        for cp, expected in [
+            (0, narrow), (0x41, narrow), (0x42, wide), (0x43, narrow),
+            (0xA1, ambiguous), (0xA2, narrow), (0xA3, wide), (0xA4, wide),
+            (0xA5, wide), (0xA6, narrow), (0xA7, narrow),
+            (0x2260, ambiguous), (0x2261, narrow),
+            (0xF8FF, narrow), (0xF900, wide), (0xFAFF, wide), (0xFB00, narrow),
+            (0x1FFFF, narrow), (0x20000, wide), (0x2FFFD, wide), (0x2FFFE, narrow),
+            (0x30000, wide), (0x3FFFD, wide), (0x3FFFE, narrow), (0x10FFFF, narrow),
+        ]:
+            with self.subTest(cp=hex(cp)):
+                self.assertEqual(widths[cp], expected)
+
+    def test_solidus_canonical_decompositions(self):
+        records = """0342;RECURSIVE;Mn;0;NSM;0341;
+0341;CANONICAL;Mn;0;NSM;0301 0301;
+0343;COMPATIBILITY;Mn;0;NSM;<compat> 0301;
+0344;EMPTY;Mn;0;NSM;;
+0345;MIXED;Mn;0;NSM;0301 0041;
+0346;NUMERAL;No;0;L;;;;300;N;;;;;
+"""
+        with patch.object(unicode, "fetch_open", side_effect=lambda name: io.StringIO(
+            records if name == "UnicodeData.txt" else "0300..0301;230\n"
+        )):
+            widths = [unicode.WidthState.NARROW] * 0x347
+            self.assertEqual(unicode.load_solidus_transparent([], widths), [(0x300, 0x301), (0x341, 0x342)])
+
+    def test_versioned_cache(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "UnicodeData.txt").write_text("stale data", encoding="utf-8")
+            expected = root / f"{unicode.UNICODE_VERSION}-UnicodeData.txt"
+
+            def download(url, path):
+                self.assertEqual(url, f"https://www.unicode.org/Public/{unicode.UNICODE_VERSION}/ucd/UnicodeData.txt")
+                self.assertEqual(Path(path), expected)
+                expected.write_text("current data", encoding="utf-8")
+
+            with patch.object(unicode.urllib.request, "urlretrieve", side_effect=download) as fetch:
+                for _ in range(2):
+                    with unicode.fetch_open("UnicodeData.txt", directory) as data:
+                        self.assertEqual(data.read(), "current data")
+                fetch.assert_called_once()
+
+    def test_version_headers(self):
+        version = unicode.UNICODE_VERSION
+        emoji_version = version.rsplit(".", 1)[0]
+        for filename, emoji, header in [
+            ("extracted/DerivedGeneralCategory.txt", False, f"# DerivedGeneralCategory-{version}.txt\n"),
+            ("ReadMe.txt", False, f"Final data files for version {version} of the\nUnicode Character Database.\n"),
+            ("emoji/emoji-data.txt", False, f"# emoji-data.txt\n# Version: {version}\n"),
+            ("emoji-test.txt", True, f"# emoji-test.txt\n# Version: {emoji_version}\n"),
+        ]:
+            with self.subTest(filename=filename), tempfile.TemporaryDirectory() as directory:
+                basename = Path(filename).name
+                cached = basename if emoji else f"{version}-{basename}"
+                path = Path(directory) / cached
+                path.write_text(header, encoding="utf-8")
+                with patch.object(unicode.urllib.request, "urlretrieve") as fetch:
+                    with unicode.fetch_open(filename, directory, emoji) as data:
+                        self.assertEqual(data.read(), header)
+                    fetch.assert_not_called()
+                for invalid in [header.replace(version, "0.0.0").replace(emoji_version, "0.0"), "no version\n"]:
+                    path.write_text(invalid, encoding="utf-8")
+                    with self.assertRaisesRegex(ValueError, "expected Unicode"):
+                        unicode.fetch_open(filename, directory, emoji)
+
+
+if __name__ == "__main__":
+    unittest.main()