From 6d0f5110479aad57bc10399e2bb7e877d4803fa2 Mon Sep 17 00:00:00 2001 From: Omar Aglan Date: Mon, 14 Sep 2026 12:30:49 +0300 Subject: [PATCH 1/4] feat(text): Add Unicode paragraph direction detection --- CMakeLists.txt | 2 + Core/Libraries/Source/WWVegas/CMakeLists.txt | 1 + Dependencies/UnicodeBidi/CMakeLists.txt | 18 ++ .../generate_supplementary_bidi.py | 54 ++++ .../UnicodeBidi/supplementarybidi.inl | 265 ++++++++++++++++++ Dependencies/UnicodeBidi/unicode-license.txt | 39 +++ Dependencies/UnicodeBidi/unicodebidi.cpp | 125 +++++++++ Dependencies/UnicodeBidi/unicodebidi.h | 32 +++ 8 files changed, 536 insertions(+) create mode 100644 Dependencies/UnicodeBidi/CMakeLists.txt create mode 100644 Dependencies/UnicodeBidi/generate_supplementary_bidi.py create mode 100644 Dependencies/UnicodeBidi/supplementarybidi.inl create mode 100644 Dependencies/UnicodeBidi/unicode-license.txt create mode 100644 Dependencies/UnicodeBidi/unicodebidi.cpp create mode 100644 Dependencies/UnicodeBidi/unicodebidi.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 39509160788..e5c605bc7e2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,9 +82,11 @@ add_subdirectory(Dependencies/Miles) if (WIN32) add_subdirectory(Dependencies/DbgHelp) add_subdirectory(Dependencies/Usp10) + add_subdirectory(Dependencies/UnicodeBidi) else() add_library(deps_dbghelp INTERFACE) add_library(deps_usp10 INTERFACE) + add_library(unicodebidi INTERFACE) endif() add_subdirectory(resources) diff --git a/Core/Libraries/Source/WWVegas/CMakeLists.txt b/Core/Libraries/Source/WWVegas/CMakeLists.txt index 9a630dce259..4933a33620a 100644 --- a/Core/Libraries/Source/WWVegas/CMakeLists.txt +++ b/Core/Libraries/Source/WWVegas/CMakeLists.txt @@ -12,6 +12,7 @@ target_link_libraries(core_wwcommon INTERFACE deps_miles deps_usp10 stlport + unicodebidi ) target_include_directories(core_wwcommon INTERFACE diff --git a/Dependencies/UnicodeBidi/CMakeLists.txt b/Dependencies/UnicodeBidi/CMakeLists.txt new file mode 100644 index 00000000000..6dc72533123 --- /dev/null +++ b/Dependencies/UnicodeBidi/CMakeLists.txt @@ -0,0 +1,18 @@ +set(UNICODEBIDI_SRC + unicodebidi.cpp + unicodebidi.h + supplementarybidi.inl +) + +add_library(unicodebidi STATIC) +set_target_properties(unicodebidi PROPERTIES OUTPUT_NAME unicodebidi) + +target_sources(unicodebidi PRIVATE ${UNICODEBIDI_SRC}) + +target_include_directories(unicodebidi PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} # include "unicodebidi.h" +) + +target_link_libraries(unicodebidi PRIVATE + deps_precompiled # Gives force included headers. Link early. +) diff --git a/Dependencies/UnicodeBidi/generate_supplementary_bidi.py b/Dependencies/UnicodeBidi/generate_supplementary_bidi.py new file mode 100644 index 00000000000..cb2c0a92271 --- /dev/null +++ b/Dependencies/UnicodeBidi/generate_supplementary_bidi.py @@ -0,0 +1,54 @@ +"""Generate the supplementary first-strong-direction table from Unicode 17.0.0. + +Input: https://www.unicode.org/Public/17.0.0/ucd/extracted/DerivedBidiClass.txt +Usage: python Dependencies/UnicodeBidi/generate_supplementary_bidi.py DerivedBidiClass.txt [--check] +""" + +import argparse +import hashlib +from pathlib import Path +import re + + +def generate(source): + if not source.startswith("# DerivedBidiClass-17.0.0.txt"): + raise ValueError("Expected Unicode 17.0.0 DerivedBidiClass.txt") + if hashlib.sha256(source.encode("utf-8")).hexdigest() != "4867b4b7f0731ed1bfcd34cc6251211ff1542541fce0734b6fbda139ee80b3a4": + raise ValueError("Unexpected Unicode data checksum") + classes = ["L"] * 0x110000 + aliases = {"Left_To_Right": "L", "Right_To_Left": "R", "Arabic_Letter": "AL"} + missing = re.findall(r"@missing: ([0-9A-F]+)\.\.([0-9A-F]+); (\w+)", source) + explicit = re.findall(r"^([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*(\w+)", source, re.M) + for lo, hi, value in missing + explicit: + lo, hi = int(lo, 16), int(hi or lo, 16) + classes[lo:hi + 1] = [aliases.get(value, value)] * (hi - lo + 1) + directions = ["LeftToRight" if c == "L" else "RightToLeft" if c in ("R", "AL") + else "Neutral" for c in classes] + lines = [ + "// Generated by generate_supplementary_bidi.py; do not edit.", + "// Unicode 17.0.0 DerivedBidiClass.txt, including its @missing defaults.", + "// Copyright 2025 Unicode, Inc. See unicode-license.txt.", + "// Omitted ranges have direction LeftToRight. Neutral means neither L, R nor AL.", + ] + start = 0x10000 + for end in range(start + 1, len(directions) + 1): + if end == len(directions) or directions[end] != directions[start]: + if directions[start] != "LeftToRight": + lines.append("\t{ 0x%05X, 0x%05X, %s }," % (start, end - 1, directions[start])) + start = end + return "\n".join(lines) + "\n" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + output = Path(__file__).resolve().parent / "supplementarybidi.inl" + generated = generate(args.source.read_text(encoding="utf-8")) + if args.check: + if output.read_text(encoding="utf-8") != generated: + raise SystemExit("Supplementary bidi table is out of date") + print("Unicode 17.0.0 supplementary bidi table matches") + else: + output.write_text(generated, encoding="utf-8") diff --git a/Dependencies/UnicodeBidi/supplementarybidi.inl b/Dependencies/UnicodeBidi/supplementarybidi.inl new file mode 100644 index 00000000000..9b2a9adada8 --- /dev/null +++ b/Dependencies/UnicodeBidi/supplementarybidi.inl @@ -0,0 +1,265 @@ +// Generated by generate_supplementary_bidi.py; do not edit. +// Unicode 17.0.0 DerivedBidiClass.txt, including its @missing defaults. +// Copyright 2025 Unicode, Inc. See unicode-license.txt. +// Omitted ranges have direction LeftToRight. Neutral means neither L, R nor AL. + { 0x10101, 0x10101, Neutral }, + { 0x10140, 0x1018C, Neutral }, + { 0x10190, 0x1019C, Neutral }, + { 0x101A0, 0x101A0, Neutral }, + { 0x101FD, 0x101FD, Neutral }, + { 0x102E0, 0x102FB, Neutral }, + { 0x10376, 0x1037A, Neutral }, + { 0x10800, 0x1091E, RightToLeft }, + { 0x1091F, 0x1091F, Neutral }, + { 0x10920, 0x10A00, RightToLeft }, + { 0x10A01, 0x10A03, Neutral }, + { 0x10A04, 0x10A04, RightToLeft }, + { 0x10A05, 0x10A06, Neutral }, + { 0x10A07, 0x10A0B, RightToLeft }, + { 0x10A0C, 0x10A0F, Neutral }, + { 0x10A10, 0x10A37, RightToLeft }, + { 0x10A38, 0x10A3A, Neutral }, + { 0x10A3B, 0x10A3E, RightToLeft }, + { 0x10A3F, 0x10A3F, Neutral }, + { 0x10A40, 0x10AE4, RightToLeft }, + { 0x10AE5, 0x10AE6, Neutral }, + { 0x10AE7, 0x10B38, RightToLeft }, + { 0x10B39, 0x10B3F, Neutral }, + { 0x10B40, 0x10D23, RightToLeft }, + { 0x10D24, 0x10D27, Neutral }, + { 0x10D28, 0x10D2F, RightToLeft }, + { 0x10D30, 0x10D39, Neutral }, + { 0x10D3A, 0x10D3F, RightToLeft }, + { 0x10D40, 0x10D49, Neutral }, + { 0x10D4A, 0x10D68, RightToLeft }, + { 0x10D69, 0x10D6E, Neutral }, + { 0x10D6F, 0x10E5F, RightToLeft }, + { 0x10E60, 0x10E7E, Neutral }, + { 0x10E7F, 0x10EAA, RightToLeft }, + { 0x10EAB, 0x10EAC, Neutral }, + { 0x10EAD, 0x10ECF, RightToLeft }, + { 0x10ED0, 0x10ED8, Neutral }, + { 0x10ED9, 0x10EF9, RightToLeft }, + { 0x10EFA, 0x10EFF, Neutral }, + { 0x10F00, 0x10F45, RightToLeft }, + { 0x10F46, 0x10F50, Neutral }, + { 0x10F51, 0x10F81, RightToLeft }, + { 0x10F82, 0x10F85, Neutral }, + { 0x10F86, 0x10FFF, RightToLeft }, + { 0x11001, 0x11001, Neutral }, + { 0x11038, 0x11046, Neutral }, + { 0x11052, 0x11065, Neutral }, + { 0x11070, 0x11070, Neutral }, + { 0x11073, 0x11074, Neutral }, + { 0x1107F, 0x11081, Neutral }, + { 0x110B3, 0x110B6, Neutral }, + { 0x110B9, 0x110BA, Neutral }, + { 0x110C2, 0x110C2, Neutral }, + { 0x11100, 0x11102, Neutral }, + { 0x11127, 0x1112B, Neutral }, + { 0x1112D, 0x11134, Neutral }, + { 0x11173, 0x11173, Neutral }, + { 0x11180, 0x11181, Neutral }, + { 0x111B6, 0x111BE, Neutral }, + { 0x111C9, 0x111CC, Neutral }, + { 0x111CF, 0x111CF, Neutral }, + { 0x1122F, 0x11231, Neutral }, + { 0x11234, 0x11234, Neutral }, + { 0x11236, 0x11237, Neutral }, + { 0x1123E, 0x1123E, Neutral }, + { 0x11241, 0x11241, Neutral }, + { 0x112DF, 0x112DF, Neutral }, + { 0x112E3, 0x112EA, Neutral }, + { 0x11300, 0x11301, Neutral }, + { 0x1133B, 0x1133C, Neutral }, + { 0x11340, 0x11340, Neutral }, + { 0x11366, 0x1136C, Neutral }, + { 0x11370, 0x11374, Neutral }, + { 0x113BB, 0x113C0, Neutral }, + { 0x113CE, 0x113CE, Neutral }, + { 0x113D0, 0x113D0, Neutral }, + { 0x113D2, 0x113D2, Neutral }, + { 0x113E1, 0x113E2, Neutral }, + { 0x11438, 0x1143F, Neutral }, + { 0x11442, 0x11444, Neutral }, + { 0x11446, 0x11446, Neutral }, + { 0x1145E, 0x1145E, Neutral }, + { 0x114B3, 0x114B8, Neutral }, + { 0x114BA, 0x114BA, Neutral }, + { 0x114BF, 0x114C0, Neutral }, + { 0x114C2, 0x114C3, Neutral }, + { 0x115B2, 0x115B5, Neutral }, + { 0x115BC, 0x115BD, Neutral }, + { 0x115BF, 0x115C0, Neutral }, + { 0x115DC, 0x115DD, Neutral }, + { 0x11633, 0x1163A, Neutral }, + { 0x1163D, 0x1163D, Neutral }, + { 0x1163F, 0x11640, Neutral }, + { 0x11660, 0x1166C, Neutral }, + { 0x116AB, 0x116AB, Neutral }, + { 0x116AD, 0x116AD, Neutral }, + { 0x116B0, 0x116B5, Neutral }, + { 0x116B7, 0x116B7, Neutral }, + { 0x1171D, 0x1171D, Neutral }, + { 0x1171F, 0x1171F, Neutral }, + { 0x11722, 0x11725, Neutral }, + { 0x11727, 0x1172B, Neutral }, + { 0x1182F, 0x11837, Neutral }, + { 0x11839, 0x1183A, Neutral }, + { 0x1193B, 0x1193C, Neutral }, + { 0x1193E, 0x1193E, Neutral }, + { 0x11943, 0x11943, Neutral }, + { 0x119D4, 0x119D7, Neutral }, + { 0x119DA, 0x119DB, Neutral }, + { 0x119E0, 0x119E0, Neutral }, + { 0x11A01, 0x11A06, Neutral }, + { 0x11A09, 0x11A0A, Neutral }, + { 0x11A33, 0x11A38, Neutral }, + { 0x11A3B, 0x11A3E, Neutral }, + { 0x11A47, 0x11A47, Neutral }, + { 0x11A51, 0x11A56, Neutral }, + { 0x11A59, 0x11A5B, Neutral }, + { 0x11A8A, 0x11A96, Neutral }, + { 0x11A98, 0x11A99, Neutral }, + { 0x11B60, 0x11B60, Neutral }, + { 0x11B62, 0x11B64, Neutral }, + { 0x11B66, 0x11B66, Neutral }, + { 0x11C30, 0x11C36, Neutral }, + { 0x11C38, 0x11C3D, Neutral }, + { 0x11C92, 0x11CA7, Neutral }, + { 0x11CAA, 0x11CB0, Neutral }, + { 0x11CB2, 0x11CB3, Neutral }, + { 0x11CB5, 0x11CB6, Neutral }, + { 0x11D31, 0x11D36, Neutral }, + { 0x11D3A, 0x11D3A, Neutral }, + { 0x11D3C, 0x11D3D, Neutral }, + { 0x11D3F, 0x11D45, Neutral }, + { 0x11D47, 0x11D47, Neutral }, + { 0x11D90, 0x11D91, Neutral }, + { 0x11D95, 0x11D95, Neutral }, + { 0x11D97, 0x11D97, Neutral }, + { 0x11EF3, 0x11EF4, Neutral }, + { 0x11F00, 0x11F01, Neutral }, + { 0x11F36, 0x11F3A, Neutral }, + { 0x11F40, 0x11F40, Neutral }, + { 0x11F42, 0x11F42, Neutral }, + { 0x11F5A, 0x11F5A, Neutral }, + { 0x11FD5, 0x11FF1, Neutral }, + { 0x13440, 0x13440, Neutral }, + { 0x13447, 0x13455, Neutral }, + { 0x1611E, 0x16129, Neutral }, + { 0x1612D, 0x1612F, Neutral }, + { 0x16AF0, 0x16AF4, Neutral }, + { 0x16B30, 0x16B36, Neutral }, + { 0x16F4F, 0x16F4F, Neutral }, + { 0x16F8F, 0x16F92, Neutral }, + { 0x16FE2, 0x16FE2, Neutral }, + { 0x16FE4, 0x16FE4, Neutral }, + { 0x1BC9D, 0x1BC9E, Neutral }, + { 0x1BCA0, 0x1BCA3, Neutral }, + { 0x1CC00, 0x1CCD5, Neutral }, + { 0x1CCF0, 0x1CCFC, Neutral }, + { 0x1CD00, 0x1CEB3, Neutral }, + { 0x1CEBA, 0x1CED0, Neutral }, + { 0x1CEE0, 0x1CEF0, Neutral }, + { 0x1CF00, 0x1CF2D, Neutral }, + { 0x1CF30, 0x1CF46, Neutral }, + { 0x1D167, 0x1D169, Neutral }, + { 0x1D173, 0x1D182, Neutral }, + { 0x1D185, 0x1D18B, Neutral }, + { 0x1D1AA, 0x1D1AD, Neutral }, + { 0x1D1E9, 0x1D1EA, Neutral }, + { 0x1D200, 0x1D245, Neutral }, + { 0x1D300, 0x1D356, Neutral }, + { 0x1D6C1, 0x1D6C1, Neutral }, + { 0x1D6DB, 0x1D6DB, Neutral }, + { 0x1D6FB, 0x1D6FB, Neutral }, + { 0x1D715, 0x1D715, Neutral }, + { 0x1D735, 0x1D735, Neutral }, + { 0x1D74F, 0x1D74F, Neutral }, + { 0x1D76F, 0x1D76F, Neutral }, + { 0x1D789, 0x1D789, Neutral }, + { 0x1D7A9, 0x1D7A9, Neutral }, + { 0x1D7C3, 0x1D7C3, Neutral }, + { 0x1D7CE, 0x1D7FF, Neutral }, + { 0x1DA00, 0x1DA36, Neutral }, + { 0x1DA3B, 0x1DA6C, Neutral }, + { 0x1DA75, 0x1DA75, Neutral }, + { 0x1DA84, 0x1DA84, Neutral }, + { 0x1DA9B, 0x1DA9F, Neutral }, + { 0x1DAA1, 0x1DAAF, Neutral }, + { 0x1E000, 0x1E006, Neutral }, + { 0x1E008, 0x1E018, Neutral }, + { 0x1E01B, 0x1E021, Neutral }, + { 0x1E023, 0x1E024, Neutral }, + { 0x1E026, 0x1E02A, Neutral }, + { 0x1E08F, 0x1E08F, Neutral }, + { 0x1E130, 0x1E136, Neutral }, + { 0x1E2AE, 0x1E2AE, Neutral }, + { 0x1E2EC, 0x1E2EF, Neutral }, + { 0x1E2FF, 0x1E2FF, Neutral }, + { 0x1E4EC, 0x1E4EF, Neutral }, + { 0x1E5EE, 0x1E5EF, Neutral }, + { 0x1E6E3, 0x1E6E3, Neutral }, + { 0x1E6E6, 0x1E6E6, Neutral }, + { 0x1E6EE, 0x1E6EF, Neutral }, + { 0x1E6F5, 0x1E6F5, Neutral }, + { 0x1E800, 0x1E8CF, RightToLeft }, + { 0x1E8D0, 0x1E8D6, Neutral }, + { 0x1E8D7, 0x1E943, RightToLeft }, + { 0x1E944, 0x1E94A, Neutral }, + { 0x1E94B, 0x1EEEF, RightToLeft }, + { 0x1EEF0, 0x1EEF1, Neutral }, + { 0x1EEF2, 0x1EFFF, RightToLeft }, + { 0x1F000, 0x1F02B, Neutral }, + { 0x1F030, 0x1F093, Neutral }, + { 0x1F0A0, 0x1F0AE, Neutral }, + { 0x1F0B1, 0x1F0BF, Neutral }, + { 0x1F0C1, 0x1F0CF, Neutral }, + { 0x1F0D1, 0x1F0F5, Neutral }, + { 0x1F100, 0x1F10F, Neutral }, + { 0x1F12F, 0x1F12F, Neutral }, + { 0x1F16A, 0x1F16F, Neutral }, + { 0x1F1AD, 0x1F1AD, Neutral }, + { 0x1F260, 0x1F265, Neutral }, + { 0x1F300, 0x1F6D8, Neutral }, + { 0x1F6DC, 0x1F6EC, Neutral }, + { 0x1F6F0, 0x1F6FC, Neutral }, + { 0x1F700, 0x1F7D9, Neutral }, + { 0x1F7E0, 0x1F7EB, Neutral }, + { 0x1F7F0, 0x1F7F0, Neutral }, + { 0x1F800, 0x1F80B, Neutral }, + { 0x1F810, 0x1F847, Neutral }, + { 0x1F850, 0x1F859, Neutral }, + { 0x1F860, 0x1F887, Neutral }, + { 0x1F890, 0x1F8AD, Neutral }, + { 0x1F8B0, 0x1F8BB, Neutral }, + { 0x1F8C0, 0x1F8C1, Neutral }, + { 0x1F8D0, 0x1F8D8, Neutral }, + { 0x1F900, 0x1FA57, Neutral }, + { 0x1FA60, 0x1FA6D, Neutral }, + { 0x1FA70, 0x1FA7C, Neutral }, + { 0x1FA80, 0x1FA8A, Neutral }, + { 0x1FA8E, 0x1FAC6, Neutral }, + { 0x1FAC8, 0x1FAC8, Neutral }, + { 0x1FACD, 0x1FADC, Neutral }, + { 0x1FADF, 0x1FAEA, Neutral }, + { 0x1FAEF, 0x1FAF8, Neutral }, + { 0x1FB00, 0x1FB92, Neutral }, + { 0x1FB94, 0x1FBFA, Neutral }, + { 0x1FFFE, 0x1FFFF, Neutral }, + { 0x2FFFE, 0x2FFFF, Neutral }, + { 0x3FFFE, 0x3FFFF, Neutral }, + { 0x4FFFE, 0x4FFFF, Neutral }, + { 0x5FFFE, 0x5FFFF, Neutral }, + { 0x6FFFE, 0x6FFFF, Neutral }, + { 0x7FFFE, 0x7FFFF, Neutral }, + { 0x8FFFE, 0x8FFFF, Neutral }, + { 0x9FFFE, 0x9FFFF, Neutral }, + { 0xAFFFE, 0xAFFFF, Neutral }, + { 0xBFFFE, 0xBFFFF, Neutral }, + { 0xCFFFE, 0xCFFFF, Neutral }, + { 0xDFFFE, 0xE0FFF, Neutral }, + { 0xEFFFE, 0xEFFFF, Neutral }, + { 0xFFFFE, 0xFFFFF, Neutral }, + { 0x10FFFE, 0x10FFFF, Neutral }, diff --git a/Dependencies/UnicodeBidi/unicode-license.txt b/Dependencies/UnicodeBidi/unicode-license.txt new file mode 100644 index 00000000000..56da5891280 --- /dev/null +++ b/Dependencies/UnicodeBidi/unicode-license.txt @@ -0,0 +1,39 @@ +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2026 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR +SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT +DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. diff --git a/Dependencies/UnicodeBidi/unicodebidi.cpp b/Dependencies/UnicodeBidi/unicodebidi.cpp new file mode 100644 index 00000000000..6906db57d8a --- /dev/null +++ b/Dependencies/UnicodeBidi/unicodebidi.cpp @@ -0,0 +1,125 @@ +/* +** Command & Conquer Generals Zero Hour(tm) +** Copyright 2026 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +#include "unicodebidi.h" + +namespace +{ + +enum Direction +{ + Neutral, + LeftToRight, + RightToLeft +}; + +struct DirectionRange +{ + unsigned First; + unsigned Last; + Direction Value; +}; + +static const DirectionRange SupplementaryDirectionRanges[] = { +#include "supplementarybidi.inl" +}; + +static Direction Get_Supplementary_Direction(unsigned codepoint) +{ + unsigned first = 0; + unsigned last = sizeof(SupplementaryDirectionRanges) / sizeof(SupplementaryDirectionRanges[0]); + + while (first < last) { + const unsigned middle = first + (last - first) / 2; + if (codepoint < SupplementaryDirectionRanges[middle].First) { + last = middle; + } else if (codepoint > SupplementaryDirectionRanges[middle].Last) { + first = middle + 1; + } else { + return SupplementaryDirectionRanges[middle].Value; + } + } + + return LeftToRight; +} + +static bool Is_Paragraph_Separator(WCHAR ch) +{ + return ch == L'\n' || ch == L'\r' || (ch >= 0x001C && ch <= 0x001E) || + ch == 0x0085 || ch == 0x2029; +} + +} + +namespace UnicodeBidi +{ + +WORD Get_Paragraph_Base_Level(const WCHAR *text, int length) +{ + int isolate_depth = 0; + + for (int index = 0; index < length; ++index) { + const WCHAR ch = text[index]; + + // First-strong detection applies to one bidi paragraph only. + if (Is_Paragraph_Separator(ch)) { + break; + } + + if (ch >= 0x2066 && ch <= 0x2068) { + ++isolate_depth; + continue; + } + if (ch == 0x2069) { + if (isolate_depth > 0) { + --isolate_depth; + } + continue; + } + + // UAX #9 P2 excludes directional-isolate contents from the paragraph direction decision. + if (isolate_depth > 0) { + continue; + } + + Direction direction = Neutral; + if (ch >= 0xD800 && ch <= 0xDBFF) { + if (index + 1 < length && text[index + 1] >= 0xDC00 && text[index + 1] <= 0xDFFF) { + const unsigned codepoint = 0x10000 + ((ch - 0xD800) << 10) + (text[++index] - 0xDC00); + direction = Get_Supplementary_Direction(codepoint); + } + } else if (ch < 0xDC00 || ch > 0xDFFF) { + WORD type = C2_NOTAPPLICABLE; + if (::GetStringTypeW(CT_CTYPE2, &ch, 1, &type)) { + if (type == C2_LEFTTORIGHT) { + direction = LeftToRight; + } else if (type == C2_RIGHTTOLEFT) { + direction = RightToLeft; + } + } + } + + if (direction != Neutral) { + return direction == RightToLeft ? 1 : 0; + } + } + + return 0; +} + +} diff --git a/Dependencies/UnicodeBidi/unicodebidi.h b/Dependencies/UnicodeBidi/unicodebidi.h new file mode 100644 index 00000000000..02239aa0282 --- /dev/null +++ b/Dependencies/UnicodeBidi/unicodebidi.h @@ -0,0 +1,32 @@ +/* +** Command & Conquer Generals Zero Hour(tm) +** Copyright 2026 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +#pragma once + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include + +namespace UnicodeBidi +{ + +// Returns the UAX #9 paragraph base embedding level: 0 for LTR, 1 for RTL. +WORD Get_Paragraph_Base_Level(const WCHAR *text, int length); + +} From 93c566a4d4d6a38711ca54d1e45761686e2ff494 Mon Sep 17 00:00:00 2001 From: Omar Aglan Date: Sat, 3 Oct 2026 11:20:35 +0300 Subject: [PATCH 2/4] fix(text): Make paragraph direction detection portable --- CMakeLists.txt | 3 +- Dependencies/UnicodeBidi/CMakeLists.txt | 2 +- Dependencies/UnicodeBidi/README.md | 32 + Dependencies/UnicodeBidi/bididirection.inl | 623 ++++++++++++++++++ ...supplementary_bidi.py => generate_bidi.py} | 14 +- .../UnicodeBidi/supplementarybidi.inl | 265 -------- Dependencies/UnicodeBidi/unicodebidi.cpp | 49 +- Dependencies/UnicodeBidi/unicodebidi.h | 10 +- 8 files changed, 690 insertions(+), 308 deletions(-) create mode 100644 Dependencies/UnicodeBidi/README.md create mode 100644 Dependencies/UnicodeBidi/bididirection.inl rename Dependencies/UnicodeBidi/{generate_supplementary_bidi.py => generate_bidi.py} (81%) delete mode 100644 Dependencies/UnicodeBidi/supplementarybidi.inl diff --git a/CMakeLists.txt b/CMakeLists.txt index e5c605bc7e2..15400f0b027 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,14 +79,13 @@ add_subdirectory(Dependencies/Precompiled) add_subdirectory(Dependencies/Utility) add_subdirectory(Dependencies/Bink) add_subdirectory(Dependencies/Miles) +add_subdirectory(Dependencies/UnicodeBidi) if (WIN32) add_subdirectory(Dependencies/DbgHelp) add_subdirectory(Dependencies/Usp10) - add_subdirectory(Dependencies/UnicodeBidi) else() add_library(deps_dbghelp INTERFACE) add_library(deps_usp10 INTERFACE) - add_library(unicodebidi INTERFACE) endif() add_subdirectory(resources) diff --git a/Dependencies/UnicodeBidi/CMakeLists.txt b/Dependencies/UnicodeBidi/CMakeLists.txt index 6dc72533123..48c4a2195d7 100644 --- a/Dependencies/UnicodeBidi/CMakeLists.txt +++ b/Dependencies/UnicodeBidi/CMakeLists.txt @@ -1,7 +1,7 @@ set(UNICODEBIDI_SRC unicodebidi.cpp unicodebidi.h - supplementarybidi.inl + bididirection.inl ) add_library(unicodebidi STATIC) diff --git a/Dependencies/UnicodeBidi/README.md b/Dependencies/UnicodeBidi/README.md new file mode 100644 index 00000000000..3d57f8fdf53 --- /dev/null +++ b/Dependencies/UnicodeBidi/README.md @@ -0,0 +1,32 @@ +# Unicode paragraph direction + +`Get_Paragraph_Base_Level` provides the initial paragraph level for complex text +layout, such as Uniscribe itemization. It implements the first-strong +decision in [UAX #9 P2/P3](https://www.unicode.org/reports/tr9/tr9-51.html#P2), +with scanning bounded to the first paragraph. Shaping and visual reordering are +left to the renderer. + +The scan skips nested directional-isolate contents, decodes UTF-16 surrogate +pairs, and ignores lone surrogates. It also accepts direct supplementary scalar +values when `wchar_t` is 32 bits. The first L character selects level 0; the first +R or AL character selects level 1. A paragraph without a strong character +defaults to level 0. + +Both BMP and supplementary classification use the same Unicode 17.0.0 data, +independent of the operating system. The compact table merges R and AL and +treats every other non-L class as neutral for this decision. Omitted ranges +default to L, including the source's `@missing` defaults. + +## Data and license + +Download [DerivedBidiClass.txt](https://www.unicode.org/Public/17.0.0/ucd/extracted/DerivedBidiClass.txt) +and regenerate or check the table from the repository root: + +``` +python Dependencies/UnicodeBidi/generate_bidi.py DerivedBidiClass.txt +python Dependencies/UnicodeBidi/generate_bidi.py DerivedBidiClass.txt --check +``` + +The generator validates the source version and SHA-256 before extracting ranges. +The generated data retains Unicode's copyright notice; `unicode-license.txt` +contains the full Unicode License v3. diff --git a/Dependencies/UnicodeBidi/bididirection.inl b/Dependencies/UnicodeBidi/bididirection.inl new file mode 100644 index 00000000000..a847556734f --- /dev/null +++ b/Dependencies/UnicodeBidi/bididirection.inl @@ -0,0 +1,623 @@ +// Generated by generate_bidi.py; do not edit. +// Unicode 17.0.0 DerivedBidiClass.txt, including its @missing defaults. +// Copyright 2025 Unicode, Inc. See unicode-license.txt. +// Omitted ranges have direction LeftToRight. Neutral means neither L, R nor AL. + { 0x00000, 0x00040, Neutral }, + { 0x0005B, 0x00060, Neutral }, + { 0x0007B, 0x000A9, Neutral }, + { 0x000AB, 0x000B4, Neutral }, + { 0x000B6, 0x000B9, Neutral }, + { 0x000BB, 0x000BF, Neutral }, + { 0x000D7, 0x000D7, Neutral }, + { 0x000F7, 0x000F7, Neutral }, + { 0x002B9, 0x002BA, Neutral }, + { 0x002C2, 0x002CF, Neutral }, + { 0x002D2, 0x002DF, Neutral }, + { 0x002E5, 0x002ED, Neutral }, + { 0x002EF, 0x0036F, Neutral }, + { 0x00374, 0x00375, Neutral }, + { 0x0037E, 0x0037E, Neutral }, + { 0x00384, 0x00385, Neutral }, + { 0x00387, 0x00387, Neutral }, + { 0x003F6, 0x003F6, Neutral }, + { 0x00483, 0x00489, Neutral }, + { 0x0058A, 0x0058A, Neutral }, + { 0x0058D, 0x0058F, Neutral }, + { 0x00590, 0x00590, RightToLeft }, + { 0x00591, 0x005BD, Neutral }, + { 0x005BE, 0x005BE, RightToLeft }, + { 0x005BF, 0x005BF, Neutral }, + { 0x005C0, 0x005C0, RightToLeft }, + { 0x005C1, 0x005C2, Neutral }, + { 0x005C3, 0x005C3, RightToLeft }, + { 0x005C4, 0x005C5, Neutral }, + { 0x005C6, 0x005C6, RightToLeft }, + { 0x005C7, 0x005C7, Neutral }, + { 0x005C8, 0x005FF, RightToLeft }, + { 0x00600, 0x00607, Neutral }, + { 0x00608, 0x00608, RightToLeft }, + { 0x00609, 0x0060A, Neutral }, + { 0x0060B, 0x0060B, RightToLeft }, + { 0x0060C, 0x0060C, Neutral }, + { 0x0060D, 0x0060D, RightToLeft }, + { 0x0060E, 0x0061A, Neutral }, + { 0x0061B, 0x0064A, RightToLeft }, + { 0x0064B, 0x0066C, Neutral }, + { 0x0066D, 0x0066F, RightToLeft }, + { 0x00670, 0x00670, Neutral }, + { 0x00671, 0x006D5, RightToLeft }, + { 0x006D6, 0x006E4, Neutral }, + { 0x006E5, 0x006E6, RightToLeft }, + { 0x006E7, 0x006ED, Neutral }, + { 0x006EE, 0x006EF, RightToLeft }, + { 0x006F0, 0x006F9, Neutral }, + { 0x006FA, 0x00710, RightToLeft }, + { 0x00711, 0x00711, Neutral }, + { 0x00712, 0x0072F, RightToLeft }, + { 0x00730, 0x0074A, Neutral }, + { 0x0074B, 0x007A5, RightToLeft }, + { 0x007A6, 0x007B0, Neutral }, + { 0x007B1, 0x007EA, RightToLeft }, + { 0x007EB, 0x007F3, Neutral }, + { 0x007F4, 0x007F5, RightToLeft }, + { 0x007F6, 0x007F9, Neutral }, + { 0x007FA, 0x007FC, RightToLeft }, + { 0x007FD, 0x007FD, Neutral }, + { 0x007FE, 0x00815, RightToLeft }, + { 0x00816, 0x00819, Neutral }, + { 0x0081A, 0x0081A, RightToLeft }, + { 0x0081B, 0x00823, Neutral }, + { 0x00824, 0x00824, RightToLeft }, + { 0x00825, 0x00827, Neutral }, + { 0x00828, 0x00828, RightToLeft }, + { 0x00829, 0x0082D, Neutral }, + { 0x0082E, 0x00858, RightToLeft }, + { 0x00859, 0x0085B, Neutral }, + { 0x0085C, 0x0088F, RightToLeft }, + { 0x00890, 0x00891, Neutral }, + { 0x00892, 0x00896, RightToLeft }, + { 0x00897, 0x0089F, Neutral }, + { 0x008A0, 0x008C9, RightToLeft }, + { 0x008CA, 0x00902, Neutral }, + { 0x0093A, 0x0093A, Neutral }, + { 0x0093C, 0x0093C, Neutral }, + { 0x00941, 0x00948, Neutral }, + { 0x0094D, 0x0094D, Neutral }, + { 0x00951, 0x00957, Neutral }, + { 0x00962, 0x00963, Neutral }, + { 0x00981, 0x00981, Neutral }, + { 0x009BC, 0x009BC, Neutral }, + { 0x009C1, 0x009C4, Neutral }, + { 0x009CD, 0x009CD, Neutral }, + { 0x009E2, 0x009E3, Neutral }, + { 0x009F2, 0x009F3, Neutral }, + { 0x009FB, 0x009FB, Neutral }, + { 0x009FE, 0x009FE, Neutral }, + { 0x00A01, 0x00A02, Neutral }, + { 0x00A3C, 0x00A3C, Neutral }, + { 0x00A41, 0x00A42, Neutral }, + { 0x00A47, 0x00A48, Neutral }, + { 0x00A4B, 0x00A4D, Neutral }, + { 0x00A51, 0x00A51, Neutral }, + { 0x00A70, 0x00A71, Neutral }, + { 0x00A75, 0x00A75, Neutral }, + { 0x00A81, 0x00A82, Neutral }, + { 0x00ABC, 0x00ABC, Neutral }, + { 0x00AC1, 0x00AC5, Neutral }, + { 0x00AC7, 0x00AC8, Neutral }, + { 0x00ACD, 0x00ACD, Neutral }, + { 0x00AE2, 0x00AE3, Neutral }, + { 0x00AF1, 0x00AF1, Neutral }, + { 0x00AFA, 0x00AFF, Neutral }, + { 0x00B01, 0x00B01, Neutral }, + { 0x00B3C, 0x00B3C, Neutral }, + { 0x00B3F, 0x00B3F, Neutral }, + { 0x00B41, 0x00B44, Neutral }, + { 0x00B4D, 0x00B4D, Neutral }, + { 0x00B55, 0x00B56, Neutral }, + { 0x00B62, 0x00B63, Neutral }, + { 0x00B82, 0x00B82, Neutral }, + { 0x00BC0, 0x00BC0, Neutral }, + { 0x00BCD, 0x00BCD, Neutral }, + { 0x00BF3, 0x00BFA, Neutral }, + { 0x00C00, 0x00C00, Neutral }, + { 0x00C04, 0x00C04, Neutral }, + { 0x00C3C, 0x00C3C, Neutral }, + { 0x00C3E, 0x00C40, Neutral }, + { 0x00C46, 0x00C48, Neutral }, + { 0x00C4A, 0x00C4D, Neutral }, + { 0x00C55, 0x00C56, Neutral }, + { 0x00C62, 0x00C63, Neutral }, + { 0x00C78, 0x00C7E, Neutral }, + { 0x00C81, 0x00C81, Neutral }, + { 0x00CBC, 0x00CBC, Neutral }, + { 0x00CCC, 0x00CCD, Neutral }, + { 0x00CE2, 0x00CE3, Neutral }, + { 0x00D00, 0x00D01, Neutral }, + { 0x00D3B, 0x00D3C, Neutral }, + { 0x00D41, 0x00D44, Neutral }, + { 0x00D4D, 0x00D4D, Neutral }, + { 0x00D62, 0x00D63, Neutral }, + { 0x00D81, 0x00D81, Neutral }, + { 0x00DCA, 0x00DCA, Neutral }, + { 0x00DD2, 0x00DD4, Neutral }, + { 0x00DD6, 0x00DD6, Neutral }, + { 0x00E31, 0x00E31, Neutral }, + { 0x00E34, 0x00E3A, Neutral }, + { 0x00E3F, 0x00E3F, Neutral }, + { 0x00E47, 0x00E4E, Neutral }, + { 0x00EB1, 0x00EB1, Neutral }, + { 0x00EB4, 0x00EBC, Neutral }, + { 0x00EC8, 0x00ECE, Neutral }, + { 0x00F18, 0x00F19, Neutral }, + { 0x00F35, 0x00F35, Neutral }, + { 0x00F37, 0x00F37, Neutral }, + { 0x00F39, 0x00F3D, Neutral }, + { 0x00F71, 0x00F7E, Neutral }, + { 0x00F80, 0x00F84, Neutral }, + { 0x00F86, 0x00F87, Neutral }, + { 0x00F8D, 0x00F97, Neutral }, + { 0x00F99, 0x00FBC, Neutral }, + { 0x00FC6, 0x00FC6, Neutral }, + { 0x0102D, 0x01030, Neutral }, + { 0x01032, 0x01037, Neutral }, + { 0x01039, 0x0103A, Neutral }, + { 0x0103D, 0x0103E, Neutral }, + { 0x01058, 0x01059, Neutral }, + { 0x0105E, 0x01060, Neutral }, + { 0x01071, 0x01074, Neutral }, + { 0x01082, 0x01082, Neutral }, + { 0x01085, 0x01086, Neutral }, + { 0x0108D, 0x0108D, Neutral }, + { 0x0109D, 0x0109D, Neutral }, + { 0x0135D, 0x0135F, Neutral }, + { 0x01390, 0x01399, Neutral }, + { 0x01400, 0x01400, Neutral }, + { 0x01680, 0x01680, Neutral }, + { 0x0169B, 0x0169C, Neutral }, + { 0x01712, 0x01714, Neutral }, + { 0x01732, 0x01733, Neutral }, + { 0x01752, 0x01753, Neutral }, + { 0x01772, 0x01773, Neutral }, + { 0x017B4, 0x017B5, Neutral }, + { 0x017B7, 0x017BD, Neutral }, + { 0x017C6, 0x017C6, Neutral }, + { 0x017C9, 0x017D3, Neutral }, + { 0x017DB, 0x017DB, Neutral }, + { 0x017DD, 0x017DD, Neutral }, + { 0x017F0, 0x017F9, Neutral }, + { 0x01800, 0x0180F, Neutral }, + { 0x01885, 0x01886, Neutral }, + { 0x018A9, 0x018A9, Neutral }, + { 0x01920, 0x01922, Neutral }, + { 0x01927, 0x01928, Neutral }, + { 0x01932, 0x01932, Neutral }, + { 0x01939, 0x0193B, Neutral }, + { 0x01940, 0x01940, Neutral }, + { 0x01944, 0x01945, Neutral }, + { 0x019DE, 0x019FF, Neutral }, + { 0x01A17, 0x01A18, Neutral }, + { 0x01A1B, 0x01A1B, Neutral }, + { 0x01A56, 0x01A56, Neutral }, + { 0x01A58, 0x01A5E, Neutral }, + { 0x01A60, 0x01A60, Neutral }, + { 0x01A62, 0x01A62, Neutral }, + { 0x01A65, 0x01A6C, Neutral }, + { 0x01A73, 0x01A7C, Neutral }, + { 0x01A7F, 0x01A7F, Neutral }, + { 0x01AB0, 0x01ADD, Neutral }, + { 0x01AE0, 0x01AEB, Neutral }, + { 0x01B00, 0x01B03, Neutral }, + { 0x01B34, 0x01B34, Neutral }, + { 0x01B36, 0x01B3A, Neutral }, + { 0x01B3C, 0x01B3C, Neutral }, + { 0x01B42, 0x01B42, Neutral }, + { 0x01B6B, 0x01B73, Neutral }, + { 0x01B80, 0x01B81, Neutral }, + { 0x01BA2, 0x01BA5, Neutral }, + { 0x01BA8, 0x01BA9, Neutral }, + { 0x01BAB, 0x01BAD, Neutral }, + { 0x01BE6, 0x01BE6, Neutral }, + { 0x01BE8, 0x01BE9, Neutral }, + { 0x01BED, 0x01BED, Neutral }, + { 0x01BEF, 0x01BF1, Neutral }, + { 0x01C2C, 0x01C33, Neutral }, + { 0x01C36, 0x01C37, Neutral }, + { 0x01CD0, 0x01CD2, Neutral }, + { 0x01CD4, 0x01CE0, Neutral }, + { 0x01CE2, 0x01CE8, Neutral }, + { 0x01CED, 0x01CED, Neutral }, + { 0x01CF4, 0x01CF4, Neutral }, + { 0x01CF8, 0x01CF9, Neutral }, + { 0x01DC0, 0x01DFF, Neutral }, + { 0x01FBD, 0x01FBD, Neutral }, + { 0x01FBF, 0x01FC1, Neutral }, + { 0x01FCD, 0x01FCF, Neutral }, + { 0x01FDD, 0x01FDF, Neutral }, + { 0x01FED, 0x01FEF, Neutral }, + { 0x01FFD, 0x01FFE, Neutral }, + { 0x02000, 0x0200D, Neutral }, + { 0x0200F, 0x0200F, RightToLeft }, + { 0x02010, 0x02070, Neutral }, + { 0x02074, 0x0207E, Neutral }, + { 0x02080, 0x0208E, Neutral }, + { 0x020A0, 0x020F0, Neutral }, + { 0x02100, 0x02101, Neutral }, + { 0x02103, 0x02106, Neutral }, + { 0x02108, 0x02109, Neutral }, + { 0x02114, 0x02114, Neutral }, + { 0x02116, 0x02118, Neutral }, + { 0x0211E, 0x02123, Neutral }, + { 0x02125, 0x02125, Neutral }, + { 0x02127, 0x02127, Neutral }, + { 0x02129, 0x02129, Neutral }, + { 0x0212E, 0x0212E, Neutral }, + { 0x0213A, 0x0213B, Neutral }, + { 0x02140, 0x02144, Neutral }, + { 0x0214A, 0x0214D, Neutral }, + { 0x02150, 0x0215F, Neutral }, + { 0x02189, 0x0218B, Neutral }, + { 0x02190, 0x02335, Neutral }, + { 0x0237B, 0x02394, Neutral }, + { 0x02396, 0x02429, Neutral }, + { 0x02440, 0x0244A, Neutral }, + { 0x02460, 0x0249B, Neutral }, + { 0x024EA, 0x026AB, Neutral }, + { 0x026AD, 0x027FF, Neutral }, + { 0x02900, 0x02B73, Neutral }, + { 0x02B76, 0x02BFF, Neutral }, + { 0x02CE5, 0x02CEA, Neutral }, + { 0x02CEF, 0x02CF1, Neutral }, + { 0x02CF9, 0x02CFF, Neutral }, + { 0x02D7F, 0x02D7F, Neutral }, + { 0x02DE0, 0x02E5D, Neutral }, + { 0x02E80, 0x02E99, Neutral }, + { 0x02E9B, 0x02EF3, Neutral }, + { 0x02F00, 0x02FD5, Neutral }, + { 0x02FF0, 0x03004, Neutral }, + { 0x03008, 0x03020, Neutral }, + { 0x0302A, 0x0302D, Neutral }, + { 0x03030, 0x03030, Neutral }, + { 0x03036, 0x03037, Neutral }, + { 0x0303D, 0x0303F, Neutral }, + { 0x03099, 0x0309C, Neutral }, + { 0x030A0, 0x030A0, Neutral }, + { 0x030FB, 0x030FB, Neutral }, + { 0x031C0, 0x031E5, Neutral }, + { 0x031EF, 0x031EF, Neutral }, + { 0x0321D, 0x0321E, Neutral }, + { 0x03250, 0x0325F, Neutral }, + { 0x0327C, 0x0327E, Neutral }, + { 0x032B1, 0x032BF, Neutral }, + { 0x032CC, 0x032CF, Neutral }, + { 0x03377, 0x0337A, Neutral }, + { 0x033DE, 0x033DF, Neutral }, + { 0x033FF, 0x033FF, Neutral }, + { 0x04DC0, 0x04DFF, Neutral }, + { 0x0A490, 0x0A4C6, Neutral }, + { 0x0A60D, 0x0A60F, Neutral }, + { 0x0A66F, 0x0A67F, Neutral }, + { 0x0A69E, 0x0A69F, Neutral }, + { 0x0A6F0, 0x0A6F1, Neutral }, + { 0x0A700, 0x0A721, Neutral }, + { 0x0A788, 0x0A788, Neutral }, + { 0x0A802, 0x0A802, Neutral }, + { 0x0A806, 0x0A806, Neutral }, + { 0x0A80B, 0x0A80B, Neutral }, + { 0x0A825, 0x0A826, Neutral }, + { 0x0A828, 0x0A82C, Neutral }, + { 0x0A838, 0x0A839, Neutral }, + { 0x0A874, 0x0A877, Neutral }, + { 0x0A8C4, 0x0A8C5, Neutral }, + { 0x0A8E0, 0x0A8F1, Neutral }, + { 0x0A8FF, 0x0A8FF, Neutral }, + { 0x0A926, 0x0A92D, Neutral }, + { 0x0A947, 0x0A951, Neutral }, + { 0x0A980, 0x0A982, Neutral }, + { 0x0A9B3, 0x0A9B3, Neutral }, + { 0x0A9B6, 0x0A9B9, Neutral }, + { 0x0A9BC, 0x0A9BD, Neutral }, + { 0x0A9E5, 0x0A9E5, Neutral }, + { 0x0AA29, 0x0AA2E, Neutral }, + { 0x0AA31, 0x0AA32, Neutral }, + { 0x0AA35, 0x0AA36, Neutral }, + { 0x0AA43, 0x0AA43, Neutral }, + { 0x0AA4C, 0x0AA4C, Neutral }, + { 0x0AA7C, 0x0AA7C, Neutral }, + { 0x0AAB0, 0x0AAB0, Neutral }, + { 0x0AAB2, 0x0AAB4, Neutral }, + { 0x0AAB7, 0x0AAB8, Neutral }, + { 0x0AABE, 0x0AABF, Neutral }, + { 0x0AAC1, 0x0AAC1, Neutral }, + { 0x0AAEC, 0x0AAED, Neutral }, + { 0x0AAF6, 0x0AAF6, Neutral }, + { 0x0AB6A, 0x0AB6B, Neutral }, + { 0x0ABE5, 0x0ABE5, Neutral }, + { 0x0ABE8, 0x0ABE8, Neutral }, + { 0x0ABED, 0x0ABED, Neutral }, + { 0x0FB1D, 0x0FB1D, RightToLeft }, + { 0x0FB1E, 0x0FB1E, Neutral }, + { 0x0FB1F, 0x0FB28, RightToLeft }, + { 0x0FB29, 0x0FB29, Neutral }, + { 0x0FB2A, 0x0FBC2, RightToLeft }, + { 0x0FBC3, 0x0FBD2, Neutral }, + { 0x0FBD3, 0x0FD3D, RightToLeft }, + { 0x0FD3E, 0x0FD4F, Neutral }, + { 0x0FD50, 0x0FD8F, RightToLeft }, + { 0x0FD90, 0x0FD91, Neutral }, + { 0x0FD92, 0x0FDC7, RightToLeft }, + { 0x0FDC8, 0x0FDEF, Neutral }, + { 0x0FDF0, 0x0FDFC, RightToLeft }, + { 0x0FDFD, 0x0FE19, Neutral }, + { 0x0FE20, 0x0FE52, Neutral }, + { 0x0FE54, 0x0FE66, Neutral }, + { 0x0FE68, 0x0FE6B, Neutral }, + { 0x0FE70, 0x0FEFE, RightToLeft }, + { 0x0FEFF, 0x0FEFF, Neutral }, + { 0x0FF01, 0x0FF20, Neutral }, + { 0x0FF3B, 0x0FF40, Neutral }, + { 0x0FF5B, 0x0FF65, Neutral }, + { 0x0FFE0, 0x0FFE6, Neutral }, + { 0x0FFE8, 0x0FFEE, Neutral }, + { 0x0FFF0, 0x0FFFF, Neutral }, + { 0x10101, 0x10101, Neutral }, + { 0x10140, 0x1018C, Neutral }, + { 0x10190, 0x1019C, Neutral }, + { 0x101A0, 0x101A0, Neutral }, + { 0x101FD, 0x101FD, Neutral }, + { 0x102E0, 0x102FB, Neutral }, + { 0x10376, 0x1037A, Neutral }, + { 0x10800, 0x1091E, RightToLeft }, + { 0x1091F, 0x1091F, Neutral }, + { 0x10920, 0x10A00, RightToLeft }, + { 0x10A01, 0x10A03, Neutral }, + { 0x10A04, 0x10A04, RightToLeft }, + { 0x10A05, 0x10A06, Neutral }, + { 0x10A07, 0x10A0B, RightToLeft }, + { 0x10A0C, 0x10A0F, Neutral }, + { 0x10A10, 0x10A37, RightToLeft }, + { 0x10A38, 0x10A3A, Neutral }, + { 0x10A3B, 0x10A3E, RightToLeft }, + { 0x10A3F, 0x10A3F, Neutral }, + { 0x10A40, 0x10AE4, RightToLeft }, + { 0x10AE5, 0x10AE6, Neutral }, + { 0x10AE7, 0x10B38, RightToLeft }, + { 0x10B39, 0x10B3F, Neutral }, + { 0x10B40, 0x10D23, RightToLeft }, + { 0x10D24, 0x10D27, Neutral }, + { 0x10D28, 0x10D2F, RightToLeft }, + { 0x10D30, 0x10D39, Neutral }, + { 0x10D3A, 0x10D3F, RightToLeft }, + { 0x10D40, 0x10D49, Neutral }, + { 0x10D4A, 0x10D68, RightToLeft }, + { 0x10D69, 0x10D6E, Neutral }, + { 0x10D6F, 0x10E5F, RightToLeft }, + { 0x10E60, 0x10E7E, Neutral }, + { 0x10E7F, 0x10EAA, RightToLeft }, + { 0x10EAB, 0x10EAC, Neutral }, + { 0x10EAD, 0x10ECF, RightToLeft }, + { 0x10ED0, 0x10ED8, Neutral }, + { 0x10ED9, 0x10EF9, RightToLeft }, + { 0x10EFA, 0x10EFF, Neutral }, + { 0x10F00, 0x10F45, RightToLeft }, + { 0x10F46, 0x10F50, Neutral }, + { 0x10F51, 0x10F81, RightToLeft }, + { 0x10F82, 0x10F85, Neutral }, + { 0x10F86, 0x10FFF, RightToLeft }, + { 0x11001, 0x11001, Neutral }, + { 0x11038, 0x11046, Neutral }, + { 0x11052, 0x11065, Neutral }, + { 0x11070, 0x11070, Neutral }, + { 0x11073, 0x11074, Neutral }, + { 0x1107F, 0x11081, Neutral }, + { 0x110B3, 0x110B6, Neutral }, + { 0x110B9, 0x110BA, Neutral }, + { 0x110C2, 0x110C2, Neutral }, + { 0x11100, 0x11102, Neutral }, + { 0x11127, 0x1112B, Neutral }, + { 0x1112D, 0x11134, Neutral }, + { 0x11173, 0x11173, Neutral }, + { 0x11180, 0x11181, Neutral }, + { 0x111B6, 0x111BE, Neutral }, + { 0x111C9, 0x111CC, Neutral }, + { 0x111CF, 0x111CF, Neutral }, + { 0x1122F, 0x11231, Neutral }, + { 0x11234, 0x11234, Neutral }, + { 0x11236, 0x11237, Neutral }, + { 0x1123E, 0x1123E, Neutral }, + { 0x11241, 0x11241, Neutral }, + { 0x112DF, 0x112DF, Neutral }, + { 0x112E3, 0x112EA, Neutral }, + { 0x11300, 0x11301, Neutral }, + { 0x1133B, 0x1133C, Neutral }, + { 0x11340, 0x11340, Neutral }, + { 0x11366, 0x1136C, Neutral }, + { 0x11370, 0x11374, Neutral }, + { 0x113BB, 0x113C0, Neutral }, + { 0x113CE, 0x113CE, Neutral }, + { 0x113D0, 0x113D0, Neutral }, + { 0x113D2, 0x113D2, Neutral }, + { 0x113E1, 0x113E2, Neutral }, + { 0x11438, 0x1143F, Neutral }, + { 0x11442, 0x11444, Neutral }, + { 0x11446, 0x11446, Neutral }, + { 0x1145E, 0x1145E, Neutral }, + { 0x114B3, 0x114B8, Neutral }, + { 0x114BA, 0x114BA, Neutral }, + { 0x114BF, 0x114C0, Neutral }, + { 0x114C2, 0x114C3, Neutral }, + { 0x115B2, 0x115B5, Neutral }, + { 0x115BC, 0x115BD, Neutral }, + { 0x115BF, 0x115C0, Neutral }, + { 0x115DC, 0x115DD, Neutral }, + { 0x11633, 0x1163A, Neutral }, + { 0x1163D, 0x1163D, Neutral }, + { 0x1163F, 0x11640, Neutral }, + { 0x11660, 0x1166C, Neutral }, + { 0x116AB, 0x116AB, Neutral }, + { 0x116AD, 0x116AD, Neutral }, + { 0x116B0, 0x116B5, Neutral }, + { 0x116B7, 0x116B7, Neutral }, + { 0x1171D, 0x1171D, Neutral }, + { 0x1171F, 0x1171F, Neutral }, + { 0x11722, 0x11725, Neutral }, + { 0x11727, 0x1172B, Neutral }, + { 0x1182F, 0x11837, Neutral }, + { 0x11839, 0x1183A, Neutral }, + { 0x1193B, 0x1193C, Neutral }, + { 0x1193E, 0x1193E, Neutral }, + { 0x11943, 0x11943, Neutral }, + { 0x119D4, 0x119D7, Neutral }, + { 0x119DA, 0x119DB, Neutral }, + { 0x119E0, 0x119E0, Neutral }, + { 0x11A01, 0x11A06, Neutral }, + { 0x11A09, 0x11A0A, Neutral }, + { 0x11A33, 0x11A38, Neutral }, + { 0x11A3B, 0x11A3E, Neutral }, + { 0x11A47, 0x11A47, Neutral }, + { 0x11A51, 0x11A56, Neutral }, + { 0x11A59, 0x11A5B, Neutral }, + { 0x11A8A, 0x11A96, Neutral }, + { 0x11A98, 0x11A99, Neutral }, + { 0x11B60, 0x11B60, Neutral }, + { 0x11B62, 0x11B64, Neutral }, + { 0x11B66, 0x11B66, Neutral }, + { 0x11C30, 0x11C36, Neutral }, + { 0x11C38, 0x11C3D, Neutral }, + { 0x11C92, 0x11CA7, Neutral }, + { 0x11CAA, 0x11CB0, Neutral }, + { 0x11CB2, 0x11CB3, Neutral }, + { 0x11CB5, 0x11CB6, Neutral }, + { 0x11D31, 0x11D36, Neutral }, + { 0x11D3A, 0x11D3A, Neutral }, + { 0x11D3C, 0x11D3D, Neutral }, + { 0x11D3F, 0x11D45, Neutral }, + { 0x11D47, 0x11D47, Neutral }, + { 0x11D90, 0x11D91, Neutral }, + { 0x11D95, 0x11D95, Neutral }, + { 0x11D97, 0x11D97, Neutral }, + { 0x11EF3, 0x11EF4, Neutral }, + { 0x11F00, 0x11F01, Neutral }, + { 0x11F36, 0x11F3A, Neutral }, + { 0x11F40, 0x11F40, Neutral }, + { 0x11F42, 0x11F42, Neutral }, + { 0x11F5A, 0x11F5A, Neutral }, + { 0x11FD5, 0x11FF1, Neutral }, + { 0x13440, 0x13440, Neutral }, + { 0x13447, 0x13455, Neutral }, + { 0x1611E, 0x16129, Neutral }, + { 0x1612D, 0x1612F, Neutral }, + { 0x16AF0, 0x16AF4, Neutral }, + { 0x16B30, 0x16B36, Neutral }, + { 0x16F4F, 0x16F4F, Neutral }, + { 0x16F8F, 0x16F92, Neutral }, + { 0x16FE2, 0x16FE2, Neutral }, + { 0x16FE4, 0x16FE4, Neutral }, + { 0x1BC9D, 0x1BC9E, Neutral }, + { 0x1BCA0, 0x1BCA3, Neutral }, + { 0x1CC00, 0x1CCD5, Neutral }, + { 0x1CCF0, 0x1CCFC, Neutral }, + { 0x1CD00, 0x1CEB3, Neutral }, + { 0x1CEBA, 0x1CED0, Neutral }, + { 0x1CEE0, 0x1CEF0, Neutral }, + { 0x1CF00, 0x1CF2D, Neutral }, + { 0x1CF30, 0x1CF46, Neutral }, + { 0x1D167, 0x1D169, Neutral }, + { 0x1D173, 0x1D182, Neutral }, + { 0x1D185, 0x1D18B, Neutral }, + { 0x1D1AA, 0x1D1AD, Neutral }, + { 0x1D1E9, 0x1D1EA, Neutral }, + { 0x1D200, 0x1D245, Neutral }, + { 0x1D300, 0x1D356, Neutral }, + { 0x1D6C1, 0x1D6C1, Neutral }, + { 0x1D6DB, 0x1D6DB, Neutral }, + { 0x1D6FB, 0x1D6FB, Neutral }, + { 0x1D715, 0x1D715, Neutral }, + { 0x1D735, 0x1D735, Neutral }, + { 0x1D74F, 0x1D74F, Neutral }, + { 0x1D76F, 0x1D76F, Neutral }, + { 0x1D789, 0x1D789, Neutral }, + { 0x1D7A9, 0x1D7A9, Neutral }, + { 0x1D7C3, 0x1D7C3, Neutral }, + { 0x1D7CE, 0x1D7FF, Neutral }, + { 0x1DA00, 0x1DA36, Neutral }, + { 0x1DA3B, 0x1DA6C, Neutral }, + { 0x1DA75, 0x1DA75, Neutral }, + { 0x1DA84, 0x1DA84, Neutral }, + { 0x1DA9B, 0x1DA9F, Neutral }, + { 0x1DAA1, 0x1DAAF, Neutral }, + { 0x1E000, 0x1E006, Neutral }, + { 0x1E008, 0x1E018, Neutral }, + { 0x1E01B, 0x1E021, Neutral }, + { 0x1E023, 0x1E024, Neutral }, + { 0x1E026, 0x1E02A, Neutral }, + { 0x1E08F, 0x1E08F, Neutral }, + { 0x1E130, 0x1E136, Neutral }, + { 0x1E2AE, 0x1E2AE, Neutral }, + { 0x1E2EC, 0x1E2EF, Neutral }, + { 0x1E2FF, 0x1E2FF, Neutral }, + { 0x1E4EC, 0x1E4EF, Neutral }, + { 0x1E5EE, 0x1E5EF, Neutral }, + { 0x1E6E3, 0x1E6E3, Neutral }, + { 0x1E6E6, 0x1E6E6, Neutral }, + { 0x1E6EE, 0x1E6EF, Neutral }, + { 0x1E6F5, 0x1E6F5, Neutral }, + { 0x1E800, 0x1E8CF, RightToLeft }, + { 0x1E8D0, 0x1E8D6, Neutral }, + { 0x1E8D7, 0x1E943, RightToLeft }, + { 0x1E944, 0x1E94A, Neutral }, + { 0x1E94B, 0x1EEEF, RightToLeft }, + { 0x1EEF0, 0x1EEF1, Neutral }, + { 0x1EEF2, 0x1EFFF, RightToLeft }, + { 0x1F000, 0x1F02B, Neutral }, + { 0x1F030, 0x1F093, Neutral }, + { 0x1F0A0, 0x1F0AE, Neutral }, + { 0x1F0B1, 0x1F0BF, Neutral }, + { 0x1F0C1, 0x1F0CF, Neutral }, + { 0x1F0D1, 0x1F0F5, Neutral }, + { 0x1F100, 0x1F10F, Neutral }, + { 0x1F12F, 0x1F12F, Neutral }, + { 0x1F16A, 0x1F16F, Neutral }, + { 0x1F1AD, 0x1F1AD, Neutral }, + { 0x1F260, 0x1F265, Neutral }, + { 0x1F300, 0x1F6D8, Neutral }, + { 0x1F6DC, 0x1F6EC, Neutral }, + { 0x1F6F0, 0x1F6FC, Neutral }, + { 0x1F700, 0x1F7D9, Neutral }, + { 0x1F7E0, 0x1F7EB, Neutral }, + { 0x1F7F0, 0x1F7F0, Neutral }, + { 0x1F800, 0x1F80B, Neutral }, + { 0x1F810, 0x1F847, Neutral }, + { 0x1F850, 0x1F859, Neutral }, + { 0x1F860, 0x1F887, Neutral }, + { 0x1F890, 0x1F8AD, Neutral }, + { 0x1F8B0, 0x1F8BB, Neutral }, + { 0x1F8C0, 0x1F8C1, Neutral }, + { 0x1F8D0, 0x1F8D8, Neutral }, + { 0x1F900, 0x1FA57, Neutral }, + { 0x1FA60, 0x1FA6D, Neutral }, + { 0x1FA70, 0x1FA7C, Neutral }, + { 0x1FA80, 0x1FA8A, Neutral }, + { 0x1FA8E, 0x1FAC6, Neutral }, + { 0x1FAC8, 0x1FAC8, Neutral }, + { 0x1FACD, 0x1FADC, Neutral }, + { 0x1FADF, 0x1FAEA, Neutral }, + { 0x1FAEF, 0x1FAF8, Neutral }, + { 0x1FB00, 0x1FB92, Neutral }, + { 0x1FB94, 0x1FBFA, Neutral }, + { 0x1FFFE, 0x1FFFF, Neutral }, + { 0x2FFFE, 0x2FFFF, Neutral }, + { 0x3FFFE, 0x3FFFF, Neutral }, + { 0x4FFFE, 0x4FFFF, Neutral }, + { 0x5FFFE, 0x5FFFF, Neutral }, + { 0x6FFFE, 0x6FFFF, Neutral }, + { 0x7FFFE, 0x7FFFF, Neutral }, + { 0x8FFFE, 0x8FFFF, Neutral }, + { 0x9FFFE, 0x9FFFF, Neutral }, + { 0xAFFFE, 0xAFFFF, Neutral }, + { 0xBFFFE, 0xBFFFF, Neutral }, + { 0xCFFFE, 0xCFFFF, Neutral }, + { 0xDFFFE, 0xE0FFF, Neutral }, + { 0xEFFFE, 0xEFFFF, Neutral }, + { 0xFFFFE, 0xFFFFF, Neutral }, + { 0x10FFFE, 0x10FFFF, Neutral }, diff --git a/Dependencies/UnicodeBidi/generate_supplementary_bidi.py b/Dependencies/UnicodeBidi/generate_bidi.py similarity index 81% rename from Dependencies/UnicodeBidi/generate_supplementary_bidi.py rename to Dependencies/UnicodeBidi/generate_bidi.py index cb2c0a92271..dd40646d90c 100644 --- a/Dependencies/UnicodeBidi/generate_supplementary_bidi.py +++ b/Dependencies/UnicodeBidi/generate_bidi.py @@ -1,7 +1,7 @@ -"""Generate the supplementary first-strong-direction table from Unicode 17.0.0. +"""Generate the first-strong-direction table for all Unicode 17.0.0 code points. Input: https://www.unicode.org/Public/17.0.0/ucd/extracted/DerivedBidiClass.txt -Usage: python Dependencies/UnicodeBidi/generate_supplementary_bidi.py DerivedBidiClass.txt [--check] +Usage: python Dependencies/UnicodeBidi/generate_bidi.py DerivedBidiClass.txt [--check] """ import argparse @@ -25,12 +25,12 @@ def generate(source): directions = ["LeftToRight" if c == "L" else "RightToLeft" if c in ("R", "AL") else "Neutral" for c in classes] lines = [ - "// Generated by generate_supplementary_bidi.py; do not edit.", + "// Generated by generate_bidi.py; do not edit.", "// Unicode 17.0.0 DerivedBidiClass.txt, including its @missing defaults.", "// Copyright 2025 Unicode, Inc. See unicode-license.txt.", "// Omitted ranges have direction LeftToRight. Neutral means neither L, R nor AL.", ] - start = 0x10000 + start = 0 for end in range(start + 1, len(directions) + 1): if end == len(directions) or directions[end] != directions[start]: if directions[start] != "LeftToRight": @@ -44,11 +44,11 @@ def generate(source): parser.add_argument("source", type=Path) parser.add_argument("--check", action="store_true") args = parser.parse_args() - output = Path(__file__).resolve().parent / "supplementarybidi.inl" + output = Path(__file__).resolve().parent / "bididirection.inl" generated = generate(args.source.read_text(encoding="utf-8")) if args.check: if output.read_text(encoding="utf-8") != generated: - raise SystemExit("Supplementary bidi table is out of date") - print("Unicode 17.0.0 supplementary bidi table matches") + raise SystemExit("Bidi direction table is out of date") + print("Unicode 17.0.0 bidi direction table matches") else: output.write_text(generated, encoding="utf-8") diff --git a/Dependencies/UnicodeBidi/supplementarybidi.inl b/Dependencies/UnicodeBidi/supplementarybidi.inl deleted file mode 100644 index 9b2a9adada8..00000000000 --- a/Dependencies/UnicodeBidi/supplementarybidi.inl +++ /dev/null @@ -1,265 +0,0 @@ -// Generated by generate_supplementary_bidi.py; do not edit. -// Unicode 17.0.0 DerivedBidiClass.txt, including its @missing defaults. -// Copyright 2025 Unicode, Inc. See unicode-license.txt. -// Omitted ranges have direction LeftToRight. Neutral means neither L, R nor AL. - { 0x10101, 0x10101, Neutral }, - { 0x10140, 0x1018C, Neutral }, - { 0x10190, 0x1019C, Neutral }, - { 0x101A0, 0x101A0, Neutral }, - { 0x101FD, 0x101FD, Neutral }, - { 0x102E0, 0x102FB, Neutral }, - { 0x10376, 0x1037A, Neutral }, - { 0x10800, 0x1091E, RightToLeft }, - { 0x1091F, 0x1091F, Neutral }, - { 0x10920, 0x10A00, RightToLeft }, - { 0x10A01, 0x10A03, Neutral }, - { 0x10A04, 0x10A04, RightToLeft }, - { 0x10A05, 0x10A06, Neutral }, - { 0x10A07, 0x10A0B, RightToLeft }, - { 0x10A0C, 0x10A0F, Neutral }, - { 0x10A10, 0x10A37, RightToLeft }, - { 0x10A38, 0x10A3A, Neutral }, - { 0x10A3B, 0x10A3E, RightToLeft }, - { 0x10A3F, 0x10A3F, Neutral }, - { 0x10A40, 0x10AE4, RightToLeft }, - { 0x10AE5, 0x10AE6, Neutral }, - { 0x10AE7, 0x10B38, RightToLeft }, - { 0x10B39, 0x10B3F, Neutral }, - { 0x10B40, 0x10D23, RightToLeft }, - { 0x10D24, 0x10D27, Neutral }, - { 0x10D28, 0x10D2F, RightToLeft }, - { 0x10D30, 0x10D39, Neutral }, - { 0x10D3A, 0x10D3F, RightToLeft }, - { 0x10D40, 0x10D49, Neutral }, - { 0x10D4A, 0x10D68, RightToLeft }, - { 0x10D69, 0x10D6E, Neutral }, - { 0x10D6F, 0x10E5F, RightToLeft }, - { 0x10E60, 0x10E7E, Neutral }, - { 0x10E7F, 0x10EAA, RightToLeft }, - { 0x10EAB, 0x10EAC, Neutral }, - { 0x10EAD, 0x10ECF, RightToLeft }, - { 0x10ED0, 0x10ED8, Neutral }, - { 0x10ED9, 0x10EF9, RightToLeft }, - { 0x10EFA, 0x10EFF, Neutral }, - { 0x10F00, 0x10F45, RightToLeft }, - { 0x10F46, 0x10F50, Neutral }, - { 0x10F51, 0x10F81, RightToLeft }, - { 0x10F82, 0x10F85, Neutral }, - { 0x10F86, 0x10FFF, RightToLeft }, - { 0x11001, 0x11001, Neutral }, - { 0x11038, 0x11046, Neutral }, - { 0x11052, 0x11065, Neutral }, - { 0x11070, 0x11070, Neutral }, - { 0x11073, 0x11074, Neutral }, - { 0x1107F, 0x11081, Neutral }, - { 0x110B3, 0x110B6, Neutral }, - { 0x110B9, 0x110BA, Neutral }, - { 0x110C2, 0x110C2, Neutral }, - { 0x11100, 0x11102, Neutral }, - { 0x11127, 0x1112B, Neutral }, - { 0x1112D, 0x11134, Neutral }, - { 0x11173, 0x11173, Neutral }, - { 0x11180, 0x11181, Neutral }, - { 0x111B6, 0x111BE, Neutral }, - { 0x111C9, 0x111CC, Neutral }, - { 0x111CF, 0x111CF, Neutral }, - { 0x1122F, 0x11231, Neutral }, - { 0x11234, 0x11234, Neutral }, - { 0x11236, 0x11237, Neutral }, - { 0x1123E, 0x1123E, Neutral }, - { 0x11241, 0x11241, Neutral }, - { 0x112DF, 0x112DF, Neutral }, - { 0x112E3, 0x112EA, Neutral }, - { 0x11300, 0x11301, Neutral }, - { 0x1133B, 0x1133C, Neutral }, - { 0x11340, 0x11340, Neutral }, - { 0x11366, 0x1136C, Neutral }, - { 0x11370, 0x11374, Neutral }, - { 0x113BB, 0x113C0, Neutral }, - { 0x113CE, 0x113CE, Neutral }, - { 0x113D0, 0x113D0, Neutral }, - { 0x113D2, 0x113D2, Neutral }, - { 0x113E1, 0x113E2, Neutral }, - { 0x11438, 0x1143F, Neutral }, - { 0x11442, 0x11444, Neutral }, - { 0x11446, 0x11446, Neutral }, - { 0x1145E, 0x1145E, Neutral }, - { 0x114B3, 0x114B8, Neutral }, - { 0x114BA, 0x114BA, Neutral }, - { 0x114BF, 0x114C0, Neutral }, - { 0x114C2, 0x114C3, Neutral }, - { 0x115B2, 0x115B5, Neutral }, - { 0x115BC, 0x115BD, Neutral }, - { 0x115BF, 0x115C0, Neutral }, - { 0x115DC, 0x115DD, Neutral }, - { 0x11633, 0x1163A, Neutral }, - { 0x1163D, 0x1163D, Neutral }, - { 0x1163F, 0x11640, Neutral }, - { 0x11660, 0x1166C, Neutral }, - { 0x116AB, 0x116AB, Neutral }, - { 0x116AD, 0x116AD, Neutral }, - { 0x116B0, 0x116B5, Neutral }, - { 0x116B7, 0x116B7, Neutral }, - { 0x1171D, 0x1171D, Neutral }, - { 0x1171F, 0x1171F, Neutral }, - { 0x11722, 0x11725, Neutral }, - { 0x11727, 0x1172B, Neutral }, - { 0x1182F, 0x11837, Neutral }, - { 0x11839, 0x1183A, Neutral }, - { 0x1193B, 0x1193C, Neutral }, - { 0x1193E, 0x1193E, Neutral }, - { 0x11943, 0x11943, Neutral }, - { 0x119D4, 0x119D7, Neutral }, - { 0x119DA, 0x119DB, Neutral }, - { 0x119E0, 0x119E0, Neutral }, - { 0x11A01, 0x11A06, Neutral }, - { 0x11A09, 0x11A0A, Neutral }, - { 0x11A33, 0x11A38, Neutral }, - { 0x11A3B, 0x11A3E, Neutral }, - { 0x11A47, 0x11A47, Neutral }, - { 0x11A51, 0x11A56, Neutral }, - { 0x11A59, 0x11A5B, Neutral }, - { 0x11A8A, 0x11A96, Neutral }, - { 0x11A98, 0x11A99, Neutral }, - { 0x11B60, 0x11B60, Neutral }, - { 0x11B62, 0x11B64, Neutral }, - { 0x11B66, 0x11B66, Neutral }, - { 0x11C30, 0x11C36, Neutral }, - { 0x11C38, 0x11C3D, Neutral }, - { 0x11C92, 0x11CA7, Neutral }, - { 0x11CAA, 0x11CB0, Neutral }, - { 0x11CB2, 0x11CB3, Neutral }, - { 0x11CB5, 0x11CB6, Neutral }, - { 0x11D31, 0x11D36, Neutral }, - { 0x11D3A, 0x11D3A, Neutral }, - { 0x11D3C, 0x11D3D, Neutral }, - { 0x11D3F, 0x11D45, Neutral }, - { 0x11D47, 0x11D47, Neutral }, - { 0x11D90, 0x11D91, Neutral }, - { 0x11D95, 0x11D95, Neutral }, - { 0x11D97, 0x11D97, Neutral }, - { 0x11EF3, 0x11EF4, Neutral }, - { 0x11F00, 0x11F01, Neutral }, - { 0x11F36, 0x11F3A, Neutral }, - { 0x11F40, 0x11F40, Neutral }, - { 0x11F42, 0x11F42, Neutral }, - { 0x11F5A, 0x11F5A, Neutral }, - { 0x11FD5, 0x11FF1, Neutral }, - { 0x13440, 0x13440, Neutral }, - { 0x13447, 0x13455, Neutral }, - { 0x1611E, 0x16129, Neutral }, - { 0x1612D, 0x1612F, Neutral }, - { 0x16AF0, 0x16AF4, Neutral }, - { 0x16B30, 0x16B36, Neutral }, - { 0x16F4F, 0x16F4F, Neutral }, - { 0x16F8F, 0x16F92, Neutral }, - { 0x16FE2, 0x16FE2, Neutral }, - { 0x16FE4, 0x16FE4, Neutral }, - { 0x1BC9D, 0x1BC9E, Neutral }, - { 0x1BCA0, 0x1BCA3, Neutral }, - { 0x1CC00, 0x1CCD5, Neutral }, - { 0x1CCF0, 0x1CCFC, Neutral }, - { 0x1CD00, 0x1CEB3, Neutral }, - { 0x1CEBA, 0x1CED0, Neutral }, - { 0x1CEE0, 0x1CEF0, Neutral }, - { 0x1CF00, 0x1CF2D, Neutral }, - { 0x1CF30, 0x1CF46, Neutral }, - { 0x1D167, 0x1D169, Neutral }, - { 0x1D173, 0x1D182, Neutral }, - { 0x1D185, 0x1D18B, Neutral }, - { 0x1D1AA, 0x1D1AD, Neutral }, - { 0x1D1E9, 0x1D1EA, Neutral }, - { 0x1D200, 0x1D245, Neutral }, - { 0x1D300, 0x1D356, Neutral }, - { 0x1D6C1, 0x1D6C1, Neutral }, - { 0x1D6DB, 0x1D6DB, Neutral }, - { 0x1D6FB, 0x1D6FB, Neutral }, - { 0x1D715, 0x1D715, Neutral }, - { 0x1D735, 0x1D735, Neutral }, - { 0x1D74F, 0x1D74F, Neutral }, - { 0x1D76F, 0x1D76F, Neutral }, - { 0x1D789, 0x1D789, Neutral }, - { 0x1D7A9, 0x1D7A9, Neutral }, - { 0x1D7C3, 0x1D7C3, Neutral }, - { 0x1D7CE, 0x1D7FF, Neutral }, - { 0x1DA00, 0x1DA36, Neutral }, - { 0x1DA3B, 0x1DA6C, Neutral }, - { 0x1DA75, 0x1DA75, Neutral }, - { 0x1DA84, 0x1DA84, Neutral }, - { 0x1DA9B, 0x1DA9F, Neutral }, - { 0x1DAA1, 0x1DAAF, Neutral }, - { 0x1E000, 0x1E006, Neutral }, - { 0x1E008, 0x1E018, Neutral }, - { 0x1E01B, 0x1E021, Neutral }, - { 0x1E023, 0x1E024, Neutral }, - { 0x1E026, 0x1E02A, Neutral }, - { 0x1E08F, 0x1E08F, Neutral }, - { 0x1E130, 0x1E136, Neutral }, - { 0x1E2AE, 0x1E2AE, Neutral }, - { 0x1E2EC, 0x1E2EF, Neutral }, - { 0x1E2FF, 0x1E2FF, Neutral }, - { 0x1E4EC, 0x1E4EF, Neutral }, - { 0x1E5EE, 0x1E5EF, Neutral }, - { 0x1E6E3, 0x1E6E3, Neutral }, - { 0x1E6E6, 0x1E6E6, Neutral }, - { 0x1E6EE, 0x1E6EF, Neutral }, - { 0x1E6F5, 0x1E6F5, Neutral }, - { 0x1E800, 0x1E8CF, RightToLeft }, - { 0x1E8D0, 0x1E8D6, Neutral }, - { 0x1E8D7, 0x1E943, RightToLeft }, - { 0x1E944, 0x1E94A, Neutral }, - { 0x1E94B, 0x1EEEF, RightToLeft }, - { 0x1EEF0, 0x1EEF1, Neutral }, - { 0x1EEF2, 0x1EFFF, RightToLeft }, - { 0x1F000, 0x1F02B, Neutral }, - { 0x1F030, 0x1F093, Neutral }, - { 0x1F0A0, 0x1F0AE, Neutral }, - { 0x1F0B1, 0x1F0BF, Neutral }, - { 0x1F0C1, 0x1F0CF, Neutral }, - { 0x1F0D1, 0x1F0F5, Neutral }, - { 0x1F100, 0x1F10F, Neutral }, - { 0x1F12F, 0x1F12F, Neutral }, - { 0x1F16A, 0x1F16F, Neutral }, - { 0x1F1AD, 0x1F1AD, Neutral }, - { 0x1F260, 0x1F265, Neutral }, - { 0x1F300, 0x1F6D8, Neutral }, - { 0x1F6DC, 0x1F6EC, Neutral }, - { 0x1F6F0, 0x1F6FC, Neutral }, - { 0x1F700, 0x1F7D9, Neutral }, - { 0x1F7E0, 0x1F7EB, Neutral }, - { 0x1F7F0, 0x1F7F0, Neutral }, - { 0x1F800, 0x1F80B, Neutral }, - { 0x1F810, 0x1F847, Neutral }, - { 0x1F850, 0x1F859, Neutral }, - { 0x1F860, 0x1F887, Neutral }, - { 0x1F890, 0x1F8AD, Neutral }, - { 0x1F8B0, 0x1F8BB, Neutral }, - { 0x1F8C0, 0x1F8C1, Neutral }, - { 0x1F8D0, 0x1F8D8, Neutral }, - { 0x1F900, 0x1FA57, Neutral }, - { 0x1FA60, 0x1FA6D, Neutral }, - { 0x1FA70, 0x1FA7C, Neutral }, - { 0x1FA80, 0x1FA8A, Neutral }, - { 0x1FA8E, 0x1FAC6, Neutral }, - { 0x1FAC8, 0x1FAC8, Neutral }, - { 0x1FACD, 0x1FADC, Neutral }, - { 0x1FADF, 0x1FAEA, Neutral }, - { 0x1FAEF, 0x1FAF8, Neutral }, - { 0x1FB00, 0x1FB92, Neutral }, - { 0x1FB94, 0x1FBFA, Neutral }, - { 0x1FFFE, 0x1FFFF, Neutral }, - { 0x2FFFE, 0x2FFFF, Neutral }, - { 0x3FFFE, 0x3FFFF, Neutral }, - { 0x4FFFE, 0x4FFFF, Neutral }, - { 0x5FFFE, 0x5FFFF, Neutral }, - { 0x6FFFE, 0x6FFFF, Neutral }, - { 0x7FFFE, 0x7FFFF, Neutral }, - { 0x8FFFE, 0x8FFFF, Neutral }, - { 0x9FFFE, 0x9FFFF, Neutral }, - { 0xAFFFE, 0xAFFFF, Neutral }, - { 0xBFFFE, 0xBFFFF, Neutral }, - { 0xCFFFE, 0xCFFFF, Neutral }, - { 0xDFFFE, 0xE0FFF, Neutral }, - { 0xEFFFE, 0xEFFFF, Neutral }, - { 0xFFFFE, 0xFFFFF, Neutral }, - { 0x10FFFE, 0x10FFFF, Neutral }, diff --git a/Dependencies/UnicodeBidi/unicodebidi.cpp b/Dependencies/UnicodeBidi/unicodebidi.cpp index 6906db57d8a..d437c3f02fa 100644 --- a/Dependencies/UnicodeBidi/unicodebidi.cpp +++ b/Dependencies/UnicodeBidi/unicodebidi.cpp @@ -35,30 +35,30 @@ struct DirectionRange Direction Value; }; -static const DirectionRange SupplementaryDirectionRanges[] = { -#include "supplementarybidi.inl" +static const DirectionRange DirectionRanges[] = { +#include "bididirection.inl" }; -static Direction Get_Supplementary_Direction(unsigned codepoint) +static Direction Get_Direction(unsigned codepoint) { unsigned first = 0; - unsigned last = sizeof(SupplementaryDirectionRanges) / sizeof(SupplementaryDirectionRanges[0]); + unsigned last = sizeof(DirectionRanges) / sizeof(DirectionRanges[0]); while (first < last) { const unsigned middle = first + (last - first) / 2; - if (codepoint < SupplementaryDirectionRanges[middle].First) { + if (codepoint < DirectionRanges[middle].First) { last = middle; - } else if (codepoint > SupplementaryDirectionRanges[middle].Last) { + } else if (codepoint > DirectionRanges[middle].Last) { first = middle + 1; } else { - return SupplementaryDirectionRanges[middle].Value; + return DirectionRanges[middle].Value; } } return LeftToRight; } -static bool Is_Paragraph_Separator(WCHAR ch) +static bool Is_Paragraph_Separator(unsigned ch) { return ch == L'\n' || ch == L'\r' || (ch >= 0x001C && ch <= 0x001E) || ch == 0x0085 || ch == 0x2029; @@ -69,23 +69,23 @@ static bool Is_Paragraph_Separator(WCHAR ch) namespace UnicodeBidi { -WORD Get_Paragraph_Base_Level(const WCHAR *text, int length) +unsigned short Get_Paragraph_Base_Level(const wchar_t *text, int length) { int isolate_depth = 0; for (int index = 0; index < length; ++index) { - const WCHAR ch = text[index]; + unsigned codepoint = static_cast(text[index]); // First-strong detection applies to one bidi paragraph only. - if (Is_Paragraph_Separator(ch)) { + if (Is_Paragraph_Separator(codepoint)) { break; } - if (ch >= 0x2066 && ch <= 0x2068) { + if (codepoint >= 0x2066 && codepoint <= 0x2068) { ++isolate_depth; continue; } - if (ch == 0x2069) { + if (codepoint == 0x2069) { if (isolate_depth > 0) { --isolate_depth; } @@ -97,23 +97,20 @@ WORD Get_Paragraph_Base_Level(const WCHAR *text, int length) continue; } - Direction direction = Neutral; - if (ch >= 0xD800 && ch <= 0xDBFF) { + // Decode a UTF-16 pair before classification; lone surrogates are ignored. + if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { if (index + 1 < length && text[index + 1] >= 0xDC00 && text[index + 1] <= 0xDFFF) { - const unsigned codepoint = 0x10000 + ((ch - 0xD800) << 10) + (text[++index] - 0xDC00); - direction = Get_Supplementary_Direction(codepoint); - } - } else if (ch < 0xDC00 || ch > 0xDFFF) { - WORD type = C2_NOTAPPLICABLE; - if (::GetStringTypeW(CT_CTYPE2, &ch, 1, &type)) { - if (type == C2_LEFTTORIGHT) { - direction = LeftToRight; - } else if (type == C2_RIGHTTOLEFT) { - direction = RightToLeft; - } + const unsigned high = codepoint - 0xD800; + const unsigned low = static_cast(text[++index]) - 0xDC00; + codepoint = 0x10000 + (high << 10) + low; + } else { + continue; } + } else if ((codepoint >= 0xDC00 && codepoint <= 0xDFFF) || codepoint > 0x10FFFF) { + continue; } + const Direction direction = Get_Direction(codepoint); if (direction != Neutral) { return direction == RightToLeft ? 1 : 0; } diff --git a/Dependencies/UnicodeBidi/unicodebidi.h b/Dependencies/UnicodeBidi/unicodebidi.h index 02239aa0282..2ec7e595a35 100644 --- a/Dependencies/UnicodeBidi/unicodebidi.h +++ b/Dependencies/UnicodeBidi/unicodebidi.h @@ -18,15 +18,11 @@ #pragma once -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include - namespace UnicodeBidi { -// Returns the UAX #9 paragraph base embedding level: 0 for LTR, 1 for RTL. -WORD Get_Paragraph_Base_Level(const WCHAR *text, int length); +// Returns the first paragraph's UAX #9 P2/P3 base level: 0 for LTR, 1 for RTL. +// Accepts UTF-16 code units or scalar values on platforms with 32-bit wchar_t. +unsigned short Get_Paragraph_Base_Level(const wchar_t *text, int length); } From 58925aa78bb9acb93ed4fe70d0175110b117687f Mon Sep 17 00:00:00 2001 From: Omar Aglan Date: Sat, 3 Oct 2026 17:23:18 +0300 Subject: [PATCH 3/4] chore(text): Address Unicode bidi review feedback --- Core/Libraries/Source/WWVegas/CMakeLists.txt | 2 +- Dependencies/UnicodeBidi/CMakeLists.txt | 12 ++++++------ Dependencies/UnicodeBidi/unicodebidi.cpp | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Core/Libraries/Source/WWVegas/CMakeLists.txt b/Core/Libraries/Source/WWVegas/CMakeLists.txt index 4933a33620a..b2ba43480fc 100644 --- a/Core/Libraries/Source/WWVegas/CMakeLists.txt +++ b/Core/Libraries/Source/WWVegas/CMakeLists.txt @@ -10,9 +10,9 @@ target_link_libraries(core_wwcommon INTERFACE d3d8lib deps_dbghelp deps_miles + deps_unicodebidi deps_usp10 stlport - unicodebidi ) target_include_directories(core_wwcommon INTERFACE diff --git a/Dependencies/UnicodeBidi/CMakeLists.txt b/Dependencies/UnicodeBidi/CMakeLists.txt index 48c4a2195d7..2f3aa78af94 100644 --- a/Dependencies/UnicodeBidi/CMakeLists.txt +++ b/Dependencies/UnicodeBidi/CMakeLists.txt @@ -4,15 +4,15 @@ set(UNICODEBIDI_SRC bididirection.inl ) -add_library(unicodebidi STATIC) -set_target_properties(unicodebidi PROPERTIES OUTPUT_NAME unicodebidi) +add_library(deps_unicodebidi STATIC) +set_target_properties(deps_unicodebidi PROPERTIES OUTPUT_NAME unicodebidi) -target_sources(unicodebidi PRIVATE ${UNICODEBIDI_SRC}) +target_sources(deps_unicodebidi PRIVATE ${UNICODEBIDI_SRC}) -target_include_directories(unicodebidi PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR} # include "unicodebidi.h" +target_include_directories(deps_unicodebidi PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} ) -target_link_libraries(unicodebidi PRIVATE +target_link_libraries(deps_unicodebidi PRIVATE deps_precompiled # Gives force included headers. Link early. ) diff --git a/Dependencies/UnicodeBidi/unicodebidi.cpp b/Dependencies/UnicodeBidi/unicodebidi.cpp index d437c3f02fa..61288761684 100644 --- a/Dependencies/UnicodeBidi/unicodebidi.cpp +++ b/Dependencies/UnicodeBidi/unicodebidi.cpp @@ -64,7 +64,7 @@ static bool Is_Paragraph_Separator(unsigned ch) ch == 0x0085 || ch == 0x2029; } -} +} // namespace namespace UnicodeBidi { @@ -119,4 +119,4 @@ unsigned short Get_Paragraph_Base_Level(const wchar_t *text, int length) return 0; } -} +} // namespace UnicodeBidi From 35b32d0e0ec55f86100a2fccf74227849701b057 Mon Sep 17 00:00:00 2001 From: Omar Aglan Date: Sat, 3 Oct 2026 17:49:18 +0300 Subject: [PATCH 4/4] chore(text): Use the default Unicode bidi library name --- Dependencies/UnicodeBidi/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/Dependencies/UnicodeBidi/CMakeLists.txt b/Dependencies/UnicodeBidi/CMakeLists.txt index 2f3aa78af94..4c19b24bf0f 100644 --- a/Dependencies/UnicodeBidi/CMakeLists.txt +++ b/Dependencies/UnicodeBidi/CMakeLists.txt @@ -5,7 +5,6 @@ set(UNICODEBIDI_SRC ) add_library(deps_unicodebidi STATIC) -set_target_properties(deps_unicodebidi PROPERTIES OUTPUT_NAME unicodebidi) target_sources(deps_unicodebidi PRIVATE ${UNICODEBIDI_SRC})