feat(icu4c): Add ICU4C support to UTF8 string operations

CMakeLists.txt

@@ -65,6 +65,7 @@ else()
 endif()
 
 include(cmake/config.cmake)
+include(cmake/icu.cmake)
 include(cmake/gamespy.cmake)
 include(cmake/lzhl.cmake)
 include(cmake/stb.cmake)
@@ -77,6 +78,7 @@ endif()
 
 add_subdirectory(Dependencies/Precompiled)
 add_subdirectory(Dependencies/Utility)
+add_subdirectory(Dependencies/ICU)
 add_subdirectory(Dependencies/Bink)
 add_subdirectory(Dependencies/Miles)
 if (WIN32)

Core/GameEngine/CMakeLists.txt

@@ -1212,6 +1212,7 @@ target_link_libraries(corei_gameengine_public INTERFACE
     core_browserdispatch
     dbghelploader
     #core_wwvegas
+    core_icu
     d3d8lib
     gamespy::gamespy
     stlport

Core/GameEngine/Source/Common/System/AsciiString.cpp

@@ -45,7 +45,7 @@
 #include "PreRTS.h"	// This must go first in EVERY cpp file in the GameEngine
 
 #include "Common/CriticalSection.h"
-#include "WWLib/utf8.h"
+#include "ICU/utf8.h"
 
 
 // -----------------------------------------------------
@@ -306,18 +306,27 @@ void AsciiString::translate(const UnicodeString& stringSrc)
 {
 	validate();
 	// TheSuperHackers @fix bobtista 02/04/2026 Implement UTF-8 conversion replacing 7-bit ASCII only implementation
+	// TheSuperHackers @bugfix CryoTheRenegade 04/08/2026 Convert wide text to UTF-8 with ICU4C.
 	const WideChar* src = stringSrc.str();
-	const size_t srcLen = wcslen(src);
+	const size_t srcLen = stringSrc.getLength();
 	const size_t dstLen = Wide_To_Utf8_Len(src, srcLen);
 	if (dstLen == 0)
 	{
 		clear();
 	}
+	else if (dstLen >= static_cast<size_t>(MAX_LEN))
+	{
+		DEBUG_ASSERTCRASH(false,
+			("AsciiString::translate exceeds max string length %d with required UTF-8 length %u",
+				MAX_LEN, static_cast<unsigned int>(dstLen)));
+		clear();
+	}
 	else
 	{
-		ensureUniqueBufferOfSize((Int)dstLen + 1, false, nullptr, nullptr);
+		ensureUniqueBufferOfSize(static_cast<Int>(dstLen) + 1, false, nullptr, nullptr);
 		Wide_To_Utf8(peek(), dstLen + 1, src, srcLen);
 	}
+
 	validate();
 }
 

Core/GameEngine/Source/Common/System/UnicodeString.cpp

@@ -45,7 +45,7 @@
 #include "PreRTS.h"	// This must go first in EVERY cpp file in the GameEngine
 
 #include "Common/CriticalSection.h"
-#include "WWLib/utf8.h"
+#include "ICU/utf8.h"
 
 
 // -----------------------------------------------------
@@ -222,31 +222,32 @@ void UnicodeString::translate(const AsciiString& stringSrc)
 	// TheSuperHackers @fix bobtista 02/04/2026 Convert UTF-8 to wide, replacing the 7-bit ASCII only
 	// implementation. Data that is not valid UTF-8 (e.g. legacy CP1252) falls back to a 1:1 byte cast
 	// to preserve the original characters instead of producing replacement characters.
+	// TheSuperHackers @bugfix CryoTheRenegade 04/08/2026 Convert UTF-8 to wide text with ICU4C.
 	const char* src = stringSrc.str();
-	const size_t srcLen = strlen(src);
+	const size_t srcLen = stringSrc.getLength();
 	const size_t dstLen = Utf8_To_Wide_Len(src, srcLen);
-	if (dstLen != UTF8_INVALID)
+	if (srcLen == 0)
 	{
-		if (dstLen == 0)
-		{
-			clear();
-		}
-		else
-		{
-			ensureUniqueBufferOfSize((Int)dstLen + 1, false, nullptr, nullptr);
-			Utf8_To_Wide(peek(), dstLen + 1, src, srcLen);
-		}
+		clear();
 	}
-	else
+	else if (dstLen == UTF8_INVALID)
 	{
-		ensureUniqueBufferOfSize((Int)srcLen + 1, false, nullptr, nullptr);
+		// Preserve legacy non-UTF-8 data with the original one-byte-to-one-wide-unit behavior.
+		ensureUniqueBufferOfSize(static_cast<Int>(srcLen) + 1, false, nullptr, nullptr);
 		WideChar* buf = peek();
 		for (size_t i = 0; i < srcLen; ++i)
 		{
-			buf[i] = (WideChar)(unsigned char)src[i];
+			buf[i] = static_cast<WideChar>(static_cast<unsigned char>(src[i]));
 		}
+
 		buf[srcLen] = 0;
 	}
+	else
+	{
+		ensureUniqueBufferOfSize(static_cast<Int>(dstLen) + 1, false, nullptr, nullptr);
+		Utf8_To_Wide(peek(), dstLen + 1, src, srcLen);
+	}
+
 	validate();
 }
 

Core/GameEngine/Source/GameNetwork/GameInfo.cpp

@@ -44,7 +44,7 @@
 #include "GameNetwork/LANAPI.h"						// for testing packet size
 #include "GameNetwork/LANAPICallbacks.h"	// for testing packet size
 #include "WWLib/strtok_r.h"
-#include "WWLib/utf8.h"
+#include "ICU/utf8.h"
 
 
 

Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp

@@ -28,62 +28,56 @@
 
 #include "PreRTS.h"	// This must go first in EVERY cpp file in the GameEngine
 
-#include "WWLib/utf8.h"
+#include "ICU/utf8.h"
 
 //-------------------------------------------------------------------------
 
 // TheSuperHackers @refactor bobtista 02/04/2026 Use WWLib UTF-8 functions instead of raw Win32 API calls
-std::wstring MultiByteToWideCharSingleLine( const char *orig )
+// TheSuperHackers @refactor CryoTheRenegade 04/08/2026 Use the shared ICU4C UTF conversion functions.
+std::wstring MultiByteToWideCharSingleLine( const char* orig )
 {
 	const size_t srcLen = strlen(orig);
 	const size_t dstLen = Utf8_To_Wide_Len(orig, srcLen);
 	if (dstLen == 0)
+	{
 		return std::wstring();
+	}
+
 	std::wstring ret;
 	if (dstLen == UTF8_INVALID)
 	{
-		// Not UTF-8. Fall back to a 1:1 byte cast so legacy data keeps its characters, matching
-		// UnicodeString::translate.
 		ret.resize(srcLen);
 		for (size_t i = 0; i < srcLen; ++i)
 		{
-			ret[i] = (WideChar)(unsigned char)orig[i];
+			ret[i] = static_cast<WideChar>(static_cast<unsigned char>(orig[i]));
 		}
 	}
 	else
 	{
 		ret.resize(dstLen);
 		Utf8_To_Wide(&ret[0], dstLen, orig, srcLen);
 	}
-	WideChar *c = nullptr;
-	do
-	{
-		c = wcschr(&ret[0], L'\n');
-		if (c)
-		{
-			*c = L' ';
-		}
-	}
-	while ( c != nullptr );
-	do
+
+	for (size_t i = 0; i < ret.size(); ++i)
 	{
-		c = wcschr(&ret[0], L'\r');
-		if (c)
+		if (ret[i] == L'\n' || ret[i] == L'\x0D')
 		{
-			*c = L' ';
+			ret[i] = L' ';
 		}
 	}
-	while ( c != nullptr );
 
 	return ret;
 }
 
-std::string WideCharStringToMultiByte( const WideChar *orig )
+std::string WideCharStringToMultiByte( const WideChar* orig )
 {
 	const size_t srcLen = wcslen(orig);
 	const size_t dstLen = Wide_To_Utf8_Len(orig, srcLen);
 	if (dstLen == 0)
+	{
 		return std::string();
+	}
+
 	std::string ret;
 	ret.resize(dstLen);
 	Wide_To_Utf8(&ret[0], dstLen, orig, srcLen);

Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt

@@ -126,8 +126,6 @@ set(WWLIB_SRC
     trim.cpp
     trim.h
     uarray.h
-    utf8.cpp
-    utf8.h
     vector.cpp
     Vector.h
     visualc.h
@@ -180,3 +178,7 @@ target_link_libraries(core_wwlib PRIVATE
     core_wwcommon
     corei_always
 )
+
+target_link_libraries(core_wwlib PUBLIC
+    core_icu
+)