feat(string): add UTF-8 string conversion and validation functions

Core/GameEngine/Source/Common/System/AsciiString.cpp

@@ -45,6 +45,7 @@
 #include "PreRTS.h"	// This must go first in EVERY cpp file in the GameEngine
 
 #include "Common/CriticalSection.h"
+#include "WWLib/utf8.h"
 
 
 // -----------------------------------------------------
@@ -272,11 +273,19 @@ char*  AsciiString::getBufferForRead(Int len)
 void AsciiString::translate(const UnicodeString& stringSrc)
 {
 	validate();
-	/// @todo srj put in a real translation here; this will only work for 7-bit ascii
-	clear();
-	Int len = stringSrc.getLength();
-	for (Int i = 0; i < len; i++)
-		concat((char)stringSrc.getCharAt(i));
+	// TheSuperHackers @fix bobtista 02/04/2026 Implement UTF-8 conversion replacing 7-bit ASCII only implementation
+	const WideChar* src = stringSrc.str();
+	const size_t srcLen = wcslen(src);
+	const size_t len = Wide_To_Utf8_Len(src, srcLen);
+	if (len == 0)
+	{
+		clear();
+	}
+	else
+	{
+		ensureUniqueBufferOfSize((Int)len + 1, false, nullptr, nullptr);
+		Wide_To_Utf8(peek(), len + 1, src, srcLen);
+	}
 	validate();
 }
 

Core/GameEngine/Source/Common/System/UnicodeString.cpp

@@ -45,6 +45,7 @@
 #include "PreRTS.h"	// This must go first in EVERY cpp file in the GameEngine
 
 #include "Common/CriticalSection.h"
+#include "WWLib/utf8.h"
 
 
 // -----------------------------------------------------
@@ -221,11 +222,31 @@ WideChar* UnicodeString::getBufferForRead(Int len)
 void UnicodeString::translate(const AsciiString& stringSrc)
 {
 	validate();
-	/// @todo srj put in a real translation here; this will only work for 7-bit ascii
-	clear();
-	Int len = stringSrc.getLength();
-	for (Int i = 0; i < len; i++)
-		concat((WideChar)stringSrc.getCharAt(i));
+	// TheSuperHackers @fix bobtista 02/04/2026 Convert UTF-8 to wide, replacing the 7-bit ASCII only
+	// implementation. Data that is not valid UTF-8 (e.g. legacy CP1252) falls back to a 1:1 byte cast
+	// to preserve the original characters instead of producing replacement characters.
+	const char* src = stringSrc.str();
+	const size_t srcLen = strlen(src);
+	const size_t len = Utf8_To_Wide_Len(src, srcLen);
+	if (srcLen == 0)
+	{
+		clear();
+	}
+	else if (len == UTF8_INVALID)
+	{
+		ensureUniqueBufferOfSize((Int)srcLen + 1, false, nullptr, nullptr);
+		WideChar* buf = peek();
+		for (size_t i = 0; i < srcLen; ++i)
+		{
+			buf[i] = (WideChar)(unsigned char)src[i];
+		}
+		buf[srcLen] = 0;
+	}
+	else
+	{
+		ensureUniqueBufferOfSize((Int)len + 1, false, nullptr, nullptr);
+		Utf8_To_Wide(peek(), len + 1, src, srcLen);
+	}
 	validate();
 }
 

Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp

@@ -28,18 +28,37 @@
 
 #include "PreRTS.h"	// This must go first in EVERY cpp file in the GameEngine
 
+#include "WWLib/utf8.h"
+
 //-------------------------------------------------------------------------
 
+// TheSuperHackers @refactor bobtista 02/04/2026 Use WWLib UTF-8 functions instead of raw Win32 API calls
 std::wstring MultiByteToWideCharSingleLine( const char *orig )
 {
-	Int len = strlen(orig);
-	WideChar *dest = NEW WideChar[len+1];
-
-	MultiByteToWideChar(CP_UTF8, 0, orig, -1, dest, len);
+	const size_t srcLen = strlen(orig);
+	const size_t len = Utf8_To_Wide_Len(orig, srcLen);
+	if (len == 0)
+		return std::wstring();
+	std::wstring ret;
+	if (len == UTF8_INVALID)
+	{
+		// Not UTF-8. Fall back to a 1:1 byte cast so legacy data keeps its characters, matching
+		// UnicodeString::translate.
+		ret.resize(srcLen);
+		for (size_t i = 0; i < srcLen; ++i)
+		{
+			ret[i] = (WideChar)(unsigned char)orig[i];
+		}
+	}
+	else
+	{
+		ret.resize(len);
+		Utf8_To_Wide(&ret[0], len, orig, srcLen);
+	}
 	WideChar *c = nullptr;
 	do
 	{
-		c = wcschr(dest, L'\n');
+		c = wcschr(&ret[0], L'\n');
 		if (c)
 		{
 			*c = L' ';
@@ -48,32 +67,26 @@ std::wstring MultiByteToWideCharSingleLine( const char *orig )
 	while ( c != nullptr );
 	do
 	{
-		c = wcschr(dest, L'\r');
+		c = wcschr(&ret[0], L'\r');
 		if (c)
 		{
 			*c = L' ';
 		}
 	}
 	while ( c != nullptr );
 
-	dest[len] = 0;
-	std::wstring ret = dest;
-	delete[] dest;
 	return ret;
 }
 
 std::string WideCharStringToMultiByte( const WideChar *orig )
 {
+	const size_t srcLen = wcslen(orig);
+	const size_t len = Wide_To_Utf8_Len(orig, srcLen);
+	if (len == 0)
+		return std::string();
 	std::string ret;
-	Int len = WideCharToMultiByte( CP_UTF8, 0, orig, wcslen(orig), nullptr, 0, nullptr, nullptr ) + 1;
-	if (len > 0)
-	{
-		char *dest = NEW char[len];
-		WideCharToMultiByte( CP_UTF8, 0, orig, -1, dest, len, nullptr, nullptr );
-		dest[len-1] = 0;
-		ret = dest;
-		delete[] dest;
-	}
+	ret.resize(len);
+	Wide_To_Utf8(&ret[0], len, orig, srcLen);
 	return ret;
 }
 

Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt

@@ -133,6 +133,8 @@ set(WWLIB_SRC
     trim.cpp
     trim.h
     uarray.h
+    utf8.cpp
+    utf8.h
     vector.cpp
     Vector.h
     visualc.h