rename [w]string_to_[w]string UTF8

add safer/more portable/less dependent on locale implementation of
wchar_t <-> UTF8

This was SVN commit r7185.

source/gui/CGUI.cpp

@@ -1557,7 +1557,7 @@ void CGUI::Xeromyces_ReadImage(XMBElement Element, CXeromyces* pFile, CGUISprite
 
 		if (attr_name == "texture")
 		{
-			image.m_TextureName = VfsPath(L"art/textures/ui")/wstring_from_string(attr_value);
+			image.m_TextureName = VfsPath(L"art/textures/ui")/wstring_from_UTF8(attr_value);
 		}
 		else
 		if (attr_name == "size")

source/gui/GUIRenderer.cpp

@@ -370,7 +370,7 @@ void GUIRenderer::UpdateDrawCallCache(DrawCalls &Calls, const CStr& SpriteName,
 		if (SpriteName.substr(0, 10) == "stretched:")
 		{
 			SGUIImage Image;
-			Image.m_TextureName = VfsPath(L"art/textures/ui")/wstring_from_string(SpriteName.substr(10));
+			Image.m_TextureName = VfsPath(L"art/textures/ui")/wstring_from_UTF8(SpriteName.substr(10));
 			CClientArea ca("0 0 100% 100%");
 			Image.m_Size = ca;
 			Image.m_TextureSize = ca;

source/lib/file/archive/archive_zip.cpp

@@ -27,7 +27,7 @@
 
 #include "lib/bits.h"
 #include "lib/byte_order.h"
-#include "lib/wchar.h"	// wstring_from_string
+#include "lib/wchar.h"	// wstring_from_UTF8
 #include "lib/fat_time.h"
 #include "lib/path_util.h"
 #include "lib/allocators/pool.h"
@@ -135,7 +135,7 @@ public:
 	{
 		const size_t length = (size_t)read_le16(&m_fn_len);
 		const char* pathname = (const char*)this + sizeof(CDFH); // not 0-terminated!
-		return wstring_from_string(std::string(pathname, length));
+		return wstring_from_UTF8(std::string(pathname, length));
 	}
 
 	off_t HeaderOffset() const

source/lib/file/file_system.cpp

@@ -23,7 +23,7 @@
 #include <string>
 
 #include "lib/path_util.h"
-#include "lib/wchar.h"	// wstring_from_string
+#include "lib/wchar.h"	// wstring_from_UTF8
 #include "lib/posix/posix_filesystem.h"
 
 
@@ -67,7 +67,7 @@ LibError GetDirectoryEntries(const fs::wpath& path, FileInfos* files, DirectoryN
 			return LibError_from_errno();
 		}
 
-		const std::wstring name = wstring_from_string(osEnt->d_name);
+		const std::wstring name = wstring_from_UTF8(osEnt->d_name);
 		RETURN_ERR(path_component_validate(name.c_str()));
 
 		// get file information (mode, size, mtime)

source/lib/path_util.cpp

@@ -128,10 +128,10 @@ const wchar_t* path_name_only(const wchar_t* path)
 
 fs::wpath wpath_from_path(const fs::path& pathname)
 {
-	return wstring_from_string(pathname.string());
+	return wstring_from_UTF8(pathname.string());
 }
 
 fs::path path_from_wpath(const fs::wpath& pathname)
 {
-	return string_from_wstring(pathname.string());
+	return UTF8_from_wstring(pathname.string());
 }

source/lib/secure_crt.cpp

@@ -257,8 +257,8 @@ int tsprintf_s(tchar* buf, size_t max_chars, const tchar* fmt, ...)
 errno_t _wfopen_s(FILE** pfile, const wchar_t* filename, const wchar_t* mode)
 {
 	*pfile = NULL;
-	const std::string filename_c = string_from_wstring(filename);
-	const std::string mode_c = string_from_wstring(mode);
+	const std::string filename_c = UTF8_from_wstring(filename);
+	const std::string mode_c = UTF8_from_wstring(mode);
 	return fopen_s(pfile, filename_c.c_str(), mode_c.c_str());
 }
 

source/lib/sysdep/os/win/wseh.cpp

@@ -281,7 +281,7 @@ long __stdcall wseh_ExceptionFilter(struct _EXCEPTION_POINTERS* ep)
 	if(ep->ExceptionRecord->ExceptionFlags & EXCEPTION_NONCONTINUABLE)
 		flags = DE_NO_CONTINUE;
 	const wchar_t* const lastFuncToSkip = WIDEN(STRINGIZE(DECORATED_NAME(wseh_ExceptionFilter)));
-	ErrorReaction er = debug_DisplayError(message, flags, ep->ContextRecord, lastFuncToSkip, file,line,string_from_wstring(func).c_str(), 0);
+	ErrorReaction er = debug_DisplayError(message, flags, ep->ContextRecord, lastFuncToSkip, file,line,UTF8_from_wstring(func).c_str(), 0);
 	debug_assert(er == ER_CONTINUE);	// nothing else possible
 
 	// invoke the Win32 default handler - it calls ExitProcess for

source/lib/sysdep/tests/test_sysdep.h

@@ -108,7 +108,7 @@ public:
 		sprintf_s(root, ARRAY_SIZE(root), "%s/pyrogenesis-test-sysdep-XXXXXX", tmpdir);
 		TS_ASSERT(mkdtemp(root));
 		std::string rootstr(root);
-		std::wstring rootstrw(wstring_from_string(rootstr));
+		std::wstring rootstrw(wstring_from_UTF8(rootstr));
 
 		const char* dirs[] = {
 			"/example",

source/lib/wchar.cpp

@@ -1,37 +1,198 @@
 /* Copyright (C) 2009 Wildfire Games.
- * This file is part of 0 A.D.
- *
- * 0 A.D. is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 2 of the License, or
- * (at your option) any later version.
- *
- * 0 A.D. is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with 0 A.D.  If not, see <http://www.gnu.org/licenses/>.
- */
+* This file is part of 0 A.D.
+*
+* 0 A.D. is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 2 of the License, or
+* (at your option) any later version.
+*
+* 0 A.D. is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with 0 A.D.  If not, see <http://www.gnu.org/licenses/>.
+*/
 
 #include "precompiled.h"
 #include "wchar.h"
 
+// adapted from http://unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
+// which bears the following notice:
+/*
+* Copyright 2001-2004 Unicode, Inc.
+* 
+* Disclaimer
+* 
+* This source code is provided as is by Unicode, Inc. No claims are
+* made as to fitness for any particular purpose. No warranties of any
+* kind are expressed or implied. The recipient agrees to determine
+* applicability of information provided. If this file has been
+* purchased on magnetic or optical media from Unicode, Inc., the
+* sole remedy for any claim will be exchange of defective media
+* within 90 days of receipt.
+* 
+* Limitations on Rights to Redistribute This Code
+* 
+* Unicode, Inc. hereby grants the right to freely use the information
+* supplied in this file in the creation of products supporting the
+* Unicode Standard, and to make copies of this file in any form
+* for internal or external distribution as long as this notice
+* remains attached.
+*/
 
-std::wstring wstring_from_string(const std::string& s)
+// design rationale:
+// - to cope with wchar_t differences between VC (UTF-16) and
+//   GCC (UCS-4), we only allow codepoints in the BMP.
+//   UTF-8 encodings are therefore no longer than 3 bytes.
+// - surrogates are disabled because variable-length strings
+//   violate the purpose of using wchar_t instead of UTF-8.
+
+// this implementation survives http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
+
+// (must be unsigned to avoid sign extension)
+typedef u8 UTF8;
+typedef u32 UTF32;
+
+
+static UTF32 ReplaceIfInvalid(UTF32 u)
 {
-	wchar_t buf[1000];
-	const size_t numConverted = mbstowcs(buf, s.c_str(), ARRAY_SIZE(buf)-1);
-	debug_assert(numConverted < ARRAY_SIZE(buf));
-	return buf;
+	struct IsValid
+	{
+		bool operator()(UTF32 u) const
+		{
+			// disallow surrogates
+			if(0xDC00ul <= u && u <= 0xDFFFul)
+				return false;
+			// greater: UTF-16 representation would require surrogates
+			// equal: permanently unassigned codepoint, may correspond to WEOF
+			// (which raises errors when used in VC's swprintf)
+			if(u >= 0xFFFFul)	
+				return false;
+			return true;
+		}
+	};
+	return IsValid()(u)? u : 0xFFFD;
 }
 
 
-std::string string_from_wstring(const std::wstring& s)
+class UTF8Codec
 {
-	char buf[1000];
-	const size_t numConverted = wcstombs(buf, s.c_str(), ARRAY_SIZE(buf)-1);
-	debug_assert(numConverted < ARRAY_SIZE(buf));
-	return buf;
+public:
+	static void Encode(UTF32 u, UTF8*& dstPos)
+	{
+		const size_t size = Size(u);
+		static const UTF8 firstByteMarks[1+3] = { 0, 0x00, 0xC0, 0xE0 };
+		*dstPos++ = (UTF8)(u | firstByteMarks[size]);
+		for(size_t i = 1; i < size; i++)
+		{
+			*dstPos++ = (UTF8)((u|0x80u) & 0xBFu);
+			u >>= 6;
+		}
+	}
+
+	static bool Decode(const UTF8*& srcPos, const UTF8* const srcEnd, UTF32& u)
+	{
+		const size_t size = SizeFromFirstByte(*srcPos);
+		if(!IsValid(srcPos, size, srcEnd))
+			return false;
+
+		u = 0;
+		for(size_t i = 0; i < size-1; i++)
+		{
+			u += UTF32(*srcPos++);
+			u <<= 6;
+		}
+		u += UTF32(*srcPos++);
+
+		static const UTF32 offsets[1+3] = { 0, 0x00000000ul, 0x00003080ul, 0x000E2080ul };
+		u -= offsets[size];
+
+		return true;
+	}
+
+private:
+	static inline size_t Size(UTF32 u)
+	{
+		if(u < 0x80)
+			return 1;
+		if(u < 0x800)
+			return 2;
+		// ReplaceIfInvalid ensures > 3 byte encodings are never used.
+		return 3;
+	}
+
+	static inline size_t SizeFromFirstByte(UTF8 firstByte)
+	{
+		if(firstByte < 0xC0)
+			return 1;
+		if(firstByte < 0xE0)
+			return 2;
+		// IsValid rejects firstByte values that would cause > 3 byte encodings.
+		return 3;
+	}
+
+	// c.f. Unicode 3.1 Table 3-7
+	// @param size obtained via SizeFromFirstByte (our caller also uses it)
+	static bool IsValid(const UTF8* const src, size_t size, const UTF8* const srcEnd)
+	{
+		if(src+size > srcEnd)	// not enough data
+			return false;
+
+		if(src[0] < 0x80)
+			return true;
+		if(!(0xC2 <= src[0] && src[0] <= 0xEF))
+			return false;
+
+		// special cases (stricter than the loop)
+		if(src[0] == 0xE0 && src[1] < 0xA0)
+			return false;
+		if(src[0] == 0xED && src[1] > 0x9F)
+			return false;
+
+		for(size_t i = 1; i < size; i++)
+		{
+			if(!(0x80 <= src[i] && src[i] <= 0xBF))
+				return false;
+		}
+
+		return true;
+	}
+};
+
+
+//-----------------------------------------------------------------------------
+
+std::string UTF8_from_wstring(const std::wstring& src)
+{
+	std::string dst(src.size()*3, ' ');	// see UTF8Codec::Size
+	UTF8* dstPos = (UTF8*)&dst[0];
+	for(size_t i = 0; i < src.size(); i++)
+	{
+		const UTF32 u = ReplaceIfInvalid(UTF32(src[i]));
+		UTF8Codec::Encode(u, dstPos);
+	}
+	dst.resize(dstPos - (UTF8*)&dst[0]);
+	return dst;
+}
+
+
+std::wstring wstring_from_UTF8(const std::string& src)
+{
+	std::wstring dst;
+	dst.reserve(src.size());
+	const UTF8* srcPos = (const UTF8*)src.data();
+	const UTF8* const srcEnd = srcPos + src.size();
+	while(srcPos < srcEnd)
+	{
+		UTF32 u;
+		if(!UTF8Codec::Decode(srcPos, srcEnd, u))
+		{
+			debug_assert(0);
+			return L"(wstring_from_UTF8: invalid input)";
+		}
+		dst.push_back((wchar_t)ReplaceIfInvalid(u));
+	}
+	return dst;
 }

source/lib/wchar.h

@@ -18,7 +18,7 @@
 #ifndef INCLUDED_WCHAR
 #define INCLUDED_WCHAR
 
-LIB_API std::wstring wstring_from_string(const std::string& s);
-LIB_API std::string string_from_wstring(const std::wstring& s);
+LIB_API std::wstring wstring_from_UTF8(const std::string& s);
+LIB_API std::string UTF8_from_wstring(const std::wstring& s);
 
 #endif	// #ifndef INCLUDED_WCHAR

source/network/NetLog.cpp

@@ -375,7 +375,7 @@ void CNetLogFileSink::OpenFile( const fs::wpath& fileName, bool append )
 	if ( m_File.is_open() ) m_File.close();
 
 	// Open the file and log start
-	m_File.open( string_from_wstring(fileName.string()).c_str(), append ? std::ios::app : std::ios::out );
+	m_File.open( UTF8_from_wstring(fileName.string()).c_str(), append ? std::ios::app : std::ios::out );
 	if ( !m_File.is_open() )
 	{
 		// throw std::ios_base::failure

source/ps/CLogger.cpp

@@ -56,10 +56,10 @@ const wchar_t* html_footer = L"";
 CLogger::CLogger()
 {
 	fs::wpath mainlogPath(psLogDir()/L"mainlog.html");
-	m_MainLog = new std::wofstream(string_from_wstring(mainlogPath.string()).c_str(), std::ofstream::out | std::ofstream::trunc);
+	m_MainLog = new std::wofstream(UTF8_from_wstring(mainlogPath.string()).c_str(), std::ofstream::out | std::ofstream::trunc);
 
 	fs::wpath interestinglogPath(psLogDir()/L"interestinglog.html");
-	m_InterestingLog = new std::wofstream(string_from_wstring(interestinglogPath.string()).c_str(), std::ofstream::out | std::ofstream::trunc);
+	m_InterestingLog = new std::wofstream(UTF8_from_wstring(interestinglogPath.string()).c_str(), std::ofstream::out | std::ofstream::trunc);
 
 	m_OwnsStreams = true;
 	m_UseDebugPrintf = true;

source/ps/GameSetup/Config.cpp

@@ -74,7 +74,7 @@ CStr g_AutostartMap = "";
 
 static void LoadProfile( const CStr& profile )
 {
-	VfsPath path = VfsPath(L"profiles") / wstring_from_string(profile);
+	VfsPath path = VfsPath(L"profiles") / wstring_from_UTF8(profile);
 
 	VfsPath configFilename = path / L"settings/user.cfg";
 	g_ConfigDB.SetConfigFile(CFG_USER, true, configFilename.string().c_str());

source/ps/GameSetup/GameSetup.cpp

@@ -567,7 +567,7 @@ static void InitVfs(const CmdLineArgs& args)
 	{
 		size_t priority = i;
 		int flags = VFS_MOUNT_WATCH|VFS_MOUNT_ARCHIVABLE;
-		std::wstring modName (wstring_from_string(mods[i]));
+		std::wstring modName (wstring_from_UTF8(mods[i]));
 		g_VFS->Mount(L"", AddSlash(modLoosePath/modName), flags, priority);
 		g_VFS->Mount(L"", AddSlash(modArchivePath/modName), flags, priority);
 	}

source/ps/GameSetup/Paths.cpp

@@ -51,7 +51,7 @@ Paths::Paths(const CmdLineArgs& args)
 #else
 		const char* envHome = getenv("HOME");
 		debug_assert(envHome);
-		const fs::wpath home(wstring_from_string(envHome));
+		const fs::wpath home(wstring_from_UTF8(envHome));
 		m_data = AddSlash(XDG_Path("XDG_DATA_HOME", home, home/L".local/share/")/subdirectoryName);
 		m_config = AddSlash(XDG_Path("XDG_CONFIG_HOME", home, home/L".config/")/subdirectoryName);
 		m_cache = AddSlash(XDG_Path("XDG_CACHE_HOME", home, home/L".cache/")/subdirectoryName);
@@ -92,8 +92,8 @@ Paths::Paths(const CmdLineArgs& args)
 	if(path)
 	{
 		if(path[0] != '/')	// relative to $HOME
-			return AddSlash(home/wstring_from_string(path));
-		return AddSlash(fs::wpath(wstring_from_string(path)));
+			return AddSlash(home/wstring_from_UTF8(path));
+		return AddSlash(fs::wpath(wstring_from_UTF8(path)));
 	}
 	return AddSlash(defaultPath);
 }

source/ps/ProfileViewer.cpp

@@ -432,7 +432,7 @@ void CProfileViewer::SaveToFile()
 		// Open the file. (It will be closed when the CProfileViewer
 		// destructor is called.)
 		fs::wpath path(psLogDir()/L"profile.txt");
-		m->outputStream.open(string_from_wstring(path.string()).c_str(), std::ofstream::out | std::ofstream::trunc);
+		m->outputStream.open(UTF8_from_wstring(path.string()).c_str(), std::ofstream::out | std::ofstream::trunc);
 
 		if (m->outputStream.fail())
 		{

source/ps/i18n.cpp

@@ -59,7 +59,7 @@ bool I18n::LoadLanguage(const char* name)
 	// Automatically delete the pointer when returning early
 	std::auto_ptr<CLocale_interface> locale (locale_ptr);
 
-	VfsPath dirname = AddSlash(VfsPath(L"language")/wstring_from_string(name));
+	VfsPath dirname = AddSlash(VfsPath(L"language")/wstring_from_UTF8(name));
 
 	// Open *.lng with LoadStrings
 	VfsPaths pathnames;