Fix tests that rely on UTF-8 literals

and the lowercase function

src/filesys.cpp

@@ -701,8 +701,8 @@ bool PathStartsWith(const std::string &path, const std::string &prefix)
 				char pathchar = path[pathpos+len];
 				char prefixchar = prefix[prefixpos+len];
 				if(FILESYS_CASE_INSENSITIVE){
-					pathchar = tolower(pathchar);
-					prefixchar = tolower(prefixchar);
+					pathchar = my_tolower(pathchar);
+					prefixchar = my_tolower(prefixchar);
 				}
 				if(pathchar != prefixchar)
 					return false;

src/unittest/test_settings.cpp

@@ -51,7 +51,7 @@ void TestSettings::runTests(IGameDef *gamedef)
 ////////////////////////////////////////////////////////////////////////////////
 
 const char *TestSettings::config_text_before =
-	"leet = 1337\n"
+	u8"leet = 1337\n"
 	"leetleet = 13371337\n"
 	"leetleet_neg = -13371337\n"
 	"floaty_thing = 1.1\n"
@@ -77,7 +77,7 @@ const char *TestSettings::config_text_before =
 	"[dummy_eof_end_tag]\n";
 
 const std::string TestSettings::config_text_after =
-	"leet = 1337\n"
+	u8"leet = 1337\n"
 	"leetleet = 13371337\n"
 	"leetleet_neg = -13371337\n"
 	"floaty_thing = 1.1\n"
@@ -154,7 +154,7 @@ void TestSettings::testAllSettings()
 
 	// Not sure if 1.1 is an exact value as a float, but doesn't matter
 	UASSERT(fabs(s.getFloat("floaty_thing") - 1.1) < 0.001);
-	UASSERT(s.get("stringy_thing") == "asd /( ¤%&(/\" BLÖÄRP");
+	UASSERT(s.get("stringy_thing") == u8"asd /( ¤%&(/\" BLÖÄRP");
 	UASSERT(fabs(s.getV3F("coord").X - 1.0) < 0.001);
 	UASSERT(fabs(s.getV3F("coord").Y - 2.0) < 0.001);
 	UASSERT(fabs(s.getV3F("coord").Z - 4.5) < 0.001);

src/unittest/test_utilities.cpp

@@ -183,9 +183,11 @@ void TestUtilities::testWrapDegrees_0_360_v3f()
 
 void TestUtilities::testLowercase()
 {
-	UASSERT(lowercase("Foo bAR") == "foo bar");
-	UASSERT(lowercase("eeeeeeaaaaaaaaaaaààààà") == "eeeeeeaaaaaaaaaaaààààà");
-	UASSERT(lowercase("MINETEST-powa") == "minetest-powa");
+	UASSERTEQ(auto, lowercase("Foo bAR"), "foo bar");
+	UASSERTEQ(auto, lowercase(u8"eeeeeeaaaaaaaaaaaààààà"), u8"eeeeeeaaaaaaaaaaaààààà");
+	// intentionally won't handle Unicode, regardless of locale
+	UASSERTEQ(auto, lowercase(u8"ÜÜ"), u8"ÜÜ");
+	UASSERTEQ(auto, lowercase("MINETEST-powa"), "minetest-powa");
 }
 
 
@@ -309,18 +311,21 @@ void TestUtilities::testAsciiPrintableHelper()
 
 void TestUtilities::testUTF8()
 {
-	UASSERT(utf8_to_wide("¤") == L"¤");
+	UASSERT(utf8_to_wide(u8"¤") == L"¤");
 
-	UASSERT(wide_to_utf8(L"¤") == "¤");
+	UASSERTEQ(std::string, wide_to_utf8(L"¤"), u8"¤");
 
 	UASSERTEQ(std::string, wide_to_utf8(utf8_to_wide("")), "");
 	UASSERTEQ(std::string, wide_to_utf8(utf8_to_wide("the shovel dug a crumbly node!")),
 		"the shovel dug a crumbly node!");
-	UASSERTEQ(std::string, wide_to_utf8(utf8_to_wide("-ä-")),
-		"-ä-");
-	UASSERTEQ(std::string, wide_to_utf8(utf8_to_wide("-\xF0\xA0\x80\x8B-")),
-		"-\xF0\xA0\x80\x8B-");
-
+	UASSERTEQ(std::string, wide_to_utf8(utf8_to_wide(u8"-ä-")),
+		u8"-ä-");
+	UASSERTEQ(std::string, wide_to_utf8(utf8_to_wide(u8"-\U0002000b-")),
+		u8"-\U0002000b-");
+	if constexpr (sizeof(wchar_t) == 4) {
+		const auto *literal = U"-\U0002000b-";
+		UASSERT(utf8_to_wide(u8"-\U0002000b-") == reinterpret_cast<const wchar_t*>(literal));
+	}
 }
 
 void TestUtilities::testRemoveEscapes()
@@ -643,7 +648,7 @@ void TestUtilities::testSanitizeDirName()
 	UASSERTEQ(auto, sanitizeDirName(" a ", "~"), "_a_");
 	UASSERTEQ(auto, sanitizeDirName("COM1", "~"), "~COM1");
 	UASSERTEQ(auto, sanitizeDirName("COM1", ":"), "_COM1");
-	UASSERTEQ(auto, sanitizeDirName("cOm\u00B2", "~"), "~cOm\u00B2");
+	UASSERTEQ(auto, sanitizeDirName(u8"cOm\u00B2", "~"), u8"~cOm\u00B2");
 	UASSERTEQ(auto, sanitizeDirName("cOnIn$", "~"), "~cOnIn$");
 	UASSERTEQ(auto, sanitizeDirName(" cOnIn$ ", "~"), "_cOnIn$_");
 }

src/util/string.h

@@ -101,6 +101,21 @@ bool parseColorString(const std::string &value, video::SColor &color, bool quiet
 		unsigned char default_alpha = 0xff);
 std::string encodeHexColorString(video::SColor color);
 
+/**
+ * Converts a letter to lowercase, with safe handling of the char type and non-ASCII.
+ * @param c input letter
+ * @returns same letter but lowercase
+*/
+inline char my_tolower(char c)
+{
+	// By design this function cannot handle any Unicode (codepoints don't fit into char),
+	// but make sure to pass it through unchanged.
+	// tolower() can mangle it if the POSIX locale is not UTF-8.
+	if (static_cast<unsigned char>(c) > 0x7f)
+		return c;
+	// toupper(3): "If the argument c is of type char, it must be cast to unsigned char"
+	return tolower(static_cast<unsigned char>(c));
+}
 
 /**
  * Returns a copy of \p str with spaces inserted at the right hand side to ensure
@@ -173,7 +188,7 @@ inline bool str_equal(std::basic_string_view<T> s1,
 		return false;
 
 	for (size_t i = 0; i < s1.size(); ++i)
-		if(tolower(s1[i]) != tolower(s2[i]))
+		if (my_tolower(s1[i]) != my_tolower(s2[i]))
 			return false;
 
 	return true;
@@ -212,7 +227,7 @@ inline bool str_starts_with(std::basic_string_view<T> str,
 		return str.compare(0, prefix.size(), prefix) == 0;
 
 	for (size_t i = 0; i < prefix.size(); ++i)
-		if (tolower(str[i]) != tolower(prefix[i]))
+		if (my_tolower(str[i]) != my_tolower(prefix[i]))
 			return false;
 	return true;
 }
@@ -253,7 +268,7 @@ inline bool str_ends_with(std::basic_string_view<T> str,
 		return str.compare(start, suffix.size(), suffix) == 0;
 
 	for (size_t i = 0; i < suffix.size(); ++i)
-		if (tolower(str[start + i]) != tolower(suffix[i]))
+		if (my_tolower(str[start + i]) != my_tolower(suffix[i]))
 			return false;
 	return true;
 }
@@ -305,7 +320,7 @@ inline std::string lowercase(std::string_view str)
 	std::string s2;
 	s2.resize(str.size());
 	for (size_t i = 0; i < str.size(); i++)
-		s2[i] = tolower(str[i]);
+		s2[i] = my_tolower(str[i]);
 	return s2;
 }