Merge e0844bff7a into 7e4462e0ac

.gitignore

@@ -114,6 +114,7 @@ CMakeDoxy*
 compile_commands.json
 *.apk
 *.zip
+src/util/languagemap.h
 # Visual Studio
 *.vcxproj*
 *.sln

android/app/src/main/java/net/minetest/minetest/GameActivity.java

@@ -24,6 +24,7 @@ import android.app.NativeActivity;
 import android.content.Intent;
 import android.net.Uri;
 import android.os.Bundle;
+import android.os.LocaleList;
 import android.text.InputType;
 import android.util.Log;
 import android.view.KeyEvent;
@@ -243,26 +244,40 @@ public class GameActivity extends NativeActivity {
 	}
 
 	public String getLanguage() {
-		String langCode = Locale.getDefault().getLanguage();
+		LocaleList locales = LocaleList.getAdjustedDefault();
+		StringBuilder listString = new StringBuilder();
+		for (int i = 0; i < locales.size(); i++) {
+			Locale lang = locales.get(i);
+			String langCode = lang.getLanguage();
 
-		// getLanguage() still uses old language codes to preserve compatibility.
-		// List of code changes in ISO 639-2:
-		// https://www.loc.gov/standards/iso639-2/php/code_changes.php
-		switch (langCode) {
-			case "in":
-				langCode = "id"; // Indonesian
-				break;
-			case "iw":
-				langCode = "he"; // Hebrew
-				break;
-			case "ji":
-				langCode = "yi"; // Yiddish
-				break;
-			case "jw":
-				langCode = "jv"; // Javanese
-				break;
+			// getLanguage() still uses old language codes to preserve compatibility.
+			// List of code changes in ISO 639-2:
+			// https://www.loc.gov/standards/iso639-2/php/code_changes.php
+			switch (langCode) {
+				case "in":
+					langCode = "id"; // Indonesian
+					break;
+				case "iw":
+					langCode = "he"; // Hebrew
+					break;
+				case "ji":
+					langCode = "yi"; // Yiddish
+					break;
+				case "jw":
+					langCode = "jv"; // Javanese
+					break;
+			}
+			if (i > 0) {
+				listString.append(':');
+			}
+			listString.append(langCode);
+
+			String countryCode = lang.getCountry();
+			if (!countryCode.isEmpty()) {
+				listString.append('_');
+				listString.append(countryCode);
+			}
 		}
-
-		return langCode;
+		return listString.toString();
 	}
 }

builtin/mainmenu/content/dlg_contentstore.lua

@@ -644,7 +644,8 @@ local function fetch_pkgs()
 	local languages
 	local current_language = core.get_language()
 	if current_language ~= "" then
-		languages = { current_language, "en;q=0.8" }
+		languages = current_language:split(":")
+		table.insert(languages, "en;q=0.8")
 	else
 		languages = { "en" }
 	end

builtin/mainmenu/settings/components.lua

@@ -403,5 +403,151 @@ end
 make.noise_params_2d = make_noise_params
 make.noise_params_3d = make_noise_params
 
+-- These must not be translated, as they need to show in the local
+-- language no matter the user's current language.
+-- This list must be kept in sync with src/unsupported_language_list.txt.
+local language_option_labels = core.language_names
+
+local language_options = {}
+for k in pairs(language_option_labels) do
+	table.insert(language_options, k)
+end
+table.sort(language_options)
+
+local language_dropdown = {}
+for idx, langcode in ipairs(language_options) do
+	local langname = language_option_labels[langcode]
+	if langname == "" then
+		langname = langcode
+	else
+		langname = ("%s [%s]"):format(langname, langcode)
+	end
+	language_dropdown[idx] = core.formspec_escape(langname)
+	language_options[langcode] = idx
+end
+language_dropdown = table.concat(language_dropdown, ",")
+
+function make.language_list(setting)
+	local selection_list = {}
+	return {
+		info_text = setting.comment,
+		setting = setting,
+		get_formspec = function(self, avail_w)
+			local fs = {}
+			local height = 0.8
+			selection_list = string.split(core.settings:get(setting.name) or setting.default, ":")
+			self.resettable = core.settings:has(setting.name)
+
+			table.insert(fs, ("label[0,0.1;%s]"):format(get_label(setting)))
+			-- TODO: Change the label to "Use system language: $1" when #14379 is merged
+			table.insert(fs, ("checkbox[0,0.55;%s;%s;%s]"):format(
+					setting.name, fgettext("Use system language"), tostring(#selection_list == 0)))
+
+			-- Note that the "Remove" button is added implicitly and only when the move buttons are present.
+			-- If the selection list includes only one language, removing it implies using the system language.
+			local move_columns = 2 -- Move up|Move down
+			if #selection_list == 1 then
+				move_columns = 0
+			elseif #selection_list == 2 then
+				move_columns = 1 -- Move (i.e. swap order with the other entry)
+			end
+			local dropdown_width = avail_w
+			if move_columns > 0 then
+				dropdown_width = avail_w - 0.9*(move_columns+1)
+			end
+
+			for idx, lang in ipairs(selection_list) do
+				local selid = language_options[lang]
+				local dropdown_entries = language_dropdown
+				if not selid then
+					dropdown_entries = dropdown_entries .. "," ..
+							fgettext("Other language: $1", lang)
+					selid = #language_options+1
+				end
+				table.insert(fs, ("dropdown[0,%f;%f,0.8;sel_%d_%s;%s;%d;true]"):format(
+						height, dropdown_width, idx, setting.name, dropdown_entries, selid))
+				if move_columns > 0 then
+					table.insert(fs, ("image_button[%f,%f;0.8,0.8;%s;rem_%d_%s;]"):format(
+							avail_w-0.8, height,
+							core.formspec_escape(defaulttexturedir .. "clear.png"), idx, setting.name))
+					local move_button_x = avail_w-0.8-0.9*move_columns
+					local move_button_width = 0.8
+					if move_columns == 2 and (idx == 1 or idx == #selection_list) then
+						move_button_width = 1.7
+					end
+					if idx < #selection_list then
+						table.insert(fs, ("image_button[%f,%f;%f,0.8;%s;mdn_%d_%s;]"):format(
+								move_button_x, height, move_button_width,
+								core.formspec_escape(defaulttexturedir .. "down_icon.png"), idx, setting.name))
+						move_button_x = move_button_x + 0.9
+					end
+					if idx > 1 then
+						table.insert(fs, ("image_button[%f,%f;%f,0.8;%s;mup_%d_%s;]"):format(
+								move_button_x, height, move_button_width,
+								core.formspec_escape(defaulttexturedir .. "up_icon.png"), idx, setting.name))
+					end
+				end
+				height = height+0.9
+			end
+			if #selection_list > 0 then
+				local dropdown_entries = language_dropdown .. "," ..
+						fgettext("Select language to add to list")
+				local selid = #language_options+1
+				table.insert(fs, ("dropdown[0,%f;%f,0.8;sel_%d_%s;%s;%d;true]"):format(
+						height, dropdown_width, #selection_list+1, setting.name, dropdown_entries, selid))
+				height = height+0.9
+			end
+
+			return table.concat(fs), height
+		end,
+		on_submit = function(self, fields)
+			local old_value = core.settings:get(setting.name) or setting.default
+			local value
+			local function update_value_from_selection()
+				value = table.concat(selection_list, ":")
+			end
+			if fields[setting.name] then
+				if core.is_yes(fields[setting.name]) then
+					value = ""
+				else
+					value = core.get_language()
+					if value == "" then
+						value = "en"
+					end
+				end
+			end
+			for k = 1, #selection_list do
+				local basekey = ("_%d_%s"):format(k, setting.name)
+				if fields["rem" .. basekey] then
+					table.remove(selection_list, k)
+					update_value_from_selection()
+					break
+				elseif fields["mdn" .. basekey] and k < #selection_list then
+					selection_list[k], selection_list[k+1] = selection_list[k+1], selection_list[k]
+					update_value_from_selection()
+					break
+				elseif fields["mup" .. basekey] and k > 1 then
+					selection_list[k], selection_list[k-1] = selection_list[k-1], selection_list[k]
+					update_value_from_selection()
+					break
+				end
+			end
+			if value == nil then
+				for k = 1, #selection_list+1 do
+					local selection = language_options[tonumber(fields[("sel_%d_%s"):format(k, setting.name)])]
+					if selection then
+						selection_list[k] = selection
+					end
+				end
+				update_value_from_selection()
+			end
+			if value == nil or value == old_value then
+				return false
+			end
+			core.settings:set(setting.name, value)
+			return true
+		end,
+	}
+end
 
 return make

builtin/mainmenu/settings/dlg_settings.lua

@@ -149,74 +149,6 @@ local function get_setting_info(name)
 	return nil
 end
 
-
--- These must not be translated, as they need to show in the local
--- language no matter the user's current language.
--- This list must be kept in sync with src/unsupported_language_list.txt.
-get_setting_info("language").option_labels = {
-	[""] = fgettext_ne("(Use system language)"),
-	--ar = " [ar]", blacklisted
-	be = "Беларуская [be]",
-	bg = "Български [bg]",
-	ca = "Català [ca]",
-	cs = "Česky [cs]",
-	cy = "Cymraeg [cy]",
-	da = "Dansk [da]",
-	de = "Deutsch [de]",
-	--dv = " [dv]", blacklisted
-	el = "Ελληνικά [el]",
-	en = "English [en]",
-	eo = "Esperanto [eo]",
-	es = "Español [es]",
-	et = "Eesti [et]",
-	eu = "Euskara [eu]",
-	fi = "Suomi [fi]",
-	fil = "Wikang Filipino [fil]",
-	fr = "Français [fr]",
-	gd = "Gàidhlig [gd]",
-	gl = "Galego [gl]",
-	--he = " [he]", blacklisted
-	--hi = " [hi]", blacklisted
-	hu = "Magyar [hu]",
-	id = "Bahasa Indonesia [id]",
-	it = "Italiano [it]",
-	ja = "日本語 [ja]",
-	jbo = "Lojban [jbo]",
-	kk = "Қазақша [kk]",
-	--kn = " [kn]", blacklisted
-	ko = "한국어 [ko]",
-	ky = "Kırgızca / Кыргызча [ky]",
-	lt = "Lietuvių [lt]",
-	lv = "Latviešu [lv]",
-	mn = "Монгол [mn]",
-	mr = "मराठी [mr]",
-	ms = "Bahasa Melayu [ms]",
-	--ms_Arab = " [ms_Arab]", blacklisted
-	nb = "Norsk Bokmål [nb]",
-	nl = "Nederlands [nl]",
-	nn = "Norsk Nynorsk [nn]",
-	oc = "Occitan [oc]",
-	pl = "Polski [pl]",
-	pt = "Português [pt]",
-	pt_BR = "Português do Brasil [pt_BR]",
-	ro = "Română [ro]",
-	ru = "Русский [ru]",
-	sk = "Slovenčina [sk]",
-	sl = "Slovenščina [sl]",
-	sr_Cyrl = "Српски [sr_Cyrl]",
-	sr_Latn = "Srpski (Latinica) [sr_Latn]",
-	sv = "Svenska [sv]",
-	sw = "Kiswahili [sw]",
-	--th = " [th]", blacklisted
-	tr = "Türkçe [tr]",
-	tt = "Tatarça [tt]",
-	uk = "Українська [uk]",
-	vi = "Tiếng Việt [vi]",
-	zh_CN = "中文 (简体) [zh_CN]",
-	zh_TW = "正體中文 (繁體) [zh_TW]",
-}
-
-
 -- See if setting matches keywords
 local function get_setting_match_weight(entry, query_keywords)
 	local setting_score = 0
@@ -574,6 +506,12 @@ local function get_formspec(dialogdata)
 
 		if show_reset then
 			local default = comp.setting.default
+			local default_description = comp.setting.default_description
+			if default_description then
+				if type(default_description) == "function" then
+					default = default_description()
+				end
+			end
 			local reset_tooltip = default and
 					fgettext("Reset setting to default ($1)", tostring(default)) or
 					fgettext("Reset setting to default")

builtin/mainmenu/settings/settingtypes.lua

@@ -366,6 +366,28 @@ local function parse_setting_line(settings, line, read_all, base_level, allow_se
 		return
 	end
 
+	if setting_type == "language_list" then
+		local default = remaining_line
+
+		local setting = {
+			name = name,
+			readable_name = readable_name,
+			type = "language_list",
+			default = default,
+			requires = requires,
+			comment = comment,
+		}
+
+		if default == "" then
+			setting.default_description = function()
+				return fgettext("Use system language")
+			end
+		end
+
+		table.insert(settings, setting)
+		return
+	end
+
 	return "Invalid setting type \"" .. setting_type .. "\""
 end
 

builtin/settingtypes.txt

@@ -18,6 +18,7 @@
 #    - noise_params_2d
 #    - noise_params_3d
 #    - v3f
+#    - language_list
 #
 # `type_args` can be:
 #   * int:
@@ -50,6 +51,9 @@
 #   * v3f:
 #            Format is (<X>, <Y>, <Z>)
 #            - default
+#   * language_list:
+#            Each entry is separated by a colon without spaces.
+#            - default
 #
 # Comments directly above a setting are bound to this setting.
 # All other comments are ignored.
@@ -683,7 +687,7 @@ mute_sound (Mute sound) bool false
 
 #    Set the language. By default, the system language is used.
 #    A restart is required after changing this.
-language (Language) enum   ,be,bg,ca,cs,da,de,el,en,eo,es,et,eu,fi,fr,gd,gl,hu,id,it,ja,jbo,kk,ko,lt,lv,ms,nb,nl,nn,pl,pt,pt_BR,ro,ru,sk,sl,sr_Cyrl,sr_Latn,sv,sw,tr,uk,vi,zh_CN,zh_TW
+language (Language) language_list
 
 [**GUI]
 

doc/menu_lua_api.md

@@ -415,6 +415,9 @@ Helpers
 * `core.urlencode(str)`: Encodes non-unreserved URI characters by a
   percent sign followed by two hex digits. See
   [RFC 3986, section 2.3](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3).
+* `core.language_names`: Table of language names recognized by Minetest.
+  * The key includes the language code; the value is the native name of the language.
+  * The value may be an empty string, which indicates that native name is unknown.
 
 
 Async

games/devtest/mods/unittests/init.lua

@@ -184,6 +184,7 @@ dofile(modpath .. "/itemstack_equals.lua")
 dofile(modpath .. "/content_ids.lua")
 dofile(modpath .. "/metadata.lua")
 dofile(modpath .. "/raycast.lua")
+dofile(modpath .. "/translations.lua")
 
 --------------
 

games/devtest/mods/unittests/locale/unittests.de.tr

@@ -0,0 +1,5 @@
+# textdomain: unittests
+# Note that the "translations" here are only for testing; it is not necessary
+# to actually translate any text here.
+Only in primary language.=Result in primary language.
+Available in both languages.=Result in primary language.

games/devtest/mods/unittests/locale/unittests.fr.tr

@@ -0,0 +1,5 @@
+# textdomain: unittests
+# Note that the "translations" here are only for testing; it is not necessary
+# to actually translate any text here.
+Only in secondary language.=Result in secondary language.
+Available in both languages.=Result in secondary language.

games/devtest/mods/unittests/translations.lua

@@ -0,0 +1,13 @@
+local S = core.get_translator("unittests")
+
+local function test_server_translation()
+	local function translate(str)
+		return core.get_translated_string("de:fr", S(str))
+	end
+	local RESULT_PRIMARY = "Result in primary language."
+	local RESULT_SECONDARY = "Result in secondary language."
+	assert(translate("Only in primary language.") == RESULT_PRIMARY, "missing translation for primary language")
+	assert(translate("Only in secondary language.") == RESULT_SECONDARY, "missing translation for secondary language")
+	assert(translate("Available in both languages.") == RESULT_PRIMARY, "incorrect translation priority list applied")
+end
+unittests.register("test_server_translation", test_server_translation)

src/CMakeLists.txt

@@ -692,6 +692,22 @@ elseif (GETTEXTLIB_FOUND)
 	set(GETTEXT_USED_LOCALES ${GETTEXT_AVAILABLE_LOCALES})
 endif()
 
+set(LANGUAGE_MAP_UNNAMED "")
+if (GETTEXTLIB_FOUND)
+	file(READ "${PROJECT_SOURCE_DIR}/util/languagemap.h.in" LANGUAGE_MAP_KNOWN)
+	string(REGEX MATCHALL "\\\\\n\t_\\(\"[^\"]+" LANGUAGE_MAP_KNOWN "${LANGUAGE_MAP_KNOWN}")
+	list(TRANSFORM LANGUAGE_MAP_KNOWN REPLACE "^[^\"]+\"" "")
+	foreach(LOCALE ${GETTEXT_USED_LOCALES})
+		if(NOT "${LOCALE}" IN_LIST LANGUAGE_MAP_KNOWN)
+			string(APPEND LANGUAGE_MAP_UNNAMED ",_(\"${LOCALE}\", \"\")")
+		endif()
+	endforeach()
+endif()
+configure_file(
+	"${PROJECT_SOURCE_DIR}/util/languagemap.h.in"
+	"${PROJECT_BINARY_DIR}/util/languagemap.h"
+)
+
 # Set some optimizations and tweaks
 
 include(CheckCSourceCompiles)

src/client/client.cpp

@@ -858,7 +858,14 @@ bool Client::loadMedia(const std::string &data, const std::string &filename,
 			return false;
 		TRACESTREAM(<< "Client: Loading translation: "
 				<< "\"" << filename << "\"" << std::endl);
-		g_client_translations->loadTranslation(data);
+		auto langpos = name.rfind('.');
+		if (langpos == name.npos) {
+			verbosestream << "Client: Cannot determine language for translation file \""
+				<< filename << "\"" << std::endl;
+			return false;
+		}
+		std::string lang = name.substr(langpos+1);
+		g_client_translations->loadTranslation(lang, data);
 		return true;
 	}
 

src/client/game.cpp

@@ -1108,6 +1108,7 @@ bool Game::startup(bool *kill,
 	m_touch_use_crosshair = g_settings->getBool("touch_use_crosshair");
 
 	g_client_translations->clear();
+	g_client_translations->setPreferredLanguages(get_current_locale());
 
 	// address can change if simple_singleplayer_mode
 	if (!init(start_data.world_spec.path, start_data.address,

src/gettext.cpp

@@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <iostream>
 #include "gettext.h"
 #include "util/string.h"
+#include "util/langcode.h"
 #include "porting.h"
 #include "log.h"
 
@@ -164,6 +165,8 @@ static void MSVC_LocaleWorkaround(int argc, char* argv[])
 
 #endif
 
+static std::string current_locale;
+
 /******************************************************************************/
 void init_gettext(const char *path, const std::string &configured_language,
 	int argc, char *argv[])
@@ -236,6 +239,19 @@ void init_gettext(const char *path, const std::string &configured_language,
 	setlocale(LC_ALL, "");
 #endif // if USE_GETTEXT
 
+	// Update locale information locally regardless of whether gettext
+	// is available.
+	if (!configured_language.empty()) {
+		current_locale = get_tr_language(configured_language);
+	} else if (current_locale.empty()) {
+		// Set locale if this was not previously set
+		char *lang = getenv("LANGUAGE");
+		if (!(lang && *lang))
+			lang = getenv("LANG");
+		if (lang && *lang)
+			current_locale = get_tr_language(lang);
+	}
+
 	/* no matter what locale is used we need number format to be "C" */
 	/* to ensure formspec parameters are evaluated correctly!        */
 
@@ -243,3 +259,8 @@ void init_gettext(const char *path, const std::string &configured_language,
 	infostream << "Message locale is now set to: "
 			<< setlocale(LC_ALL, 0) << std::endl;
 }
+
+const std::string &get_current_locale()
+{
+	return current_locale;
+}

src/gettext.h

@@ -46,6 +46,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 void init_gettext(const char *path, const std::string &configured_language,
 	int argc, char *argv[]);
 
+const std::string &get_current_locale();
+
 inline std::string strgettext(const char *str)
 {
 	// We must check here that is not an empty string to avoid trying to translate it

src/gui/guiEngine.cpp

@@ -254,6 +254,7 @@ Translations *GUIEngine::getContentTranslations(const std::string &path,
 
 	m_last_translations_key = key;
 	m_last_translations = {};
+	m_last_translations.setPreferredLanguages(lang_code);
 
 	std::string data;
 	if (fs::ReadFile(trans_path, data)) {

src/network/clientpackethandler.cpp

@@ -42,6 +42,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "util/serialize.h"
 #include "util/srp.h"
 #include "util/sha1.h"
+#include "util/langcode.h"
 #include "tileanimation.h"
 #include "gettext.h"
 #include "skyparams.h"
@@ -150,13 +151,10 @@ void Client::handleCommand_AuthAccept(NetworkPacket* pkt)
 					<< m_recommended_send_interval<<std::endl;
 
 	// Reply to server
-	/*~ DO NOT TRANSLATE THIS LITERALLY!
-	This is a special string which needs to contain the translation's
-	language code (e.g. "de" for German). */
-	std::string lang = gettext("LANG_CODE");
-	if (lang == "LANG_CODE")
-		lang.clear();
-
+	std::string lang = get_current_locale();
+	// Old server: only send the primary language
+	if (m_proto_ver < 44)
+		lang = get_primary_language(lang);
 	NetworkPacket resp_pkt(TOSERVER_INIT2, sizeof(u16) + lang.size());
 	resp_pkt << lang;
 	Send(&resp_pkt);

src/network/networkprotocol.h

@@ -223,6 +223,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 		AO_CMD_SET_BONE_POSITION extended
 		Add TOCLIENT_MOVE_PLAYER_REL
 		Move default minimap from client-side C++ to server-side builtin Lua
+		Server handles language priority lists for translations
 		[scheduled bump for 5.9.0]
 */
 

src/porting_android.cpp

@@ -29,6 +29,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "filesys.h"
 #include "log.h"
 #include "settings.h"
+#include "util/langcode.h"
 
 #include <sstream>
 #include <exception>
@@ -111,9 +112,10 @@ void osSpecificInit()
 			std::endl;
 
 	// Set default language
-	auto lang = getLanguageAndroid();
-	unsetenv("LANGUAGE");
-	setenv("LANG", lang.c_str(), 1);
+	std::string languages = getLanguageAndroid();
+	std::string primary_language = get_primary_language(languages);
+	setenv("LANGUAGE", languages.c_str(), 1);
+	setenv("LANG", primary_language.c_str(), 1);
 
 #ifdef GPROF
 	// in the start-up code

src/script/lua_api/l_client.cpp

@@ -214,10 +214,7 @@ int ModApiClient::l_get_language(lua_State *L)
 #else
 	char *locale = setlocale(LC_MESSAGES, NULL);
 #endif
-	std::string lang = gettext("LANG_CODE");
-	if (lang == "LANG_CODE")
-		lang.clear();
-
+	std::string lang = get_current_locale();
 	lua_pushstring(L, locale);
 	lua_pushstring(L, lang.c_str());
 	return 2;

src/script/lua_api/l_mainmenu.cpp

@@ -40,6 +40,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "content/mod_configuration.h"
 #include "threading/mutex_auto_lock.h"
 #include "common/c_converter.h"
+#include "util/languagemap.h"
 
 /******************************************************************************/
 std::string ModApiMainMenu::getTextData(lua_State *L, const std::string &name)
@@ -516,9 +517,7 @@ int ModApiMainMenu::l_get_content_translation(lua_State *L)
 	std::string path = luaL_checkstring(L, 1);
 	std::string domain = luaL_checkstring(L, 2);
 	std::string string = luaL_checkstring(L, 3);
-	std::string lang = gettext("LANG_CODE");
-	if (lang == "LANG_CODE")
-		lang = "";
+	std::string lang = get_current_locale();
 
 	auto *translations = engine->getContentTranslations(path, domain, lang);
 	string = wide_to_utf8(translate_string(utf8_to_wide(string), translations));
@@ -936,10 +935,7 @@ int ModApiMainMenu::l_get_video_drivers(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_get_language(lua_State *L)
 {
-	std::string lang = gettext("LANG_CODE");
-	if (lang == "LANG_CODE")
-		lang = "";
-
+	std::string lang = get_current_locale();
 	lua_pushstring(L, lang.c_str());
 	return 1;
 }
@@ -1132,6 +1128,15 @@ void ModApiMainMenu::Initialize(lua_State *L, int top)
 	API_FCT(open_dir);
 	API_FCT(share_file);
 	API_FCT(do_async_callback);
+
+	// Insert table of language names
+	lua_newtable(L);
+#define NAME(code, name) \
+	lua_pushstring(L, name), \
+	lua_setfield(L, -2, code)
+	LANGUAGE_MAP(NAME);
+#undef NAME
+	lua_setfield(L, top, "language_names");
 }
 
 /******************************************************************************/

src/server.cpp

@@ -59,6 +59,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "util/base64.h"
 #include "util/sha1.h"
 #include "util/hex.h"
+#include "util/langcode.h"
 #include "database/database.h"
 #include "chatmessage.h"
 #include "chat_interface.h"
@@ -2620,14 +2621,19 @@ void Server::fillMediaCache()
 
 void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_code)
 {
-	std::string lang_suffix = ".";
-	lang_suffix.append(lang_code).append(".tr");
+	auto lang_list = parse_language_list(lang_code);
 
 	auto include = [&] (const std::string &name, const MediaInfo &info) -> bool {
 		if (info.no_announce)
 			return false;
-		if (str_ends_with(name, ".tr") && !str_ends_with(name, lang_suffix))
+		if (str_ends_with(name, ".tr")) {
+			for (const auto &lang: lang_list) {
+				std::string suffix = "." + lang + ".tr";
+				if (str_ends_with(name, suffix))
+					return true;
+			}
 			return false;
+		}
 		return true;
 	};
 
@@ -4191,14 +4197,22 @@ Translations *Server::getTranslationLanguage(const std::string &lang_code)
 		return &it->second; // Already loaded
 
 	// [] will create an entry
+	server_translations[lang_code] = base_server_translations;
 	auto *translations = &server_translations[lang_code];
+	translations->setPreferredLanguages(lang_code);
 
-	std::string suffix = "." + lang_code + ".tr";
-	for (const auto &i : m_media) {
-		if (str_ends_with(i.first, suffix)) {
-			std::string data;
-			if (fs::ReadFile(i.second.path, data)) {
-				translations->loadTranslation(data);
+	if (wide_to_utf8(translations->getPreferredPrimaryLanguage()) != lang_code) {
+		// Load translation file for each language listed in lang_code
+		for (auto language: translations->getPreferredLanguages())
+			getTranslationLanguage(wide_to_utf8(language));
+	} else {
+		std::string suffix = "." + lang_code + ".tr";
+		for (const auto &i : m_media) {
+			if (str_ends_with(i.first, suffix)) {
+				std::string data;
+				if (fs::ReadFile(i.second.path, data)) {
+					translations->loadTranslation(data);
+				}
 			}
 		}
 	}

src/server.h

@@ -661,6 +661,7 @@ private:
 	IWritableCraftDefManager *m_craftdef;
 
 	std::unordered_map<std::string, Translations> server_translations;
+	Translations base_server_translations;
 
 	/*
 		Threads

src/server/clientiface.cpp

@@ -655,7 +655,7 @@ void RemoteClient::setVersionInfo(u8 major, u8 minor, u8 patch, const std::strin
 
 void RemoteClient::setLangCode(const std::string &code)
 {
-	m_lang_code = string_sanitize_ascii(code, 12);
+	m_lang_code = string_sanitize_ascii(code, 30);
 }
 
 ClientInterface::ClientInterface(const std::shared_ptr<con::Connection> & con)

src/translation.cpp

@@ -20,6 +20,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "translation.h"
 #include "log.h"
 #include "util/string.h"
+#include "util/langcode.h"
 #include <unordered_map>
 
 
@@ -29,23 +30,36 @@ static Translations client_translations;
 Translations *g_client_translations = &client_translations;
 #endif
 
+#define INDEX(language, textdomain, s) (language + L"|" + (textdomain) + L"|" + (s))
+
+Translations::Translations():
+	m_translations(new std::unordered_map<std::wstring, std::wstring>)
+{}
 
 void Translations::clear()
 {
-	m_translations.clear();
+	m_translations->clear();
+}
+
+const std::wstring &Translations::getTranslation(const std::vector<std::wstring> &languages,
+		const std::wstring &textdomain, const std::wstring &s) const
+{
+	for (const auto &language: languages) {
+		std::wstring key = INDEX(language, textdomain, s);
+		auto it = m_translations->find(key);
+		if (it != m_translations->end())
+			return it->second;
+	}
+	return s;
 }
 
 const std::wstring &Translations::getTranslation(
 		const std::wstring &textdomain, const std::wstring &s) const
 {
-	std::wstring key = textdomain + L"|" + s;
-	auto it = m_translations.find(key);
-	if (it != m_translations.end())
-		return it->second;
-	return s;
+	return getTranslation(m_preferred_languages, textdomain, s);
 }
 
-void Translations::loadTranslation(const std::string &data)
+void Translations::loadTranslation(const std::wstring &language, const std::string &data)
 {
 	std::istringstream is(data);
 	std::string textdomain_narrow;
@@ -147,9 +161,28 @@ void Translations::loadTranslation(const std::string &data)
 
 		std::wstring oword1 = word1.str(), oword2 = word2.str();
 		if (!oword2.empty()) {
-			std::wstring translation_index = textdomain + L"|";
-			translation_index.append(oword1);
-			m_translations.emplace(std::move(translation_index), std::move(oword2));
+			std::wstring translation_index = INDEX(language, textdomain, oword1);
+			m_translations->emplace(std::move(translation_index), std::move(oword2));
 		}
 	}
 }
+
+void Translations::loadTranslation(const std::string &language, const std::string &data)
+{
+	loadTranslation(utf8_to_wide(language), data);
+}
+
+void Translations::loadTranslation(const std::string &data)
+{
+	loadTranslation(getPreferredPrimaryLanguage(), data);
+}
+
+void Translations::setPreferredLanguages(const std::wstring &languages)
+{
+	setPreferredLanguages(parse_language_list(languages));
+}
+
+void Translations::setPreferredLanguages(const std::string &languages)
+{
+	setPreferredLanguages(utf8_to_wide(languages));
+}

src/translation.h

@@ -21,6 +21,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include <unordered_map>
 #include <string>
+#include <memory>
+#include <vector>
 
 class Translations;
 #ifndef SERVER
@@ -30,11 +32,33 @@ extern Translations *g_client_translations;
 class Translations
 {
 public:
+	Translations();
+
+	void loadTranslation(const std::wstring &language, const std::string &data);
+	void loadTranslation(const std::string &language, const std::string &data);
 	void loadTranslation(const std::string &data);
 	void clear();
-	const std::wstring &getTranslation(const std::wstring &textdomain,
-		const std::wstring &s) const;
+
+	const std::wstring &getTranslation(const std::vector<std::wstring> &languages,
+			const std::wstring &textdomain, const std::wstring &s) const;
+	const std::wstring &getTranslation(
+			const std::wstring &textdomain, const std::wstring &s) const;
+
+	inline const std::wstring getPreferredPrimaryLanguage() const {
+		if (m_preferred_languages.empty())
+			return L"";
+		return m_preferred_languages[0];
+	}
+	inline const std::vector<std::wstring> &getPreferredLanguages() const {
+		return m_preferred_languages;
+	}
+	inline void setPreferredLanguages(const std::vector<std::wstring> &languages) {
+		m_preferred_languages = languages;
+	}
+	void setPreferredLanguages(const std::wstring &languages);
+	void setPreferredLanguages(const std::string &languages);
 
 private:
-	std::unordered_map<std::wstring, std::wstring> m_translations;
+	std::vector<std::wstring> m_preferred_languages;
+	std::shared_ptr<std::unordered_map<std::wstring, std::wstring>> m_translations;
 };

src/unsupported_language_list.txt

@@ -1,6 +1,7 @@
 List of languages that are not supported. See issue #4638.
 ar
 dv
+fa
 he
 hi
 kn

src/util/CMakeLists.txt

@@ -6,6 +6,7 @@ set(UTIL_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/directiontables.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/enriched_string.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/ieee_float.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/langcode.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/metricsbackend.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/numeric.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/pointedthing.cpp

src/util/langcode.cpp

@@ -0,0 +1,44 @@
+#include "langcode.h"
+#include "util/languagemap.h"
+#include <set>
+#include <utility>
+
+// We don't need the lanugae name here, only the language code
+#define ENTRY(code, name) code
+static const std::set<std::string> language_codes { LANGUAGE_MAP(ENTRY) };
+#undef ENTRY
+
+// We are not using C++20 so set::contains is not available
+static inline bool contains(const std::set<std::string> &set, const std::string &entry)
+{
+	return set.find(entry) != set.end();
+}
+
+static std::string find_tr_language(const std::string &code)
+{
+	if (contains(language_codes, code))
+		return code;
+
+	// Strip encoding (".UTF-8") and variant ("@...") if they are present as we currently don't use them
+	auto pos = code.find_first_of(".@");
+	if (pos != code.npos)
+		return find_tr_language(code.substr(0, pos));
+
+	// Strip regional information if they are present
+	pos = code.find('_');
+	if (pos != code.npos)
+		return find_tr_language(code.substr(0, pos));
+
+	return "";
+}
+
+std::vector<std::string> get_tr_language(const std::vector<std::string> &languages)
+{
+	std::vector<std::string> list;
+	for (const auto &language: languages) {
+		std::string tr_lang = find_tr_language(language);
+		if (!tr_lang.empty())
+			list.push_back(std::move(tr_lang));
+	}
+	return list;
+}

src/util/langcode.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "util/string.h"
+#include <vector>
+
+template<typename T>
+inline std::vector<std::basic_string<T>> parse_language_list(const std::basic_string<T> &list)
+{
+	return str_split(list, T(':'));
+}
+
+inline std::string language_list_to_string(const std::vector<std::string> &list)
+{
+	return str_join(list, ":");
+}
+
+std::vector<std::string> get_tr_language(const std::vector<std::string> &lang);
+
+inline const std::string get_tr_language(const std::string &lang)
+{
+	return language_list_to_string(get_tr_language(parse_language_list(lang)));
+}
+
+template<typename T>
+inline const std::basic_string<T> get_primary_language(const std::basic_string<T> &lang)
+{
+	auto delimiter_pos = lang.find(':');
+	if (delimiter_pos == std::basic_string<T>::npos)
+		return lang;
+	return lang.substr(0, delimiter_pos);
+}

src/util/languagemap.h.in

@@ -0,0 +1,56 @@
+#pragma once
+#define LANGUAGE_MAP(_) \
+	_("be", "Беларуская"), \
+	_("bg", "Български"), \
+	_("ca", "Català"), \
+	_("cs", "Česky"), \
+	_("cy", "Cymraeg"), \
+	_("da", "Dansk"), \
+	_("de", "Deutsch"), \
+	_("el", "Ελληνικά"), \
+	_("en", "English"), \
+	_("eo", "Esperanto"), \
+	_("es", "Español"), \
+	_("et", "Eesti"), \
+	_("eu", "Euskara"), \
+	_("fi", "Suomi"), \
+	_("fil", "Wikang Filipino"), \
+	_("fr", "Français"), \
+	_("gd", "Gàidhlig"), \
+	_("gl", "Galego"), \
+	_("hu", "Magyar"), \
+	_("id", "Bahasa Indonesia"), \
+	_("it", "Italiano"), \
+	_("ja", "日本語"), \
+	_("jbo", "Lojban"), \
+	_("kk", "Қазақша"), \
+	_("ko", "한국어"), \
+	_("ky", "Kırgızca / Кыргызча"), \
+	_("lt", "Lietuvių"), \
+	_("lv", "Latviešu"), \
+	_("mn", "Монгол"), \
+	_("mr", "मराठी"),\
+	_("ms", "Bahasa Melayu"), \
+	_("nb", "Norsk Bokmål"), \
+	_("nl", "Nederlands"), \
+	_("nn", "Norsk Nynorsk"), \
+	_("oc", "Occitan"), \
+	_("pl", "Polski"), \
+	_("pt", "Português"), \
+	_("pt_BR", "Português do Brasil"), \
+	_("ro", "Română"), \
+	_("ru", "Русский"), \
+	_("sk", "Slovenčina"), \
+	_("sl", "Slovenščina"), \
+	_("sr_Cyrl", "Српски"), \
+	_("sr_Latn", "Srpski (Latinica)"), \
+	_("sv", "Svenska"), \
+	_("sw", "Kiswahili"), \
+	_("tr", "Türkçe"), \
+	_("tt", "Tatarça"), \
+	_("uk", "Українська"), \
+	_("vi", "Tiếng Việt"), \
+	_("yue", "粵語"), \
+	_("zh_CN", "中文 (简体)"), \
+	_("zh_TW", "正體中文 (繁體)") \
+	@LANGUAGE_MAP_UNNAMED@