diff --git a/.gitignore b/.gitignore index 37e27bfed..e5e0e2f25 100644 --- a/.gitignore +++ b/.gitignore @@ -114,6 +114,7 @@ CMakeDoxy* compile_commands.json *.apk *.zip +src/util/languagemap.h # Visual Studio *.vcxproj* *.sln diff --git a/android/app/src/main/java/net/minetest/minetest/GameActivity.java b/android/app/src/main/java/net/minetest/minetest/GameActivity.java index 85336ac7e..9ec0d887d 100644 --- a/android/app/src/main/java/net/minetest/minetest/GameActivity.java +++ b/android/app/src/main/java/net/minetest/minetest/GameActivity.java @@ -24,6 +24,7 @@ import android.app.NativeActivity; import android.content.Intent; import android.net.Uri; import android.os.Bundle; +import android.os.LocaleList; import android.text.InputType; import android.util.Log; import android.view.KeyEvent; @@ -243,26 +244,40 @@ public class GameActivity extends NativeActivity { } public String getLanguage() { - String langCode = Locale.getDefault().getLanguage(); + LocaleList locales = LocaleList.getAdjustedDefault(); + StringBuilder listString = new StringBuilder(); + for (int i = 0; i < locales.size(); i++) { + Locale lang = locales.get(i); + String langCode = lang.getLanguage(); - // getLanguage() still uses old language codes to preserve compatibility. - // List of code changes in ISO 639-2: - // https://www.loc.gov/standards/iso639-2/php/code_changes.php - switch (langCode) { - case "in": - langCode = "id"; // Indonesian - break; - case "iw": - langCode = "he"; // Hebrew - break; - case "ji": - langCode = "yi"; // Yiddish - break; - case "jw": - langCode = "jv"; // Javanese - break; + // getLanguage() still uses old language codes to preserve compatibility. + // List of code changes in ISO 639-2: + // https://www.loc.gov/standards/iso639-2/php/code_changes.php + switch (langCode) { + case "in": + langCode = "id"; // Indonesian + break; + case "iw": + langCode = "he"; // Hebrew + break; + case "ji": + langCode = "yi"; // Yiddish + break; + case "jw": + langCode = "jv"; // Javanese + break; + } + if (i > 0) { + listString.append(':'); + } + listString.append(langCode); + + String countryCode = lang.getCountry(); + if (!countryCode.isEmpty()) { + listString.append('_'); + listString.append(countryCode); + } } - - return langCode; + return listString.toString(); } } diff --git a/builtin/mainmenu/content/dlg_contentstore.lua b/builtin/mainmenu/content/dlg_contentstore.lua index f42b3d4e5..5efb5f58a 100644 --- a/builtin/mainmenu/content/dlg_contentstore.lua +++ b/builtin/mainmenu/content/dlg_contentstore.lua @@ -644,7 +644,8 @@ local function fetch_pkgs() local languages local current_language = core.get_language() if current_language ~= "" then - languages = { current_language, "en;q=0.8" } + languages = current_language:split(":") + table.insert(languages, "en;q=0.8") else languages = { "en" } end diff --git a/builtin/mainmenu/settings/components.lua b/builtin/mainmenu/settings/components.lua index 51cc0c95b..ea84e922b 100644 --- a/builtin/mainmenu/settings/components.lua +++ b/builtin/mainmenu/settings/components.lua @@ -403,5 +403,151 @@ end make.noise_params_2d = make_noise_params make.noise_params_3d = make_noise_params +-- These must not be translated, as they need to show in the local +-- language no matter the user's current language. +-- This list must be kept in sync with src/unsupported_language_list.txt. +local language_option_labels = core.language_names + +local language_options = {} +for k in pairs(language_option_labels) do + table.insert(language_options, k) +end +table.sort(language_options) + +local language_dropdown = {} +for idx, langcode in ipairs(language_options) do + local langname = language_option_labels[langcode] + if langname == "" then + langname = langcode + else + langname = ("%s [%s]"):format(langname, langcode) + end + language_dropdown[idx] = core.formspec_escape(langname) + language_options[langcode] = idx +end +language_dropdown = table.concat(language_dropdown, ",") + +function make.language_list(setting) + local selection_list = {} + return { + info_text = setting.comment, + setting = setting, + get_formspec = function(self, avail_w) + local fs = {} + local height = 0.8 + selection_list = string.split(core.settings:get(setting.name) or setting.default, ":") + self.resettable = core.settings:has(setting.name) + + table.insert(fs, ("label[0,0.1;%s]"):format(get_label(setting))) + -- TODO: Change the label to "Use system language: $1" when #14379 is merged + table.insert(fs, ("checkbox[0,0.55;%s;%s;%s]"):format( + setting.name, fgettext("Use system language"), tostring(#selection_list == 0))) + + -- Note that the "Remove" button is added implicitly and only when the move buttons are present. + -- If the selection list includes only one language, removing it implies using the system language. + local move_columns = 2 -- Move up|Move down + if #selection_list == 1 then + move_columns = 0 + elseif #selection_list == 2 then + move_columns = 1 -- Move (i.e. swap order with the other entry) + end + local dropdown_width = avail_w + if move_columns > 0 then + dropdown_width = avail_w - 0.9*(move_columns+1) + end + + for idx, lang in ipairs(selection_list) do + local selid = language_options[lang] + local dropdown_entries = language_dropdown + if not selid then + dropdown_entries = dropdown_entries .. "," .. + fgettext("Other language: $1", lang) + selid = #language_options+1 + end + table.insert(fs, ("dropdown[0,%f;%f,0.8;sel_%d_%s;%s;%d;true]"):format( + height, dropdown_width, idx, setting.name, dropdown_entries, selid)) + if move_columns > 0 then + table.insert(fs, ("image_button[%f,%f;0.8,0.8;%s;rem_%d_%s;]"):format( + avail_w-0.8, height, + core.formspec_escape(defaulttexturedir .. "clear.png"), idx, setting.name)) + local move_button_x = avail_w-0.8-0.9*move_columns + local move_button_width = 0.8 + if move_columns == 2 and (idx == 1 or idx == #selection_list) then + move_button_width = 1.7 + end + if idx < #selection_list then + table.insert(fs, ("image_button[%f,%f;%f,0.8;%s;mdn_%d_%s;]"):format( + move_button_x, height, move_button_width, + core.formspec_escape(defaulttexturedir .. "down_icon.png"), idx, setting.name)) + move_button_x = move_button_x + 0.9 + end + if idx > 1 then + table.insert(fs, ("image_button[%f,%f;%f,0.8;%s;mup_%d_%s;]"):format( + move_button_x, height, move_button_width, + core.formspec_escape(defaulttexturedir .. "up_icon.png"), idx, setting.name)) + end + end + height = height+0.9 + end + if #selection_list > 0 then + local dropdown_entries = language_dropdown .. "," .. + fgettext("Select language to add to list") + local selid = #language_options+1 + table.insert(fs, ("dropdown[0,%f;%f,0.8;sel_%d_%s;%s;%d;true]"):format( + height, dropdown_width, #selection_list+1, setting.name, dropdown_entries, selid)) + height = height+0.9 + end + + return table.concat(fs), height + end, + on_submit = function(self, fields) + local old_value = core.settings:get(setting.name) or setting.default + local value + local function update_value_from_selection() + value = table.concat(selection_list, ":") + end + if fields[setting.name] then + if core.is_yes(fields[setting.name]) then + value = "" + else + value = core.get_language() + if value == "" then + value = "en" + end + end + end + for k = 1, #selection_list do + local basekey = ("_%d_%s"):format(k, setting.name) + if fields["rem" .. basekey] then + table.remove(selection_list, k) + update_value_from_selection() + break + elseif fields["mdn" .. basekey] and k < #selection_list then + selection_list[k], selection_list[k+1] = selection_list[k+1], selection_list[k] + update_value_from_selection() + break + elseif fields["mup" .. basekey] and k > 1 then + selection_list[k], selection_list[k-1] = selection_list[k-1], selection_list[k] + update_value_from_selection() + break + end + end + if value == nil then + for k = 1, #selection_list+1 do + local selection = language_options[tonumber(fields[("sel_%d_%s"):format(k, setting.name)])] + if selection then + selection_list[k] = selection + end + end + update_value_from_selection() + end + if value == nil or value == old_value then + return false + end + core.settings:set(setting.name, value) + return true + end, + } +end return make diff --git a/builtin/mainmenu/settings/dlg_settings.lua b/builtin/mainmenu/settings/dlg_settings.lua index d6bbf2570..20beca2ee 100644 --- a/builtin/mainmenu/settings/dlg_settings.lua +++ b/builtin/mainmenu/settings/dlg_settings.lua @@ -149,74 +149,6 @@ local function get_setting_info(name) return nil end - --- These must not be translated, as they need to show in the local --- language no matter the user's current language. --- This list must be kept in sync with src/unsupported_language_list.txt. -get_setting_info("language").option_labels = { - [""] = fgettext_ne("(Use system language)"), - --ar = " [ar]", blacklisted - be = "Беларуская [be]", - bg = "Български [bg]", - ca = "Català [ca]", - cs = "Česky [cs]", - cy = "Cymraeg [cy]", - da = "Dansk [da]", - de = "Deutsch [de]", - --dv = " [dv]", blacklisted - el = "Ελληνικά [el]", - en = "English [en]", - eo = "Esperanto [eo]", - es = "Español [es]", - et = "Eesti [et]", - eu = "Euskara [eu]", - fi = "Suomi [fi]", - fil = "Wikang Filipino [fil]", - fr = "Français [fr]", - gd = "Gàidhlig [gd]", - gl = "Galego [gl]", - --he = " [he]", blacklisted - --hi = " [hi]", blacklisted - hu = "Magyar [hu]", - id = "Bahasa Indonesia [id]", - it = "Italiano [it]", - ja = "日本語 [ja]", - jbo = "Lojban [jbo]", - kk = "Қазақша [kk]", - --kn = " [kn]", blacklisted - ko = "한국어 [ko]", - ky = "Kırgızca / Кыргызча [ky]", - lt = "Lietuvių [lt]", - lv = "Latviešu [lv]", - mn = "Монгол [mn]", - mr = "मराठी [mr]", - ms = "Bahasa Melayu [ms]", - --ms_Arab = " [ms_Arab]", blacklisted - nb = "Norsk Bokmål [nb]", - nl = "Nederlands [nl]", - nn = "Norsk Nynorsk [nn]", - oc = "Occitan [oc]", - pl = "Polski [pl]", - pt = "Português [pt]", - pt_BR = "Português do Brasil [pt_BR]", - ro = "Română [ro]", - ru = "Русский [ru]", - sk = "Slovenčina [sk]", - sl = "Slovenščina [sl]", - sr_Cyrl = "Српски [sr_Cyrl]", - sr_Latn = "Srpski (Latinica) [sr_Latn]", - sv = "Svenska [sv]", - sw = "Kiswahili [sw]", - --th = " [th]", blacklisted - tr = "Türkçe [tr]", - tt = "Tatarça [tt]", - uk = "Українська [uk]", - vi = "Tiếng Việt [vi]", - zh_CN = "中文 (简体) [zh_CN]", - zh_TW = "正體中文 (繁體) [zh_TW]", -} - - -- See if setting matches keywords local function get_setting_match_weight(entry, query_keywords) local setting_score = 0 @@ -574,6 +506,12 @@ local function get_formspec(dialogdata) if show_reset then local default = comp.setting.default + local default_description = comp.setting.default_description + if default_description then + if type(default_description) == "function" then + default = default_description() + end + end local reset_tooltip = default and fgettext("Reset setting to default ($1)", tostring(default)) or fgettext("Reset setting to default") diff --git a/builtin/mainmenu/settings/settingtypes.lua b/builtin/mainmenu/settings/settingtypes.lua index 1174c9b76..8882f8375 100644 --- a/builtin/mainmenu/settings/settingtypes.lua +++ b/builtin/mainmenu/settings/settingtypes.lua @@ -366,6 +366,28 @@ local function parse_setting_line(settings, line, read_all, base_level, allow_se return end + if setting_type == "language_list" then + local default = remaining_line + + local setting = { + name = name, + readable_name = readable_name, + type = "language_list", + default = default, + requires = requires, + comment = comment, + } + + if default == "" then + setting.default_description = function() + return fgettext("Use system language") + end + end + + table.insert(settings, setting) + return + end + return "Invalid setting type \"" .. setting_type .. "\"" end diff --git a/builtin/settingtypes.txt b/builtin/settingtypes.txt index 8ffb1c3c1..08191907b 100644 --- a/builtin/settingtypes.txt +++ b/builtin/settingtypes.txt @@ -18,6 +18,7 @@ # - noise_params_2d # - noise_params_3d # - v3f +# - language_list # # `type_args` can be: # * int: @@ -50,6 +51,9 @@ # * v3f: # Format is (, , ) # - default +# * language_list: +# Each entry is separated by a colon without spaces. +# - default # # Comments directly above a setting are bound to this setting. # All other comments are ignored. @@ -683,7 +687,7 @@ mute_sound (Mute sound) bool false # Set the language. By default, the system language is used. # A restart is required after changing this. -language (Language) enum ,be,bg,ca,cs,da,de,el,en,eo,es,et,eu,fi,fr,gd,gl,hu,id,it,ja,jbo,kk,ko,lt,lv,ms,nb,nl,nn,pl,pt,pt_BR,ro,ru,sk,sl,sr_Cyrl,sr_Latn,sv,sw,tr,uk,vi,zh_CN,zh_TW +language (Language) language_list [**GUI] diff --git a/doc/menu_lua_api.md b/doc/menu_lua_api.md index cb1f07a90..708fdae81 100644 --- a/doc/menu_lua_api.md +++ b/doc/menu_lua_api.md @@ -415,6 +415,9 @@ Helpers * `core.urlencode(str)`: Encodes non-unreserved URI characters by a percent sign followed by two hex digits. See [RFC 3986, section 2.3](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3). +* `core.language_names`: Table of language names recognized by Minetest. + * The key includes the language code; the value is the native name of the language. + * The value may be an empty string, which indicates that native name is unknown. Async diff --git a/games/devtest/mods/unittests/init.lua b/games/devtest/mods/unittests/init.lua index b01a6271c..1336b394a 100644 --- a/games/devtest/mods/unittests/init.lua +++ b/games/devtest/mods/unittests/init.lua @@ -184,6 +184,7 @@ dofile(modpath .. "/itemstack_equals.lua") dofile(modpath .. "/content_ids.lua") dofile(modpath .. "/metadata.lua") dofile(modpath .. "/raycast.lua") +dofile(modpath .. "/translations.lua") -------------- diff --git a/games/devtest/mods/unittests/locale/unittests.de.tr b/games/devtest/mods/unittests/locale/unittests.de.tr new file mode 100644 index 000000000..e79e5feef --- /dev/null +++ b/games/devtest/mods/unittests/locale/unittests.de.tr @@ -0,0 +1,5 @@ +# textdomain: unittests +# Note that the "translations" here are only for testing; it is not necessary +# to actually translate any text here. +Only in primary language.=Result in primary language. +Available in both languages.=Result in primary language. diff --git a/games/devtest/mods/unittests/locale/unittests.fr.tr b/games/devtest/mods/unittests/locale/unittests.fr.tr new file mode 100644 index 000000000..efde0ab6e --- /dev/null +++ b/games/devtest/mods/unittests/locale/unittests.fr.tr @@ -0,0 +1,5 @@ +# textdomain: unittests +# Note that the "translations" here are only for testing; it is not necessary +# to actually translate any text here. +Only in secondary language.=Result in secondary language. +Available in both languages.=Result in secondary language. diff --git a/games/devtest/mods/unittests/translations.lua b/games/devtest/mods/unittests/translations.lua new file mode 100644 index 000000000..8bcfdf70a --- /dev/null +++ b/games/devtest/mods/unittests/translations.lua @@ -0,0 +1,13 @@ +local S = core.get_translator("unittests") + +local function test_server_translation() + local function translate(str) + return core.get_translated_string("de:fr", S(str)) + end + local RESULT_PRIMARY = "Result in primary language." + local RESULT_SECONDARY = "Result in secondary language." + assert(translate("Only in primary language.") == RESULT_PRIMARY, "missing translation for primary language") + assert(translate("Only in secondary language.") == RESULT_SECONDARY, "missing translation for secondary language") + assert(translate("Available in both languages.") == RESULT_PRIMARY, "incorrect translation priority list applied") +end +unittests.register("test_server_translation", test_server_translation) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0f06d3f4d..403ffc7a6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -692,6 +692,22 @@ elseif (GETTEXTLIB_FOUND) set(GETTEXT_USED_LOCALES ${GETTEXT_AVAILABLE_LOCALES}) endif() +set(LANGUAGE_MAP_UNNAMED "") +if (GETTEXTLIB_FOUND) + file(READ "${PROJECT_SOURCE_DIR}/util/languagemap.h.in" LANGUAGE_MAP_KNOWN) + string(REGEX MATCHALL "\\\\\n\t_\\(\"[^\"]+" LANGUAGE_MAP_KNOWN "${LANGUAGE_MAP_KNOWN}") + list(TRANSFORM LANGUAGE_MAP_KNOWN REPLACE "^[^\"]+\"" "") + foreach(LOCALE ${GETTEXT_USED_LOCALES}) + if(NOT "${LOCALE}" IN_LIST LANGUAGE_MAP_KNOWN) + string(APPEND LANGUAGE_MAP_UNNAMED ",_(\"${LOCALE}\", \"\")") + endif() + endforeach() +endif() +configure_file( + "${PROJECT_SOURCE_DIR}/util/languagemap.h.in" + "${PROJECT_BINARY_DIR}/util/languagemap.h" +) + # Set some optimizations and tweaks include(CheckCSourceCompiles) diff --git a/src/client/client.cpp b/src/client/client.cpp index 789a37c36..e207ef8c0 100644 --- a/src/client/client.cpp +++ b/src/client/client.cpp @@ -858,7 +858,14 @@ bool Client::loadMedia(const std::string &data, const std::string &filename, return false; TRACESTREAM(<< "Client: Loading translation: " << "\"" << filename << "\"" << std::endl); - g_client_translations->loadTranslation(data); + auto langpos = name.rfind('.'); + if (langpos == name.npos) { + verbosestream << "Client: Cannot determine language for translation file \"" + << filename << "\"" << std::endl; + return false; + } + std::string lang = name.substr(langpos+1); + g_client_translations->loadTranslation(lang, data); return true; } diff --git a/src/client/game.cpp b/src/client/game.cpp index 55ec480b5..c589490bd 100644 --- a/src/client/game.cpp +++ b/src/client/game.cpp @@ -1108,6 +1108,7 @@ bool Game::startup(bool *kill, m_touch_use_crosshair = g_settings->getBool("touch_use_crosshair"); g_client_translations->clear(); + g_client_translations->setPreferredLanguages(get_current_locale()); // address can change if simple_singleplayer_mode if (!init(start_data.world_spec.path, start_data.address, diff --git a/src/gettext.cpp b/src/gettext.cpp index 68b6f728a..172442d76 100644 --- a/src/gettext.cpp +++ b/src/gettext.cpp @@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #include "gettext.h" #include "util/string.h" +#include "util/langcode.h" #include "porting.h" #include "log.h" @@ -164,6 +165,8 @@ static void MSVC_LocaleWorkaround(int argc, char* argv[]) #endif +static std::string current_locale; + /******************************************************************************/ void init_gettext(const char *path, const std::string &configured_language, int argc, char *argv[]) @@ -236,6 +239,19 @@ void init_gettext(const char *path, const std::string &configured_language, setlocale(LC_ALL, ""); #endif // if USE_GETTEXT + // Update locale information locally regardless of whether gettext + // is available. + if (!configured_language.empty()) { + current_locale = get_tr_language(configured_language); + } else if (current_locale.empty()) { + // Set locale if this was not previously set + char *lang = getenv("LANGUAGE"); + if (!(lang && *lang)) + lang = getenv("LANG"); + if (lang && *lang) + current_locale = get_tr_language(lang); + } + /* no matter what locale is used we need number format to be "C" */ /* to ensure formspec parameters are evaluated correctly! */ @@ -243,3 +259,8 @@ void init_gettext(const char *path, const std::string &configured_language, infostream << "Message locale is now set to: " << setlocale(LC_ALL, 0) << std::endl; } + +const std::string &get_current_locale() +{ + return current_locale; +} diff --git a/src/gettext.h b/src/gettext.h index 042729c1a..fa3410267 100644 --- a/src/gettext.h +++ b/src/gettext.h @@ -46,6 +46,8 @@ with this program; if not, write to the Free Software Foundation, Inc., void init_gettext(const char *path, const std::string &configured_language, int argc, char *argv[]); +const std::string &get_current_locale(); + inline std::string strgettext(const char *str) { // We must check here that is not an empty string to avoid trying to translate it diff --git a/src/gui/guiEngine.cpp b/src/gui/guiEngine.cpp index 4d35f38ab..dba167eb1 100644 --- a/src/gui/guiEngine.cpp +++ b/src/gui/guiEngine.cpp @@ -254,6 +254,7 @@ Translations *GUIEngine::getContentTranslations(const std::string &path, m_last_translations_key = key; m_last_translations = {}; + m_last_translations.setPreferredLanguages(lang_code); std::string data; if (fs::ReadFile(trans_path, data)) { diff --git a/src/network/clientpackethandler.cpp b/src/network/clientpackethandler.cpp index 90f2bed5b..82848ef46 100644 --- a/src/network/clientpackethandler.cpp +++ b/src/network/clientpackethandler.cpp @@ -42,6 +42,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "util/serialize.h" #include "util/srp.h" #include "util/sha1.h" +#include "util/langcode.h" #include "tileanimation.h" #include "gettext.h" #include "skyparams.h" @@ -150,13 +151,10 @@ void Client::handleCommand_AuthAccept(NetworkPacket* pkt) << m_recommended_send_interval< #include @@ -111,9 +112,10 @@ void osSpecificInit() std::endl; // Set default language - auto lang = getLanguageAndroid(); - unsetenv("LANGUAGE"); - setenv("LANG", lang.c_str(), 1); + std::string languages = getLanguageAndroid(); + std::string primary_language = get_primary_language(languages); + setenv("LANGUAGE", languages.c_str(), 1); + setenv("LANG", primary_language.c_str(), 1); #ifdef GPROF // in the start-up code diff --git a/src/script/lua_api/l_client.cpp b/src/script/lua_api/l_client.cpp index da19ed0ea..845f16b4e 100644 --- a/src/script/lua_api/l_client.cpp +++ b/src/script/lua_api/l_client.cpp @@ -214,10 +214,7 @@ int ModApiClient::l_get_language(lua_State *L) #else char *locale = setlocale(LC_MESSAGES, NULL); #endif - std::string lang = gettext("LANG_CODE"); - if (lang == "LANG_CODE") - lang.clear(); - + std::string lang = get_current_locale(); lua_pushstring(L, locale); lua_pushstring(L, lang.c_str()); return 2; diff --git a/src/script/lua_api/l_mainmenu.cpp b/src/script/lua_api/l_mainmenu.cpp index a5913e807..69963a317 100644 --- a/src/script/lua_api/l_mainmenu.cpp +++ b/src/script/lua_api/l_mainmenu.cpp @@ -40,6 +40,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "content/mod_configuration.h" #include "threading/mutex_auto_lock.h" #include "common/c_converter.h" +#include "util/languagemap.h" /******************************************************************************/ std::string ModApiMainMenu::getTextData(lua_State *L, const std::string &name) @@ -516,9 +517,7 @@ int ModApiMainMenu::l_get_content_translation(lua_State *L) std::string path = luaL_checkstring(L, 1); std::string domain = luaL_checkstring(L, 2); std::string string = luaL_checkstring(L, 3); - std::string lang = gettext("LANG_CODE"); - if (lang == "LANG_CODE") - lang = ""; + std::string lang = get_current_locale(); auto *translations = engine->getContentTranslations(path, domain, lang); string = wide_to_utf8(translate_string(utf8_to_wide(string), translations)); @@ -936,10 +935,7 @@ int ModApiMainMenu::l_get_video_drivers(lua_State *L) /******************************************************************************/ int ModApiMainMenu::l_get_language(lua_State *L) { - std::string lang = gettext("LANG_CODE"); - if (lang == "LANG_CODE") - lang = ""; - + std::string lang = get_current_locale(); lua_pushstring(L, lang.c_str()); return 1; } @@ -1132,6 +1128,15 @@ void ModApiMainMenu::Initialize(lua_State *L, int top) API_FCT(open_dir); API_FCT(share_file); API_FCT(do_async_callback); + + // Insert table of language names + lua_newtable(L); +#define NAME(code, name) \ + lua_pushstring(L, name), \ + lua_setfield(L, -2, code) + LANGUAGE_MAP(NAME); +#undef NAME + lua_setfield(L, top, "language_names"); } /******************************************************************************/ diff --git a/src/server.cpp b/src/server.cpp index 5dd2d4691..91295780e 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -59,6 +59,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "util/base64.h" #include "util/sha1.h" #include "util/hex.h" +#include "util/langcode.h" #include "database/database.h" #include "chatmessage.h" #include "chat_interface.h" @@ -2620,14 +2621,19 @@ void Server::fillMediaCache() void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_code) { - std::string lang_suffix = "."; - lang_suffix.append(lang_code).append(".tr"); + auto lang_list = parse_language_list(lang_code); auto include = [&] (const std::string &name, const MediaInfo &info) -> bool { if (info.no_announce) return false; - if (str_ends_with(name, ".tr") && !str_ends_with(name, lang_suffix)) + if (str_ends_with(name, ".tr")) { + for (const auto &lang: lang_list) { + std::string suffix = "." + lang + ".tr"; + if (str_ends_with(name, suffix)) + return true; + } return false; + } return true; }; @@ -4191,14 +4197,22 @@ Translations *Server::getTranslationLanguage(const std::string &lang_code) return &it->second; // Already loaded // [] will create an entry + server_translations[lang_code] = base_server_translations; auto *translations = &server_translations[lang_code]; + translations->setPreferredLanguages(lang_code); - std::string suffix = "." + lang_code + ".tr"; - for (const auto &i : m_media) { - if (str_ends_with(i.first, suffix)) { - std::string data; - if (fs::ReadFile(i.second.path, data)) { - translations->loadTranslation(data); + if (wide_to_utf8(translations->getPreferredPrimaryLanguage()) != lang_code) { + // Load translation file for each language listed in lang_code + for (auto language: translations->getPreferredLanguages()) + getTranslationLanguage(wide_to_utf8(language)); + } else { + std::string suffix = "." + lang_code + ".tr"; + for (const auto &i : m_media) { + if (str_ends_with(i.first, suffix)) { + std::string data; + if (fs::ReadFile(i.second.path, data)) { + translations->loadTranslation(data); + } } } } diff --git a/src/server.h b/src/server.h index 16c1ea4cc..fab11257c 100644 --- a/src/server.h +++ b/src/server.h @@ -661,6 +661,7 @@ private: IWritableCraftDefManager *m_craftdef; std::unordered_map server_translations; + Translations base_server_translations; /* Threads diff --git a/src/server/clientiface.cpp b/src/server/clientiface.cpp index 451e74407..e1565cfc3 100644 --- a/src/server/clientiface.cpp +++ b/src/server/clientiface.cpp @@ -655,7 +655,7 @@ void RemoteClient::setVersionInfo(u8 major, u8 minor, u8 patch, const std::strin void RemoteClient::setLangCode(const std::string &code) { - m_lang_code = string_sanitize_ascii(code, 12); + m_lang_code = string_sanitize_ascii(code, 30); } ClientInterface::ClientInterface(const std::shared_ptr & con) diff --git a/src/translation.cpp b/src/translation.cpp index 5d5491e56..0f4a92f59 100644 --- a/src/translation.cpp +++ b/src/translation.cpp @@ -20,6 +20,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "translation.h" #include "log.h" #include "util/string.h" +#include "util/langcode.h" #include @@ -29,23 +30,36 @@ static Translations client_translations; Translations *g_client_translations = &client_translations; #endif +#define INDEX(language, textdomain, s) (language + L"|" + (textdomain) + L"|" + (s)) + +Translations::Translations(): + m_translations(new std::unordered_map) +{} void Translations::clear() { - m_translations.clear(); + m_translations->clear(); +} + +const std::wstring &Translations::getTranslation(const std::vector &languages, + const std::wstring &textdomain, const std::wstring &s) const +{ + for (const auto &language: languages) { + std::wstring key = INDEX(language, textdomain, s); + auto it = m_translations->find(key); + if (it != m_translations->end()) + return it->second; + } + return s; } const std::wstring &Translations::getTranslation( const std::wstring &textdomain, const std::wstring &s) const { - std::wstring key = textdomain + L"|" + s; - auto it = m_translations.find(key); - if (it != m_translations.end()) - return it->second; - return s; + return getTranslation(m_preferred_languages, textdomain, s); } -void Translations::loadTranslation(const std::string &data) +void Translations::loadTranslation(const std::wstring &language, const std::string &data) { std::istringstream is(data); std::string textdomain_narrow; @@ -147,9 +161,28 @@ void Translations::loadTranslation(const std::string &data) std::wstring oword1 = word1.str(), oword2 = word2.str(); if (!oword2.empty()) { - std::wstring translation_index = textdomain + L"|"; - translation_index.append(oword1); - m_translations.emplace(std::move(translation_index), std::move(oword2)); + std::wstring translation_index = INDEX(language, textdomain, oword1); + m_translations->emplace(std::move(translation_index), std::move(oword2)); } } } + +void Translations::loadTranslation(const std::string &language, const std::string &data) +{ + loadTranslation(utf8_to_wide(language), data); +} + +void Translations::loadTranslation(const std::string &data) +{ + loadTranslation(getPreferredPrimaryLanguage(), data); +} + +void Translations::setPreferredLanguages(const std::wstring &languages) +{ + setPreferredLanguages(parse_language_list(languages)); +} + +void Translations::setPreferredLanguages(const std::string &languages) +{ + setPreferredLanguages(utf8_to_wide(languages)); +} diff --git a/src/translation.h b/src/translation.h index d7ed15505..cc9fc4035 100644 --- a/src/translation.h +++ b/src/translation.h @@ -21,6 +21,8 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #include +#include +#include class Translations; #ifndef SERVER @@ -30,11 +32,33 @@ extern Translations *g_client_translations; class Translations { public: + Translations(); + + void loadTranslation(const std::wstring &language, const std::string &data); + void loadTranslation(const std::string &language, const std::string &data); void loadTranslation(const std::string &data); void clear(); - const std::wstring &getTranslation(const std::wstring &textdomain, - const std::wstring &s) const; + + const std::wstring &getTranslation(const std::vector &languages, + const std::wstring &textdomain, const std::wstring &s) const; + const std::wstring &getTranslation( + const std::wstring &textdomain, const std::wstring &s) const; + + inline const std::wstring getPreferredPrimaryLanguage() const { + if (m_preferred_languages.empty()) + return L""; + return m_preferred_languages[0]; + } + inline const std::vector &getPreferredLanguages() const { + return m_preferred_languages; + } + inline void setPreferredLanguages(const std::vector &languages) { + m_preferred_languages = languages; + } + void setPreferredLanguages(const std::wstring &languages); + void setPreferredLanguages(const std::string &languages); private: - std::unordered_map m_translations; + std::vector m_preferred_languages; + std::shared_ptr> m_translations; }; diff --git a/src/unsupported_language_list.txt b/src/unsupported_language_list.txt index 20b8445e1..2721aa7ab 100644 --- a/src/unsupported_language_list.txt +++ b/src/unsupported_language_list.txt @@ -1,6 +1,7 @@ List of languages that are not supported. See issue #4638. ar dv +fa he hi kn diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 68e9eae5f..05f07909b 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -6,6 +6,7 @@ set(UTIL_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/directiontables.cpp ${CMAKE_CURRENT_SOURCE_DIR}/enriched_string.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ieee_float.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/langcode.cpp ${CMAKE_CURRENT_SOURCE_DIR}/metricsbackend.cpp ${CMAKE_CURRENT_SOURCE_DIR}/numeric.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pointedthing.cpp diff --git a/src/util/langcode.cpp b/src/util/langcode.cpp new file mode 100644 index 000000000..5eb195fa8 --- /dev/null +++ b/src/util/langcode.cpp @@ -0,0 +1,44 @@ +#include "langcode.h" +#include "util/languagemap.h" +#include +#include + +// We don't need the lanugae name here, only the language code +#define ENTRY(code, name) code +static const std::set language_codes { LANGUAGE_MAP(ENTRY) }; +#undef ENTRY + +// We are not using C++20 so set::contains is not available +static inline bool contains(const std::set &set, const std::string &entry) +{ + return set.find(entry) != set.end(); +} + +static std::string find_tr_language(const std::string &code) +{ + if (contains(language_codes, code)) + return code; + + // Strip encoding (".UTF-8") and variant ("@...") if they are present as we currently don't use them + auto pos = code.find_first_of(".@"); + if (pos != code.npos) + return find_tr_language(code.substr(0, pos)); + + // Strip regional information if they are present + pos = code.find('_'); + if (pos != code.npos) + return find_tr_language(code.substr(0, pos)); + + return ""; +} + +std::vector get_tr_language(const std::vector &languages) +{ + std::vector list; + for (const auto &language: languages) { + std::string tr_lang = find_tr_language(language); + if (!tr_lang.empty()) + list.push_back(std::move(tr_lang)); + } + return list; +} diff --git a/src/util/langcode.h b/src/util/langcode.h new file mode 100644 index 000000000..22aedf66b --- /dev/null +++ b/src/util/langcode.h @@ -0,0 +1,31 @@ +#pragma once + +#include "util/string.h" +#include + +template +inline std::vector> parse_language_list(const std::basic_string &list) +{ + return str_split(list, T(':')); +} + +inline std::string language_list_to_string(const std::vector &list) +{ + return str_join(list, ":"); +} + +std::vector get_tr_language(const std::vector &lang); + +inline const std::string get_tr_language(const std::string &lang) +{ + return language_list_to_string(get_tr_language(parse_language_list(lang))); +} + +template +inline const std::basic_string get_primary_language(const std::basic_string &lang) +{ + auto delimiter_pos = lang.find(':'); + if (delimiter_pos == std::basic_string::npos) + return lang; + return lang.substr(0, delimiter_pos); +} diff --git a/src/util/languagemap.h.in b/src/util/languagemap.h.in new file mode 100644 index 000000000..5fc7c3fba --- /dev/null +++ b/src/util/languagemap.h.in @@ -0,0 +1,56 @@ +#pragma once +#define LANGUAGE_MAP(_) \ + _("be", "Беларуская"), \ + _("bg", "Български"), \ + _("ca", "Català"), \ + _("cs", "Česky"), \ + _("cy", "Cymraeg"), \ + _("da", "Dansk"), \ + _("de", "Deutsch"), \ + _("el", "Ελληνικά"), \ + _("en", "English"), \ + _("eo", "Esperanto"), \ + _("es", "Español"), \ + _("et", "Eesti"), \ + _("eu", "Euskara"), \ + _("fi", "Suomi"), \ + _("fil", "Wikang Filipino"), \ + _("fr", "Français"), \ + _("gd", "Gàidhlig"), \ + _("gl", "Galego"), \ + _("hu", "Magyar"), \ + _("id", "Bahasa Indonesia"), \ + _("it", "Italiano"), \ + _("ja", "日本語"), \ + _("jbo", "Lojban"), \ + _("kk", "Қазақша"), \ + _("ko", "한국어"), \ + _("ky", "Kırgızca / Кыргызча"), \ + _("lt", "Lietuvių"), \ + _("lv", "Latviešu"), \ + _("mn", "Монгол"), \ + _("mr", "मराठी"),\ + _("ms", "Bahasa Melayu"), \ + _("nb", "Norsk Bokmål"), \ + _("nl", "Nederlands"), \ + _("nn", "Norsk Nynorsk"), \ + _("oc", "Occitan"), \ + _("pl", "Polski"), \ + _("pt", "Português"), \ + _("pt_BR", "Português do Brasil"), \ + _("ro", "Română"), \ + _("ru", "Русский"), \ + _("sk", "Slovenčina"), \ + _("sl", "Slovenščina"), \ + _("sr_Cyrl", "Српски"), \ + _("sr_Latn", "Srpski (Latinica)"), \ + _("sv", "Svenska"), \ + _("sw", "Kiswahili"), \ + _("tr", "Türkçe"), \ + _("tt", "Tatarça"), \ + _("uk", "Українська"), \ + _("vi", "Tiếng Việt"), \ + _("yue", "粵語"), \ + _("zh_CN", "中文 (简体)"), \ + _("zh_TW", "正體中文 (繁體)") \ + @LANGUAGE_MAP_UNNAMED@ diff --git a/textures/base/pack/down_icon.png b/textures/base/pack/down_icon.png new file mode 100644 index 000000000..adc19a8c9 Binary files /dev/null and b/textures/base/pack/down_icon.png differ diff --git a/textures/base/pack/up_icon.png b/textures/base/pack/up_icon.png new file mode 100644 index 000000000..3fe29e053 Binary files /dev/null and b/textures/base/pack/up_icon.png differ