#!/usr/bin/env python # -*- coding: utf-8 -*- # # Script to generate the template file and update the translation files. # Copy the script into the mod or modpack root folder and run it there. # # Copyright (C) 2019 Joachim Stolberg, 2020 FaceDeer # LGPLv2.1+ from __future__ import print_function import os, fnmatch, re, shutil, errno verbose = False #group 2 will be the string, groups 1 and 3 will be the delimiters (" or ') #See https://stackoverflow.com/questions/46967465/regex-match-text-in-either-single-or-double-quote pattern_lua = re.compile(r'[\.=^\t,{\(\s]N?S\(\s*(["\'])((?:\\\1|(?:(?!\1)).)*)(\1)[\s,\)]', re.DOTALL) pattern_lua_bracketed = re.compile(r'[\.=^\t,{\(\s]N?S\(\s*\[\[(.*?)\]\][\s,\)]', re.DOTALL) # Handles "concatenation" .. " of strings" pattern_concat = re.compile(r'["\'][\s]*\.\.[\s]*["\']', re.DOTALL) pattern_tr = re.compile(r'(.+?[^@])=(.+)') pattern_name = re.compile(r'^name[ ]*=[ ]*([^ \n]*)') pattern_tr_filename = re.compile(r'\.tr$') pattern_po_language_code = re.compile(r'(.*)\.po$') #attempt to read the mod's name from the mod.conf file. Returns None on failure def get_modname(folder): try: with open(folder + "mod.conf", "r", encoding='utf-8') as mod_conf: for line in mod_conf: match = pattern_name.match(line) if match: return match.group(1) except FileNotFoundError: pass return None #If there are already .tr files in /locale, returns a list of their names def get_existing_tr_files(folder): out = [] for root, dirs, files in os.walk(folder + 'locale/'): for name in files: if pattern_tr_filename.search(name): out.append(name) return out # A series of search and replaces that massage a .po file's contents into # a .tr file's equivalent def process_po_file(text): # The first three items are for unused matches text = re.sub(r'#~ msgid "', "", text) text = re.sub(r'"\n#~ msgstr ""\n"', "=", text) text = re.sub(r'"\n#~ msgstr "', "=", text) # comment lines text = re.sub(r'#.*\n', "", text) # converting msg pairs into "=" pairs text = re.sub(r'msgid "', "", text) text = re.sub(r'"\nmsgstr ""\n"', "=", text) text = re.sub(r'"\nmsgstr "', "=", text) # various line breaks and escape codes text = re.sub(r'"\n"', "", text) text = re.sub(r'"\n', "\n", text) text = re.sub(r'\\"', '"', text) text = re.sub(r'\\n', '@n', text) # remove header text text = re.sub(r'=Project-Id-Version:.*\n', "", text) # remove double-spaced lines text = re.sub(r'\n\n', '\n', text) return text # Go through existing .po files and, if a .tr file for that language # *doesn't* exist, convert it and create it. # The .tr file that results will subsequently be reprocessed so # any "no longer used" strings will be preserved. # Note that "fuzzy" tags will be lost in this process. def process_po_files(folder, modname): for root, dirs, files in os.walk(folder + 'locale/'): for name in files: code_match = pattern_po_language_code.match(name) if code_match == None: continue language_code = code_match.group(1) tr_name = modname + "." + language_code + ".tr" tr_file = os.path.join(root, tr_name) if os.path.exists(tr_file): if verbose: print(tr_name + " already exists, ignoring " + name) continue fname = os.path.join(root, name) with open(fname, "r", encoding='utf-8') as po_file: if verbose: print("Importing translations from " + name) text = process_po_file(po_file.read()) with open(tr_file, "wt", encoding='utf-8') as tr_out: tr_out.write(text) # from https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python/600612#600612 # Creates a directory if it doesn't exist, silently does # nothing if it already exists def mkdir_p(path): try: os.makedirs(path) except OSError as exc: # Python >2.5 if exc.errno == errno.EEXIST and os.path.isdir(path): pass else: raise # Converts the template dictionary to a text to be written as a file # dKeyStrings is a dictionary of localized string to source file sets # dOld is a dictionary of existing translations, for use when updating # existing .tr files def strings_to_text(dkeyStrings, dOld, mod_name): lOut = ["# textdomain: %s\n" % mod_name] dGroupedBySource = {} for key in dkeyStrings: sourceList = list(dkeyStrings[key]) sourceList.sort() sourceString = "\n".join(sourceList) listForSource = dGroupedBySource.get(sourceString, []) listForSource.append(key) dGroupedBySource[sourceString] = listForSource lSourceKeys = list(dGroupedBySource.keys()) lSourceKeys.sort() for source in lSourceKeys: lOut.append("") localizedStrings = dGroupedBySource[source] localizedStrings.sort() lOut.append(source) for localizedString in localizedStrings: val = dOld.get(localizedString, "") lOut.append("%s=%s" % (localizedString, val)) unusedExist = False for key in dOld: if key not in dkeyStrings: if not unusedExist: unusedExist = True lOut.append("\n##### not used anymore #####") lOut.append("%s=%s" % (key, dOld[key])) return "\n".join(lOut) # Writes a template.txt file # dkeyStrings is the dictionary returned by generate_template def write_template(templ_file, dkeyStrings, mod_name): text = strings_to_text(dkeyStrings, {}, mod_name) mkdir_p(os.path.dirname(templ_file)) with open(templ_file, "wt", encoding='utf-8') as template_file: template_file.write(text) # Gets all translatable strings from a lua file def read_lua_file_strings(lua_file): lOut = [] with open(lua_file, encoding='utf-8') as text_file: text = text_file.read() #TODO remove comments here text = re.sub(pattern_concat, "", text) strings = [] for s in pattern_lua.findall(text): strings.append(s[1]) for s in pattern_lua_bracketed.findall(text): strings.append(s) for s in strings: s = re.sub(r'"\.\.\s+"', "", s) s = re.sub("@[^@=0-9]", "@@", s) s = s.replace('\\"', '"') s = s.replace("\\'", "'") s = s.replace("\n", "@n") s = s.replace("\\n", "@n") s = s.replace("=", "@=") lOut.append(s) return lOut # Gets strings from an existing translation file # returns both a dictionary of translations # and the full original source text so that the new text # can be compared to it for changes. def import_tr_file(tr_file): dOut = {} text = None if os.path.exists(tr_file): with open(tr_file, "r", encoding='utf-8') as existing_file : text = existing_file.read() existing_file.seek(0) for line in existing_file.readlines(): s = line.strip() if s == "" or s[0] == "#": continue match = pattern_tr.match(s) if match: dOut[match.group(1)] = match.group(2) return (dOut, text) # Walks all lua files in the mod folder, collects translatable strings, # and writes it to a template.txt file # Returns a dictionary of localized strings to source file sets # that can be used with the strings_to_text function. def generate_template(folder, mod_name): dOut = {} for root, dirs, files in os.walk(folder): for name in files: if fnmatch.fnmatch(name, "*.lua"): fname = os.path.join(root, name) found = read_lua_file_strings(fname) if verbose: print(fname + ": " + str(len(found)) + " translatable strings") for s in found: sources = dOut.get(s, set()) sources.add("# " + fname) dOut[s] = sources if len(dOut) == 0: return None templ_file = folder + "locale/template.txt" write_template(templ_file, dOut, mod_name) return dOut # Updates an existing .tr file, copying the old one to a ".old" file # if any changes have happened # dNew is the data used to generate the template, it has all the # currently-existing localized strings def update_tr_file(dNew, mod_name, tr_file): if verbose: print("updating " + tr_file) tr_import = import_tr_file(tr_file) dOld = tr_import[0] textOld = tr_import[1] textNew = strings_to_text(dNew, dOld, mod_name) if textOld and textOld != textNew: print(tr_file + " has changed.") shutil.copyfile(tr_file, tr_file+".old") with open(tr_file, "w", encoding='utf-8') as new_tr_file: new_tr_file.write(textNew) # Updates translation files for the mod in the given folder def update_mod(folder): modname = get_modname(folder) if modname is not None: process_po_files(folder, modname) print("Updating translations for " + modname) data = generate_template(folder, modname) if data == None: print("No translatable strings found in " + modname) else: for tr_file in get_existing_tr_files(folder): update_tr_file(data, modname, folder + "locale/" + tr_file) else: print("Unable to find modname in folder " + folder) # Determines if the folder being pointed to is a mod or a mod pack # and then runs update_mod accordingly def update_folder(folder): is_modpack = os.path.exists(folder+"modpack.txt") or os.path.exists(folder+"modpack.conf") if is_modpack: subfolders = [f.path for f in os.scandir(folder) if f.is_dir()] for subfolder in subfolders: update_mod(subfolder + "/") else: update_mod(folder) print("Done.") update_folder("./") # Runs this script on each sub-folder in the parent folder. # I'm using this for testing this script on all installed mods. #for modfolder in [f.path for f in os.scandir("../") if f.is_dir()]: # update_folder(modfolder + "/")