Use PEP 8 naming conventions for i18n tools
This commit is contained in:
parent
80f808df4a
commit
4a049c5f3a
GPG Key ID:
885B16854284E0B2
@ -133,16 +133,16 @@ pipeline {
|
||||
stage("Update translations") {
|
||||
steps {
|
||||
ws("workspace/nightly-svn") {
|
||||
bat "cd source\\tools\\i18n && python updateTemplates.py"
|
||||
bat "cd source\\tools\\i18n && python update_templates.py"
|
||||
withCredentials([string(credentialsId: 'TX_TOKEN', variable: 'TX_TOKEN')]) {
|
||||
bat "cd source\\tools\\i18n && python pullTranslations.py"
|
||||
bat "cd source\\tools\\i18n && python pull_translations.py"
|
||||
}
|
||||
bat "cd source\\tools\\i18n && python generateDebugTranslation.py --long"
|
||||
bat "cd source\\tools\\i18n && python cleanTranslationFiles.py"
|
||||
bat "cd source\\tools\\i18n && python generate_debug_translation.py --long"
|
||||
bat "cd source\\tools\\i18n && python clean_translation_files.py"
|
||||
script { if (!params.NEW_REPO) {
|
||||
bat "python source\\tools\\i18n\\checkDiff.py --verbose"
|
||||
bat "python source\\tools\\i18n\\check_diff.py --verbose"
|
||||
}}
|
||||
bat "cd source\\tools\\i18n && python creditTranslators.py"
|
||||
bat "cd source\\tools\\i18n && python credit_translators.py"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -23,12 +23,12 @@ import os
|
||||
import subprocess
|
||||
from typing import List
|
||||
|
||||
from i18n_helper import projectRootDirectory
|
||||
from i18n_helper import PROJECT_ROOT_DIRECTORY
|
||||
|
||||
|
||||
def get_diff():
|
||||
"""Return a diff using svn diff."""
|
||||
os.chdir(projectRootDirectory)
|
||||
os.chdir(PROJECT_ROOT_DIRECTORY)
|
||||
|
||||
diff_process = subprocess.run(["svn", "diff", "binaries"], capture_output=True, check=False)
|
||||
if diff_process.returncode != 0:
|
||||
@ -21,9 +21,9 @@ import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from i18n_helper import l10nFolderName, projectRootDirectory
|
||||
from i18n_helper import L10N_FOLDER_NAME, PROJECT_ROOT_DIRECTORY
|
||||
from i18n_helper.catalog import Catalog
|
||||
from i18n_helper.globber import getCatalogs
|
||||
from i18n_helper.globber import get_catalogs
|
||||
|
||||
|
||||
VERBOSE = 0
|
||||
@ -36,76 +36,76 @@ class MessageChecker:
|
||||
self.regex = re.compile(regex, re.IGNORECASE)
|
||||
self.human_name = human_name
|
||||
|
||||
def check(self, inputFilePath, templateMessage, translatedCatalogs):
|
||||
def check(self, input_file_path, template_message, translated_catalogs):
|
||||
patterns = set(
|
||||
self.regex.findall(
|
||||
templateMessage.id[0] if templateMessage.pluralizable else templateMessage.id
|
||||
template_message.id[0] if template_message.pluralizable else template_message.id
|
||||
)
|
||||
)
|
||||
|
||||
# As a sanity check, verify that the template message is coherent.
|
||||
# Note that these tend to be false positives.
|
||||
# TODO: the pssible tags are usually comments, we ought be able to find them.
|
||||
if templateMessage.pluralizable:
|
||||
pluralUrls = set(self.regex.findall(templateMessage.id[1]))
|
||||
if pluralUrls.difference(patterns):
|
||||
if template_message.pluralizable:
|
||||
plural_urls = set(self.regex.findall(template_message.id[1]))
|
||||
if plural_urls.difference(patterns):
|
||||
print(
|
||||
f"{inputFilePath} - Different {self.human_name} in "
|
||||
f"{input_file_path} - Different {self.human_name} in "
|
||||
f"singular and plural source strings "
|
||||
f"for '{templateMessage}' in '{inputFilePath}'"
|
||||
f"for '{template_message}' in '{input_file_path}'"
|
||||
)
|
||||
|
||||
for translationCatalog in translatedCatalogs:
|
||||
translationMessage = translationCatalog.get(
|
||||
templateMessage.id, templateMessage.context
|
||||
for translation_catalog in translated_catalogs:
|
||||
translation_message = translation_catalog.get(
|
||||
template_message.id, template_message.context
|
||||
)
|
||||
if not translationMessage:
|
||||
if not translation_message:
|
||||
continue
|
||||
|
||||
translatedPatterns = set(
|
||||
translated_patterns = set(
|
||||
self.regex.findall(
|
||||
translationMessage.string[0]
|
||||
if translationMessage.pluralizable
|
||||
else translationMessage.string
|
||||
translation_message.string[0]
|
||||
if translation_message.pluralizable
|
||||
else translation_message.string
|
||||
)
|
||||
)
|
||||
unknown_patterns = translatedPatterns.difference(patterns)
|
||||
unknown_patterns = translated_patterns.difference(patterns)
|
||||
if unknown_patterns:
|
||||
print(
|
||||
f'{inputFilePath} - {translationCatalog.locale}: '
|
||||
f'{input_file_path} - {translation_catalog.locale}: '
|
||||
f'Found unknown {self.human_name} '
|
||||
f'{", ".join(["`" + x + "`" for x in unknown_patterns])} '
|
||||
f'in the translation which do not match any of the URLs '
|
||||
f'in the template: {", ".join(["`" + x + "`" for x in patterns])}'
|
||||
)
|
||||
|
||||
if templateMessage.pluralizable and translationMessage.pluralizable:
|
||||
for indx, val in enumerate(translationMessage.string):
|
||||
if template_message.pluralizable and translation_message.pluralizable:
|
||||
for indx, val in enumerate(translation_message.string):
|
||||
if indx == 0:
|
||||
continue
|
||||
translatedPatternsMulti = set(self.regex.findall(val))
|
||||
unknown_patterns_multi = translatedPatternsMulti.difference(pluralUrls)
|
||||
translated_patterns_multi = set(self.regex.findall(val))
|
||||
unknown_patterns_multi = translated_patterns_multi.difference(plural_urls)
|
||||
if unknown_patterns_multi:
|
||||
print(
|
||||
f'{inputFilePath} - {translationCatalog.locale}: '
|
||||
f'{input_file_path} - {translation_catalog.locale}: '
|
||||
f'Found unknown {self.human_name} '
|
||||
f'{", ".join(["`" + x + "`" for x in unknown_patterns_multi])} '
|
||||
f'in the pluralised translation which do not '
|
||||
f'match any of the URLs in the template: '
|
||||
f'{", ".join(["`" + x + "`" for x in pluralUrls])}'
|
||||
f'{", ".join(["`" + x + "`" for x in plural_urls])}'
|
||||
)
|
||||
|
||||
|
||||
def check_translations(inputFilePath):
|
||||
def check_translations(input_file_path):
|
||||
if VERBOSE:
|
||||
print(f"Checking {inputFilePath}")
|
||||
templateCatalog = Catalog.readFrom(inputFilePath)
|
||||
print(f"Checking {input_file_path}")
|
||||
template_catalog = Catalog.read_from(input_file_path)
|
||||
|
||||
# If language codes were specified on the command line, filter by those.
|
||||
filters = sys.argv[1:]
|
||||
|
||||
# Load existing translation catalogs.
|
||||
existingTranslationCatalogs = getCatalogs(inputFilePath, filters)
|
||||
existing_translation_catalogs = get_catalogs(input_file_path, filters)
|
||||
|
||||
spam = MessageChecker("url", r"https?://(?:[a-z0-9-_$@./&+]|(?:%[0-9a-fA-F][0-9a-fA-F]))+")
|
||||
sprintf = MessageChecker("sprintf", r"%\([^)]+\)s")
|
||||
@ -115,37 +115,37 @@ def check_translations(inputFilePath):
|
||||
# Loop through all messages in the .POT catalog for URLs.
|
||||
# For each, check for the corresponding key in the .PO catalogs.
|
||||
# If found, check that URLS in the .PO keys are the same as those in the .POT key.
|
||||
for templateMessage in templateCatalog:
|
||||
spam.check(inputFilePath, templateMessage, existingTranslationCatalogs)
|
||||
sprintf.check(inputFilePath, templateMessage, existingTranslationCatalogs)
|
||||
tags.check(inputFilePath, templateMessage, existingTranslationCatalogs)
|
||||
for template_message in template_catalog:
|
||||
spam.check(input_file_path, template_message, existing_translation_catalogs)
|
||||
sprintf.check(input_file_path, template_message, existing_translation_catalogs)
|
||||
tags.check(input_file_path, template_message, existing_translation_catalogs)
|
||||
|
||||
if VERBOSE:
|
||||
print(f"Done checking {inputFilePath}")
|
||||
print(f"Done checking {input_file_path}")
|
||||
|
||||
|
||||
def main():
|
||||
print(
|
||||
"\n\tWARNING: Remember to regenerate the POT files with “updateTemplates.py” "
|
||||
"\n\tWARNING: Remember to regenerate the POT files with “update_templates.py” "
|
||||
"before you run this script.\n\tPOT files are not in the repository.\n"
|
||||
)
|
||||
foundPots = 0
|
||||
for root, _folders, filenames in os.walk(projectRootDirectory):
|
||||
found_pots = 0
|
||||
for root, _folders, filenames in os.walk(PROJECT_ROOT_DIRECTORY):
|
||||
for filename in filenames:
|
||||
if (
|
||||
len(filename) > 4
|
||||
and filename[-4:] == ".pot"
|
||||
and os.path.basename(root) == l10nFolderName
|
||||
and os.path.basename(root) == L10N_FOLDER_NAME
|
||||
):
|
||||
foundPots += 1
|
||||
found_pots += 1
|
||||
multiprocessing.Process(
|
||||
target=check_translations, args=(os.path.join(root, filename),)
|
||||
).start()
|
||||
if foundPots == 0:
|
||||
if found_pots == 0:
|
||||
print(
|
||||
"This script did not work because no '.pot' files were found. "
|
||||
"Please run 'updateTemplates.py' to generate the '.pot' files, "
|
||||
"and run 'pullTranslations.py' to pull the latest translations from Transifex. "
|
||||
"Please run 'update_templates.py' to generate the '.pot' files, "
|
||||
"and run 'pull_translations.py' to pull the latest translations from Transifex. "
|
||||
"Then you can run this script to check for spam in translations."
|
||||
)
|
||||
|
||||
@ -33,19 +33,19 @@ import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from i18n_helper import l10nFolderName, projectRootDirectory, transifexClientFolder
|
||||
from i18n_helper import L10N_FOLDER_NAME, PROJECT_ROOT_DIRECTORY, TRANSIFEX_CLIENT_FOLDER
|
||||
|
||||
|
||||
def main():
|
||||
translatorMatch = re.compile(r"^(#\s+[^,<]*)\s+<.*>(.*)")
|
||||
lastTranslatorMatch = re.compile(r"^(\"Last-Translator:[^,<]*)\s+<.*>(.*)")
|
||||
translator_match = re.compile(r"^(#\s+[^,<]*)\s+<.*>(.*)")
|
||||
last_translator_match = re.compile(r"^(\"Last-Translator:[^,<]*)\s+<.*>(.*)")
|
||||
|
||||
for root, folders, _ in os.walk(projectRootDirectory):
|
||||
for root, folders, _ in os.walk(PROJECT_ROOT_DIRECTORY):
|
||||
for folder in folders:
|
||||
if folder != l10nFolderName:
|
||||
if folder != L10N_FOLDER_NAME:
|
||||
continue
|
||||
|
||||
if not os.path.exists(os.path.join(root, folder, transifexClientFolder)):
|
||||
if not os.path.exists(os.path.join(root, folder, TRANSIFEX_CLIENT_FOLDER)):
|
||||
continue
|
||||
|
||||
path = os.path.join(root, folder, "*.po")
|
||||
@ -59,16 +59,16 @@ def main():
|
||||
if reached:
|
||||
if line == "# \n":
|
||||
line = ""
|
||||
m = translatorMatch.match(line)
|
||||
m = translator_match.match(line)
|
||||
if m:
|
||||
if m.group(1) in usernames:
|
||||
line = ""
|
||||
else:
|
||||
line = m.group(1) + m.group(2) + "\n"
|
||||
usernames.append(m.group(1))
|
||||
m2 = lastTranslatorMatch.match(line)
|
||||
m2 = last_translator_match.match(line)
|
||||
if m2:
|
||||
line = re.sub(lastTranslatorMatch, r"\1\2", line)
|
||||
line = re.sub(last_translator_match, r"\1\2", line)
|
||||
elif line.strip() == "# Translators:":
|
||||
reached = True
|
||||
sys.stdout.write(line)
|
||||
@ -27,7 +27,7 @@ automatic deletion. This has not been needed so far. A possibility would be to a
|
||||
optional boolean entry to the dictionary containing the name.
|
||||
|
||||
Translatable strings will be extracted from the generated file, so this should be run
|
||||
once before updateTemplates.py.
|
||||
once before update_templates.py.
|
||||
"""
|
||||
|
||||
import json
|
||||
@ -37,20 +37,20 @@ from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from babel import Locale, UnknownLocaleError
|
||||
from i18n_helper import l10nFolderName, projectRootDirectory, transifexClientFolder
|
||||
from i18n_helper import L10N_FOLDER_NAME, PROJECT_ROOT_DIRECTORY, TRANSIFEX_CLIENT_FOLDER
|
||||
|
||||
|
||||
poLocations = []
|
||||
for root, folders, _filenames in os.walk(projectRootDirectory):
|
||||
po_locations = []
|
||||
for root, folders, _filenames in os.walk(PROJECT_ROOT_DIRECTORY):
|
||||
for folder in folders:
|
||||
if folder != l10nFolderName:
|
||||
if folder != L10N_FOLDER_NAME:
|
||||
continue
|
||||
|
||||
if os.path.exists(os.path.join(root, folder, transifexClientFolder)):
|
||||
poLocations.append(os.path.join(root, folder))
|
||||
if os.path.exists(os.path.join(root, folder, TRANSIFEX_CLIENT_FOLDER)):
|
||||
po_locations.append(os.path.join(root, folder))
|
||||
|
||||
creditsLocation = os.path.join(
|
||||
projectRootDirectory,
|
||||
credits_location = os.path.join(
|
||||
PROJECT_ROOT_DIRECTORY,
|
||||
"binaries",
|
||||
"data",
|
||||
"mods",
|
||||
@ -62,19 +62,19 @@ creditsLocation = os.path.join(
|
||||
)
|
||||
|
||||
# This dictionary will hold creditors lists for each language, indexed by code
|
||||
langsLists = defaultdict(list)
|
||||
langs_lists = defaultdict(list)
|
||||
|
||||
# Create the new JSON data
|
||||
newJSONData = {"Title": "Translators", "Content": []}
|
||||
new_json_data = {"Title": "Translators", "Content": []}
|
||||
|
||||
# Now go through the list of languages and search the .po files for people
|
||||
|
||||
# Prepare some regexes
|
||||
translatorMatch = re.compile(r"^#\s+([^,<]*)")
|
||||
deletedUsernameMatch = re.compile(r"[0-9a-f]{32}(_[0-9a-f]{7})?")
|
||||
translator_match = re.compile(r"^#\s+([^,<]*)")
|
||||
deleted_username_match = re.compile(r"[0-9a-f]{32}(_[0-9a-f]{7})?")
|
||||
|
||||
# Search
|
||||
for location in poLocations:
|
||||
for location in po_locations:
|
||||
files = Path(location).glob("*.po")
|
||||
|
||||
for file in files:
|
||||
@ -84,46 +84,46 @@ for location in poLocations:
|
||||
if lang in ("debug", "long"):
|
||||
continue
|
||||
|
||||
with file.open(encoding="utf-8") as poFile:
|
||||
with file.open(encoding="utf-8") as po_file:
|
||||
reached = False
|
||||
for line in poFile:
|
||||
for line in po_file:
|
||||
if reached:
|
||||
m = translatorMatch.match(line)
|
||||
m = translator_match.match(line)
|
||||
if not m:
|
||||
break
|
||||
|
||||
username = m.group(1)
|
||||
if not deletedUsernameMatch.fullmatch(username):
|
||||
langsLists[lang].append(username)
|
||||
if not deleted_username_match.fullmatch(username):
|
||||
langs_lists[lang].append(username)
|
||||
if line.strip() == "# Translators:":
|
||||
reached = True
|
||||
|
||||
# Sort translator names and remove duplicates
|
||||
# Sorting should ignore case, but prefer versions of names starting
|
||||
# with an upper case letter to have a neat credits list.
|
||||
for lang in langsLists:
|
||||
for lang in langs_lists:
|
||||
translators = {}
|
||||
for name in sorted(langsLists[lang], reverse=True):
|
||||
for name in sorted(langs_lists[lang], reverse=True):
|
||||
if name.lower() not in translators or name.istitle():
|
||||
translators[name.lower()] = name
|
||||
langsLists[lang] = sorted(translators.values(), key=lambda s: s.lower())
|
||||
langs_lists[lang] = sorted(translators.values(), key=lambda s: s.lower())
|
||||
|
||||
# Now insert the new data into the new JSON file
|
||||
for langCode, langList in sorted(langsLists.items()):
|
||||
for lang_code, lang_list in sorted(langs_lists.items()):
|
||||
try:
|
||||
lang_name = Locale.parse(langCode).english_name
|
||||
lang_name = Locale.parse(lang_code).english_name
|
||||
except UnknownLocaleError:
|
||||
lang_name = Locale.parse("en").languages.get(langCode)
|
||||
lang_name = Locale.parse("en").languages.get(lang_code)
|
||||
|
||||
if not lang_name:
|
||||
raise
|
||||
|
||||
translators = [{"name": name} for name in langList]
|
||||
newJSONData["Content"].append({"LangName": lang_name, "List": translators})
|
||||
translators = [{"name": name} for name in lang_list]
|
||||
new_json_data["Content"].append({"LangName": lang_name, "List": translators})
|
||||
|
||||
# Sort languages by their English names
|
||||
newJSONData["Content"] = sorted(newJSONData["Content"], key=lambda x: x["LangName"])
|
||||
new_json_data["Content"] = sorted(new_json_data["Content"], key=lambda x: x["LangName"])
|
||||
|
||||
# Save the JSON data to the credits file
|
||||
with open(creditsLocation, "w", encoding="utf-8") as creditsFile:
|
||||
json.dump(newJSONData, creditsFile, indent=4)
|
||||
with open(credits_location, "w", encoding="utf-8") as credits_file:
|
||||
json.dump(new_json_data, credits_file, indent=4)
|
||||
@ -24,7 +24,7 @@
|
||||
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
import codecs
|
||||
import json as jsonParser
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
@ -54,8 +54,8 @@ def pathmatch(mask, path):
|
||||
|
||||
|
||||
class Extractor:
|
||||
def __init__(self, directoryPath, filemasks, options):
|
||||
self.directoryPath = directoryPath
|
||||
def __init__(self, directory_path, filemasks, options):
|
||||
self.directoryPath = directory_path
|
||||
self.options = options
|
||||
|
||||
if isinstance(filemasks, dict):
|
||||
@ -73,8 +73,8 @@ class Extractor:
|
||||
:rtype: ``iterator``
|
||||
"""
|
||||
empty_string_pattern = re.compile(r"^\s*$")
|
||||
directoryAbsolutePath = os.path.abspath(self.directoryPath)
|
||||
for root, folders, filenames in os.walk(directoryAbsolutePath):
|
||||
directory_absolute_path = os.path.abspath(self.directoryPath)
|
||||
for root, folders, filenames in os.walk(directory_absolute_path):
|
||||
for subdir in folders:
|
||||
if subdir.startswith((".", "_")):
|
||||
folders.remove(subdir)
|
||||
@ -90,14 +90,14 @@ class Extractor:
|
||||
else:
|
||||
for filemask in self.includeMasks:
|
||||
if pathmatch(filemask, filename):
|
||||
filepath = os.path.join(directoryAbsolutePath, filename)
|
||||
filepath = os.path.join(directory_absolute_path, filename)
|
||||
for (
|
||||
message,
|
||||
plural,
|
||||
context,
|
||||
position,
|
||||
comments,
|
||||
) in self.extractFromFile(filepath):
|
||||
) in self.extract_from_file(filepath):
|
||||
if empty_string_pattern.match(message):
|
||||
continue
|
||||
|
||||
@ -105,7 +105,7 @@ class Extractor:
|
||||
filename = "\u2068" + filename + "\u2069"
|
||||
yield message, plural, context, (filename, position), comments
|
||||
|
||||
def extractFromFile(self, filepath):
|
||||
def extract_from_file(self, filepath):
|
||||
"""Extract messages from a specific file.
|
||||
|
||||
:return: An iterator over ``(message, plural, context, position, comments)`` tuples.
|
||||
@ -113,7 +113,7 @@ class Extractor:
|
||||
"""
|
||||
|
||||
|
||||
class javascript(Extractor):
|
||||
class JavascriptExtractor(Extractor):
|
||||
"""Extract messages from JavaScript source code."""
|
||||
|
||||
empty_msgid_warning = (
|
||||
@ -121,7 +121,7 @@ class javascript(Extractor):
|
||||
"returns the header entry with meta information, not the empty string."
|
||||
)
|
||||
|
||||
def extractJavascriptFromFile(self, fileObject):
|
||||
def extract_javascript_from_file(self, file_object):
|
||||
from babel.messages.jslexer import tokenize, unquote_string
|
||||
|
||||
funcname = message_lineno = None
|
||||
@ -134,7 +134,7 @@ class javascript(Extractor):
|
||||
comment_tags = self.options.get("commentTags", [])
|
||||
keywords = self.options.get("keywords", {}).keys()
|
||||
|
||||
for token in tokenize(fileObject.read(), dotted=False):
|
||||
for token in tokenize(file_object.read(), dotted=False):
|
||||
if token.type == "operator" and (
|
||||
token.value == "(" or (call_stack != -1 and (token.value in ("[", "{")))
|
||||
):
|
||||
@ -236,9 +236,11 @@ class javascript(Extractor):
|
||||
|
||||
last_token = token
|
||||
|
||||
def extractFromFile(self, filepath):
|
||||
with codecs.open(filepath, "r", encoding="utf-8-sig") as fileObject:
|
||||
for lineno, funcname, messages, comments in self.extractJavascriptFromFile(fileObject):
|
||||
def extract_from_file(self, filepath):
|
||||
with codecs.open(filepath, "r", encoding="utf-8-sig") as file_object:
|
||||
for lineno, funcname, messages, comments in self.extract_javascript_from_file(
|
||||
file_object
|
||||
):
|
||||
spec = self.options.get("keywords", {})[funcname] or (1,) if funcname else (1,)
|
||||
if not isinstance(messages, (list, tuple)):
|
||||
messages = [messages]
|
||||
@ -276,7 +278,7 @@ class javascript(Extractor):
|
||||
if not messages[first_msg_index]:
|
||||
# An empty string msgid isn't valid, emit a warning
|
||||
where = "%s:%i" % (
|
||||
hasattr(fileObject, "name") and fileObject.name or "(unknown)",
|
||||
hasattr(file_object, "name") and file_object.name or "(unknown)",
|
||||
lineno,
|
||||
)
|
||||
print(self.empty_msgid_warning % where, file=sys.stderr)
|
||||
@ -291,54 +293,54 @@ class javascript(Extractor):
|
||||
yield message, plural, context, lineno, comments
|
||||
|
||||
|
||||
class cpp(javascript):
|
||||
class CppExtractor(JavascriptExtractor):
|
||||
"""Extract messages from C++ source code."""
|
||||
|
||||
|
||||
class txt(Extractor):
|
||||
class TXTExtractor(Extractor):
|
||||
"""Extract messages from plain text files."""
|
||||
|
||||
def extractFromFile(self, filepath):
|
||||
with codecs.open(filepath, "r", encoding="utf-8-sig") as fileObject:
|
||||
def extract_from_file(self, filepath):
|
||||
with codecs.open(filepath, "r", encoding="utf-8-sig") as file_object:
|
||||
for lineno, line in enumerate(
|
||||
[line.strip("\n\r") for line in fileObject.readlines()], start=1
|
||||
[line.strip("\n\r") for line in file_object.readlines()], start=1
|
||||
):
|
||||
if line:
|
||||
yield line, None, None, lineno, []
|
||||
|
||||
|
||||
class json(Extractor):
|
||||
class JsonExtractor(Extractor):
|
||||
"""Extract messages from JSON files."""
|
||||
|
||||
def __init__(self, directoryPath=None, filemasks=None, options=None):
|
||||
def __init__(self, directory_path=None, filemasks=None, options=None):
|
||||
if options is None:
|
||||
options = {}
|
||||
if filemasks is None:
|
||||
filemasks = []
|
||||
super().__init__(directoryPath, filemasks, options)
|
||||
super().__init__(directory_path, filemasks, options)
|
||||
self.keywords = self.options.get("keywords", {})
|
||||
self.context = self.options.get("context", None)
|
||||
self.comments = self.options.get("comments", [])
|
||||
|
||||
def setOptions(self, options):
|
||||
def set_options(self, options):
|
||||
self.options = options
|
||||
self.keywords = self.options.get("keywords", {})
|
||||
self.context = self.options.get("context", None)
|
||||
self.comments = self.options.get("comments", [])
|
||||
|
||||
def extractFromFile(self, filepath):
|
||||
with codecs.open(filepath, "r", "utf-8") as fileObject:
|
||||
for message, context in self.extractFromString(fileObject.read()):
|
||||
def extract_from_file(self, filepath):
|
||||
with codecs.open(filepath, "r", "utf-8") as file_object:
|
||||
for message, context in self.extract_from_string(file_object.read()):
|
||||
yield message, None, context, None, self.comments
|
||||
|
||||
def extractFromString(self, string):
|
||||
jsonDocument = jsonParser.loads(string)
|
||||
if isinstance(jsonDocument, list):
|
||||
for message, context in self.parseList(jsonDocument):
|
||||
def extract_from_string(self, string):
|
||||
json_document = json.loads(string)
|
||||
if isinstance(json_document, list):
|
||||
for message, context in self.parse_list(json_document):
|
||||
if message: # Skip empty strings.
|
||||
yield message, context
|
||||
elif isinstance(jsonDocument, dict):
|
||||
for message, context in self.parseDictionary(jsonDocument):
|
||||
elif isinstance(json_document, dict):
|
||||
for message, context in self.parse_dictionary(json_document):
|
||||
if message: # Skip empty strings.
|
||||
yield message, context
|
||||
else:
|
||||
@ -347,22 +349,22 @@ class json(Extractor):
|
||||
"You must extend the JSON extractor to support it."
|
||||
)
|
||||
|
||||
def parseList(self, itemsList):
|
||||
for listItem in itemsList:
|
||||
if isinstance(listItem, list):
|
||||
for message, context in self.parseList(listItem):
|
||||
def parse_list(self, items_list):
|
||||
for list_item in items_list:
|
||||
if isinstance(list_item, list):
|
||||
for message, context in self.parse_list(list_item):
|
||||
yield message, context
|
||||
elif isinstance(listItem, dict):
|
||||
for message, context in self.parseDictionary(listItem):
|
||||
elif isinstance(list_item, dict):
|
||||
for message, context in self.parse_dictionary(list_item):
|
||||
yield message, context
|
||||
|
||||
def parseDictionary(self, dictionary):
|
||||
def parse_dictionary(self, dictionary):
|
||||
for keyword in dictionary:
|
||||
if keyword in self.keywords:
|
||||
if isinstance(dictionary[keyword], str):
|
||||
yield self.extractString(dictionary[keyword], keyword)
|
||||
yield self.extract_string(dictionary[keyword], keyword)
|
||||
elif isinstance(dictionary[keyword], list):
|
||||
for message, context in self.extractList(dictionary[keyword], keyword):
|
||||
for message, context in self.extract_list(dictionary[keyword], keyword):
|
||||
yield message, context
|
||||
elif isinstance(dictionary[keyword], dict):
|
||||
extract = None
|
||||
@ -370,22 +372,22 @@ class json(Extractor):
|
||||
"extractFromInnerKeys" in self.keywords[keyword]
|
||||
and self.keywords[keyword]["extractFromInnerKeys"]
|
||||
):
|
||||
for message, context in self.extractDictionaryInnerKeys(
|
||||
for message, context in self.extract_dictionary_inner_keys(
|
||||
dictionary[keyword], keyword
|
||||
):
|
||||
yield message, context
|
||||
else:
|
||||
extract = self.extractDictionary(dictionary[keyword], keyword)
|
||||
extract = self.extract_dictionary(dictionary[keyword], keyword)
|
||||
if extract:
|
||||
yield extract
|
||||
elif isinstance(dictionary[keyword], list):
|
||||
for message, context in self.parseList(dictionary[keyword]):
|
||||
for message, context in self.parse_list(dictionary[keyword]):
|
||||
yield message, context
|
||||
elif isinstance(dictionary[keyword], dict):
|
||||
for message, context in self.parseDictionary(dictionary[keyword]):
|
||||
for message, context in self.parse_dictionary(dictionary[keyword]):
|
||||
yield message, context
|
||||
|
||||
def extractString(self, string, keyword):
|
||||
def extract_string(self, string, keyword):
|
||||
context = None
|
||||
if "tagAsContext" in self.keywords[keyword]:
|
||||
context = keyword
|
||||
@ -395,16 +397,16 @@ class json(Extractor):
|
||||
context = self.context
|
||||
return string, context
|
||||
|
||||
def extractList(self, itemsList, keyword):
|
||||
for listItem in itemsList:
|
||||
if isinstance(listItem, str):
|
||||
yield self.extractString(listItem, keyword)
|
||||
elif isinstance(listItem, dict):
|
||||
extract = self.extractDictionary(listItem[keyword], keyword)
|
||||
def extract_list(self, items_list, keyword):
|
||||
for list_item in items_list:
|
||||
if isinstance(list_item, str):
|
||||
yield self.extract_string(list_item, keyword)
|
||||
elif isinstance(list_item, dict):
|
||||
extract = self.extract_dictionary(list_item[keyword], keyword)
|
||||
if extract:
|
||||
yield extract
|
||||
|
||||
def extractDictionary(self, dictionary, keyword):
|
||||
def extract_dictionary(self, dictionary, keyword):
|
||||
message = dictionary.get("_string", None)
|
||||
if message and isinstance(message, str):
|
||||
context = None
|
||||
@ -419,45 +421,47 @@ class json(Extractor):
|
||||
return message, context
|
||||
return None
|
||||
|
||||
def extractDictionaryInnerKeys(self, dictionary, keyword):
|
||||
for innerKeyword in dictionary:
|
||||
if isinstance(dictionary[innerKeyword], str):
|
||||
yield self.extractString(dictionary[innerKeyword], keyword)
|
||||
elif isinstance(dictionary[innerKeyword], list):
|
||||
yield from self.extractList(dictionary[innerKeyword], keyword)
|
||||
elif isinstance(dictionary[innerKeyword], dict):
|
||||
extract = self.extractDictionary(dictionary[innerKeyword], keyword)
|
||||
def extract_dictionary_inner_keys(self, dictionary, keyword):
|
||||
for inner_keyword in dictionary:
|
||||
if isinstance(dictionary[inner_keyword], str):
|
||||
yield self.extract_string(dictionary[inner_keyword], keyword)
|
||||
elif isinstance(dictionary[inner_keyword], list):
|
||||
yield from self.extract_list(dictionary[inner_keyword], keyword)
|
||||
elif isinstance(dictionary[inner_keyword], dict):
|
||||
extract = self.extract_dictionary(dictionary[inner_keyword], keyword)
|
||||
if extract:
|
||||
yield extract
|
||||
|
||||
|
||||
class xml(Extractor):
|
||||
class XmlExtractor(Extractor):
|
||||
"""Extract messages from XML files."""
|
||||
|
||||
def __init__(self, directoryPath, filemasks, options):
|
||||
super().__init__(directoryPath, filemasks, options)
|
||||
def __init__(self, directory_path, filemasks, options):
|
||||
super().__init__(directory_path, filemasks, options)
|
||||
self.keywords = self.options.get("keywords", {})
|
||||
self.jsonExtractor = None
|
||||
|
||||
def getJsonExtractor(self):
|
||||
def get_json_extractor(self):
|
||||
if not self.jsonExtractor:
|
||||
self.jsonExtractor = json()
|
||||
self.jsonExtractor = JsonExtractor()
|
||||
return self.jsonExtractor
|
||||
|
||||
def extractFromFile(self, filepath):
|
||||
def extract_from_file(self, filepath):
|
||||
from lxml import etree
|
||||
|
||||
with codecs.open(filepath, "r", encoding="utf-8-sig") as fileObject:
|
||||
xmlDocument = etree.parse(fileObject)
|
||||
with codecs.open(filepath, "r", encoding="utf-8-sig") as file_object:
|
||||
xml_document = etree.parse(file_object)
|
||||
for keyword in self.keywords:
|
||||
for element in xmlDocument.iter(keyword):
|
||||
for element in xml_document.iter(keyword):
|
||||
lineno = element.sourceline
|
||||
if element.text is not None:
|
||||
comments = []
|
||||
if "extractJson" in self.keywords[keyword]:
|
||||
jsonExtractor = self.getJsonExtractor()
|
||||
jsonExtractor.setOptions(self.keywords[keyword]["extractJson"])
|
||||
for message, context in jsonExtractor.extractFromString(element.text):
|
||||
json_extractor = self.get_json_extractor()
|
||||
json_extractor.set_options(self.keywords[keyword]["extractJson"])
|
||||
for message, context in json_extractor.extract_from_string(
|
||||
element.text
|
||||
):
|
||||
yield message, None, context, lineno, comments
|
||||
else:
|
||||
context = None
|
||||
@ -474,12 +478,12 @@ class xml(Extractor):
|
||||
) # Remove tabs, line breaks and unecessary spaces.
|
||||
comments.append(comment)
|
||||
if "splitOnWhitespace" in self.keywords[keyword]:
|
||||
for splitText in element.text.split():
|
||||
for split_text in element.text.split():
|
||||
# split on whitespace is used for token lists, there, a
|
||||
# leading '-' means the token has to be removed, so it's not
|
||||
# to be processed here either
|
||||
if splitText[0] != "-":
|
||||
yield str(splitText), None, context, lineno, comments
|
||||
if split_text[0] != "-":
|
||||
yield str(split_text), None, context, lineno, comments
|
||||
else:
|
||||
yield str(element.text), None, context, lineno, comments
|
||||
|
||||
@ -500,14 +504,14 @@ class FakeSectionHeader:
|
||||
return self.fp.readline()
|
||||
|
||||
|
||||
class ini(Extractor):
|
||||
class IniExtractor(Extractor):
|
||||
"""Extract messages from INI files."""
|
||||
|
||||
def __init__(self, directoryPath, filemasks, options):
|
||||
super().__init__(directoryPath, filemasks, options)
|
||||
def __init__(self, directory_path, filemasks, options):
|
||||
super().__init__(directory_path, filemasks, options)
|
||||
self.keywords = self.options.get("keywords", [])
|
||||
|
||||
def extractFromFile(self, filepath):
|
||||
def extract_from_file(self, filepath):
|
||||
import ConfigParser
|
||||
|
||||
config = ConfigParser.RawConfigParser()
|
||||
|
||||
@ -21,9 +21,9 @@ import multiprocessing
|
||||
import os
|
||||
import sys
|
||||
|
||||
from i18n_helper import l10nFolderName, projectRootDirectory
|
||||
from i18n_helper import L10N_FOLDER_NAME, PROJECT_ROOT_DIRECTORY
|
||||
from i18n_helper.catalog import Catalog
|
||||
from i18n_helper.globber import getCatalogs
|
||||
from i18n_helper.globber import get_catalogs
|
||||
|
||||
|
||||
DEBUG_PREFIX = "X_X "
|
||||
@ -41,7 +41,7 @@ def generate_long_strings(root_path, input_file_name, output_file_name, language
|
||||
input_file_path = os.path.join(root_path, input_file_name)
|
||||
output_file_path = os.path.join(root_path, output_file_name)
|
||||
|
||||
template_catalog = Catalog.readFrom(input_file_path)
|
||||
template_catalog = Catalog.read_from(input_file_path)
|
||||
# Pretend we write English to get plurals.
|
||||
long_string_catalog = Catalog(locale="en")
|
||||
|
||||
@ -55,7 +55,7 @@ def generate_long_strings(root_path, input_file_name, output_file_name, language
|
||||
)
|
||||
|
||||
# Load existing translation catalogs.
|
||||
existing_translation_catalogs = getCatalogs(input_file_path, languages)
|
||||
existing_translation_catalogs = get_catalogs(input_file_path, languages)
|
||||
|
||||
# If any existing translation has more characters than the average expansion, use that instead.
|
||||
for translation_catalog in existing_translation_catalogs:
|
||||
@ -100,7 +100,7 @@ def generate_long_strings(root_path, input_file_name, output_file_name, language
|
||||
longest_plural_string,
|
||||
]
|
||||
translation_message = long_string_catalog_message
|
||||
long_string_catalog.writeTo(output_file_path)
|
||||
long_string_catalog.write_to(output_file_path)
|
||||
|
||||
|
||||
def generate_debug(root_path, input_file_name, output_file_name):
|
||||
@ -114,7 +114,7 @@ def generate_debug(root_path, input_file_name, output_file_name):
|
||||
input_file_path = os.path.join(root_path, input_file_name)
|
||||
output_file_path = os.path.join(root_path, output_file_name)
|
||||
|
||||
template_catalog = Catalog.readFrom(input_file_path)
|
||||
template_catalog = Catalog.read_from(input_file_path)
|
||||
# Pretend we write English to get plurals.
|
||||
out_catalog = Catalog(locale="en")
|
||||
|
||||
@ -134,7 +134,7 @@ def generate_debug(root_path, input_file_name, output_file_name):
|
||||
auto_comments=message.auto_comments,
|
||||
)
|
||||
|
||||
out_catalog.writeTo(output_file_path)
|
||||
out_catalog.write_to(output_file_path)
|
||||
|
||||
|
||||
def main():
|
||||
@ -159,12 +159,12 @@ def main():
|
||||
sys.exit(0)
|
||||
|
||||
found_pot_files = 0
|
||||
for root, _, filenames in os.walk(projectRootDirectory):
|
||||
for root, _, filenames in os.walk(PROJECT_ROOT_DIRECTORY):
|
||||
for filename in filenames:
|
||||
if (
|
||||
len(filename) > 4
|
||||
and filename[-4:] == ".pot"
|
||||
and os.path.basename(root) == l10nFolderName
|
||||
and os.path.basename(root) == L10N_FOLDER_NAME
|
||||
):
|
||||
found_pot_files += 1
|
||||
if args.debug:
|
||||
@ -180,8 +180,8 @@ def main():
|
||||
if found_pot_files == 0:
|
||||
print(
|
||||
"This script did not work because no '.pot' files were found. "
|
||||
"Please, run 'updateTemplates.py' to generate the '.pot' files, and run "
|
||||
"'pullTranslations.py' to pull the latest translations from Transifex. "
|
||||
"Please, run 'update_templates.py' to generate the '.pot' files, and run "
|
||||
"'pull_translations.py' to pull the latest translations from Transifex. "
|
||||
"Then you can run this script to generate '.po' files with obvious debug strings."
|
||||
)
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
import os
|
||||
|
||||
|
||||
l10nFolderName = "l10n"
|
||||
transifexClientFolder = ".tx"
|
||||
l10nToolsDirectory = os.path.dirname(os.path.realpath(__file__))
|
||||
projectRootDirectory = os.path.abspath(
|
||||
os.path.join(l10nToolsDirectory, os.pardir, os.pardir, os.pardir, os.pardir)
|
||||
L10N_FOLDER_NAME = "l10n"
|
||||
TRANSIFEX_CLIENT_FOLDER = ".tx"
|
||||
L10N_TOOLS_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
|
||||
PROJECT_ROOT_DIRECTORY = os.path.abspath(
|
||||
os.path.join(L10N_TOOLS_DIRECTORY, os.pardir, os.pardir, os.pardir, os.pardir)
|
||||
)
|
||||
|
||||
@ -44,10 +44,10 @@ class Catalog(BabelCatalog):
|
||||
return [("Project-Id-Version", self._project), *headers]
|
||||
|
||||
@staticmethod
|
||||
def readFrom(file_path, locale=None):
|
||||
def read_from(file_path, locale=None):
|
||||
with open(file_path, "r+", encoding="utf-8") as fd:
|
||||
return read_po(fd, locale=locale)
|
||||
|
||||
def writeTo(self, file_path):
|
||||
def write_to(self, file_path):
|
||||
with open(file_path, "wb+") as fd:
|
||||
return write_po(fileobj=fd, catalog=self, width=90, sort_by_file=True)
|
||||
|
||||
@ -6,22 +6,22 @@ from typing import List, Optional
|
||||
from i18n_helper.catalog import Catalog
|
||||
|
||||
|
||||
def getCatalogs(inputFilePath, filters: Optional[List[str]] = None) -> List[Catalog]:
|
||||
def get_catalogs(input_file_path, filters: Optional[List[str]] = None) -> List[Catalog]:
|
||||
"""Return a list of "real" catalogs (.po) in the given folder."""
|
||||
existingTranslationCatalogs = []
|
||||
l10nFolderPath = os.path.dirname(inputFilePath)
|
||||
inputFileName = os.path.basename(inputFilePath)
|
||||
existing_translation_catalogs = []
|
||||
l10n_folder_path = os.path.dirname(input_file_path)
|
||||
input_file_name = os.path.basename(input_file_path)
|
||||
|
||||
for filename in os.listdir(str(l10nFolderPath)):
|
||||
for filename in os.listdir(str(l10n_folder_path)):
|
||||
if filename.startswith("long") or not filename.endswith(".po"):
|
||||
continue
|
||||
if filename.split(".")[1] != inputFileName.split(".")[0]:
|
||||
if filename.split(".")[1] != input_file_name.split(".")[0]:
|
||||
continue
|
||||
if not filters or filename.split(".")[0] in filters:
|
||||
existingTranslationCatalogs.append(
|
||||
Catalog.readFrom(
|
||||
os.path.join(l10nFolderPath, filename), locale=filename.split(".")[0]
|
||||
existing_translation_catalogs.append(
|
||||
Catalog.read_from(
|
||||
os.path.join(l10n_folder_path, filename), locale=filename.split(".")[0]
|
||||
)
|
||||
)
|
||||
|
||||
return existingTranslationCatalogs
|
||||
return existing_translation_catalogs
|
||||
|
||||
@ -19,16 +19,16 @@
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from i18n_helper import l10nFolderName, projectRootDirectory, transifexClientFolder
|
||||
from i18n_helper import L10N_FOLDER_NAME, PROJECT_ROOT_DIRECTORY, TRANSIFEX_CLIENT_FOLDER
|
||||
|
||||
|
||||
def main():
|
||||
for root, folders, _ in os.walk(projectRootDirectory):
|
||||
for root, folders, _ in os.walk(PROJECT_ROOT_DIRECTORY):
|
||||
for folder in folders:
|
||||
if folder != l10nFolderName:
|
||||
if folder != L10N_FOLDER_NAME:
|
||||
continue
|
||||
|
||||
if os.path.exists(os.path.join(root, folder, transifexClientFolder)):
|
||||
if os.path.exists(os.path.join(root, folder, TRANSIFEX_CLIENT_FOLDER)):
|
||||
path = os.path.join(root, folder)
|
||||
os.chdir(path)
|
||||
print(f"INFO: Starting to pull translations in {path}...")
|
||||
@ -1,7 +1,7 @@
|
||||
import io
|
||||
|
||||
import pytest
|
||||
from checkDiff import check_diff
|
||||
from check_diff import check_diff
|
||||
|
||||
|
||||
PATCHES = [
|
||||
@ -1,132 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright (C) 2022 Wildfire Games.
|
||||
# This file is part of 0 A.D.
|
||||
#
|
||||
# 0 A.D. is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# 0 A.D. is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with 0 A.D. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
from importlib import import_module
|
||||
|
||||
from i18n_helper import l10nFolderName, projectRootDirectory
|
||||
from i18n_helper.catalog import Catalog
|
||||
|
||||
|
||||
messagesFilename = "messages.json"
|
||||
|
||||
|
||||
def warnAboutUntouchedMods():
|
||||
"""Warn about mods that are not properly configured to get their messages extracted."""
|
||||
modsRootFolder = os.path.join(projectRootDirectory, "binaries", "data", "mods")
|
||||
untouchedMods = {}
|
||||
for modFolder in os.listdir(modsRootFolder):
|
||||
if modFolder[0] != "_" and modFolder[0] != ".":
|
||||
if not os.path.exists(os.path.join(modsRootFolder, modFolder, l10nFolderName)):
|
||||
untouchedMods[modFolder] = (
|
||||
f"There is no '{l10nFolderName}' folder in the root folder of this mod."
|
||||
)
|
||||
elif not os.path.exists(
|
||||
os.path.join(modsRootFolder, modFolder, l10nFolderName, messagesFilename)
|
||||
):
|
||||
untouchedMods[modFolder] = (
|
||||
f"There is no '{messagesFilename}' file within the '{l10nFolderName}' folder "
|
||||
f"in the root folder of this mod."
|
||||
)
|
||||
if untouchedMods:
|
||||
print("Warning: No messages were extracted from the following mods:")
|
||||
for mod in untouchedMods:
|
||||
print(f"• {mod}: {untouchedMods[mod]}")
|
||||
print(
|
||||
""
|
||||
f"For this script to extract messages from a mod folder, this mod folder must contain "
|
||||
f"a '{l10nFolderName}' folder, and this folder must contain a '{messagesFilename}' "
|
||||
f"file that describes how to extract messages for the mod. See the folder of the main "
|
||||
f"mod ('public') for an example, and see the documentation for more information."
|
||||
)
|
||||
|
||||
|
||||
def generatePOT(templateSettings, rootPath):
|
||||
if "skip" in templateSettings and templateSettings["skip"] == "yes":
|
||||
return
|
||||
|
||||
inputRootPath = rootPath
|
||||
if "inputRoot" in templateSettings:
|
||||
inputRootPath = os.path.join(rootPath, templateSettings["inputRoot"])
|
||||
|
||||
template = Catalog(
|
||||
project=templateSettings["project"],
|
||||
copyright_holder=templateSettings["copyrightHolder"],
|
||||
locale="en",
|
||||
)
|
||||
|
||||
for rule in templateSettings["rules"]:
|
||||
if "skip" in rule and rule["skip"] == "yes":
|
||||
return
|
||||
|
||||
options = rule.get("options", {})
|
||||
extractorClass = getattr(import_module("extractors.extractors"), rule["extractor"])
|
||||
extractor = extractorClass(inputRootPath, rule["filemasks"], options)
|
||||
formatFlag = None
|
||||
if "format" in options:
|
||||
formatFlag = options["format"]
|
||||
for message, plural, context, location, comments in extractor.run():
|
||||
message_id = (message, plural) if plural else message
|
||||
|
||||
saved_message = template.get(message_id, context) or template.add(
|
||||
id=message_id,
|
||||
context=context,
|
||||
auto_comments=comments,
|
||||
flags=[formatFlag] if formatFlag and message.find("%") != -1 else [],
|
||||
)
|
||||
saved_message.locations.append(location)
|
||||
saved_message.flags.discard("python-format")
|
||||
|
||||
template.writeTo(os.path.join(rootPath, templateSettings["output"]))
|
||||
print('Generated "{}" with {} messages.'.format(templateSettings["output"], len(template)))
|
||||
|
||||
|
||||
def generateTemplatesForMessagesFile(messagesFilePath):
|
||||
with open(messagesFilePath, encoding="utf-8") as fileObject:
|
||||
settings = json.load(fileObject)
|
||||
|
||||
for templateSettings in settings:
|
||||
multiprocessing.Process(
|
||||
target=generatePOT, args=(templateSettings, os.path.dirname(messagesFilePath))
|
||||
).start()
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--scandir",
|
||||
help="Directory to start scanning for l10n folders in. "
|
||||
"Type '.' for current working directory",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
for root, folders, _filenames in os.walk(args.scandir or projectRootDirectory):
|
||||
for folder in folders:
|
||||
if folder == l10nFolderName:
|
||||
messagesFilePath = os.path.join(root, folder, messagesFilename)
|
||||
if os.path.exists(messagesFilePath):
|
||||
generateTemplatesForMessagesFile(messagesFilePath)
|
||||
|
||||
warnAboutUntouchedMods()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
134
source/tools/i18n/update_templates.py
Executable file
134
source/tools/i18n/update_templates.py
Executable file
@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright (C) 2022 Wildfire Games.
|
||||
# This file is part of 0 A.D.
|
||||
#
|
||||
# 0 A.D. is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# 0 A.D. is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with 0 A.D. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
from importlib import import_module
|
||||
|
||||
from i18n_helper import L10N_FOLDER_NAME, PROJECT_ROOT_DIRECTORY
|
||||
from i18n_helper.catalog import Catalog
|
||||
|
||||
|
||||
messages_filename = "messages.json"
|
||||
|
||||
|
||||
def warn_about_untouched_mods():
|
||||
"""Warn about mods that are not properly configured to get their messages extracted."""
|
||||
mods_root_folder = os.path.join(PROJECT_ROOT_DIRECTORY, "binaries", "data", "mods")
|
||||
untouched_mods = {}
|
||||
for mod_folder in os.listdir(mods_root_folder):
|
||||
if mod_folder[0] != "_" and mod_folder[0] != ".":
|
||||
if not os.path.exists(os.path.join(mods_root_folder, mod_folder, L10N_FOLDER_NAME)):
|
||||
untouched_mods[mod_folder] = (
|
||||
f"There is no '{L10N_FOLDER_NAME}' folder in the root folder of this mod."
|
||||
)
|
||||
elif not os.path.exists(
|
||||
os.path.join(mods_root_folder, mod_folder, L10N_FOLDER_NAME, messages_filename)
|
||||
):
|
||||
untouched_mods[mod_folder] = (
|
||||
f"There is no '{messages_filename}' file within the '{L10N_FOLDER_NAME}' "
|
||||
f"folder in the root folder of this mod."
|
||||
)
|
||||
if untouched_mods:
|
||||
print("Warning: No messages were extracted from the following mods:")
|
||||
for mod in untouched_mods:
|
||||
print(f"• {mod}: {untouched_mods[mod]}")
|
||||
print(
|
||||
""
|
||||
f"For this script to extract messages from a mod folder, this mod folder must contain "
|
||||
f"a '{L10N_FOLDER_NAME}' folder, and this folder must contain a '{messages_filename}' "
|
||||
f"file that describes how to extract messages for the mod. See the folder of the main "
|
||||
f"mod ('public') for an example, and see the documentation for more information."
|
||||
)
|
||||
|
||||
|
||||
def generate_pot(template_settings, root_path):
|
||||
if "skip" in template_settings and template_settings["skip"] == "yes":
|
||||
return
|
||||
|
||||
input_root_path = root_path
|
||||
if "inputRoot" in template_settings:
|
||||
input_root_path = os.path.join(root_path, template_settings["inputRoot"])
|
||||
|
||||
template = Catalog(
|
||||
project=template_settings["project"],
|
||||
copyright_holder=template_settings["copyrightHolder"],
|
||||
locale="en",
|
||||
)
|
||||
|
||||
for rule in template_settings["rules"]:
|
||||
if "skip" in rule and rule["skip"] == "yes":
|
||||
return
|
||||
|
||||
options = rule.get("options", {})
|
||||
extractor_class = getattr(
|
||||
import_module("extractors.extractors"), f'{rule["extractor"].title()}Extractor'
|
||||
)
|
||||
extractor = extractor_class(input_root_path, rule["filemasks"], options)
|
||||
format_flag = None
|
||||
if "format" in options:
|
||||
format_flag = options["format"]
|
||||
for message, plural, context, location, comments in extractor.run():
|
||||
message_id = (message, plural) if plural else message
|
||||
|
||||
saved_message = template.get(message_id, context) or template.add(
|
||||
id=message_id,
|
||||
context=context,
|
||||
auto_comments=comments,
|
||||
flags=[format_flag] if format_flag and message.find("%") != -1 else [],
|
||||
)
|
||||
saved_message.locations.append(location)
|
||||
saved_message.flags.discard("python-format")
|
||||
|
||||
template.write_to(os.path.join(root_path, template_settings["output"]))
|
||||
print('Generated "{}" with {} messages.'.format(template_settings["output"], len(template)))
|
||||
|
||||
|
||||
def generate_templates_for_messages_file(messages_file_path):
|
||||
with open(messages_file_path, encoding="utf-8") as file_object:
|
||||
settings = json.load(file_object)
|
||||
|
||||
for template_settings in settings:
|
||||
multiprocessing.Process(
|
||||
target=generate_pot, args=(template_settings, os.path.dirname(messages_file_path))
|
||||
).start()
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--scandir",
|
||||
help="Directory to start scanning for l10n folders in. "
|
||||
"Type '.' for current working directory",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
for root, folders, _filenames in os.walk(args.scandir or PROJECT_ROOT_DIRECTORY):
|
||||
for folder in folders:
|
||||
if folder == L10N_FOLDER_NAME:
|
||||
messages_file_path = os.path.join(root, folder, messages_filename)
|
||||
if os.path.exists(messages_file_path):
|
||||
generate_templates_for_messages_file(messages_file_path)
|
||||
|
||||
warn_about_untouched_mods()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user