From c49d4eedd02daec7af70ebd73a47c895e4332aaf Mon Sep 17 00:00:00 2001 From: Dunedan Date: Thu, 22 Aug 2024 09:18:20 +0200 Subject: [PATCH] Lint and format Python files using ruff To improve quality und uniformity of the included Python code this lints and formats the included Python files with ruff. --- source/collada/tests/tests.py | 192 ++--- source/tools/entity/checkrefs.py | 703 +++++++++++------- source/tools/entity/creationgraph.py | 53 +- source/tools/entity/entvalidate.py | 81 +- source/tools/entity/scriptlib/__init__.py | 60 +- source/tools/fontbuilder2/FontLoader.py | 47 +- source/tools/fontbuilder2/Packer.py | 129 ++-- source/tools/fontbuilder2/dumpfontchars.py | 11 +- source/tools/fontbuilder2/fontbuilder.py | 83 ++- source/tools/i18n/checkDiff.py | 38 +- source/tools/i18n/checkTranslations.py | 68 +- source/tools/i18n/cleanTranslationFiles.py | 12 +- source/tools/i18n/creditTranslators.py | 34 +- source/tools/i18n/extractors/extractors.py | 186 +++-- source/tools/i18n/generateDebugTranslation.py | 92 ++- source/tools/i18n/i18n_helper/__init__.py | 4 +- source/tools/i18n/i18n_helper/catalog.py | 19 +- source/tools/i18n/i18n_helper/globber.py | 9 +- source/tools/i18n/pullTranslations.py | 1 + source/tools/i18n/tests/test_checkDiff.py | 15 +- source/tools/i18n/updateTemplates.py | 67 +- source/tools/mapcompatibility/a18_to_a19.py | 208 +++--- .../rlclient/python/samples/simple-example.py | 47 +- source/tools/rlclient/python/setup.py | 23 +- .../rlclient/python/tests/test_actions.py | 53 +- .../rlclient/python/tests/test_evaluate.py | 36 +- .../tools/rlclient/python/zero_ad/__init__.py | 3 +- .../tools/rlclient/python/zero_ad/actions.py | 78 +- source/tools/rlclient/python/zero_ad/api.py | 24 +- .../rlclient/python/zero_ad/environment.py | 53 +- source/tools/spirv/compile.py | 575 ++++++++------ source/tools/templatesanalyzer/unitTables.py | 217 +++--- source/tools/xmlvalidator/validate_grammar.py | 159 ++-- source/tools/xmlvalidator/validator.py | 145 ++-- 34 files changed, 2051 insertions(+), 1474 deletions(-) diff --git a/source/collada/tests/tests.py b/source/collada/tests/tests.py index ba389a3707..49a49c5e7f 100644 --- a/source/collada/tests/tests.py +++ b/source/collada/tests/tests.py @@ -1,138 +1,158 @@ #!/usr/bin/env python3 +# ruff: noqa: F403, F405 + from ctypes import * -import sys import os import xml.etree.ElementTree as ET -binaries = '../../../binaries' +binaries = "../../../binaries" # Work out the platform-dependent library filename dll_filename = { - 'posix': './libCollada_dbg.so', - 'nt': 'Collada_dbg.dll', + "posix": "./libCollada_dbg.so", + "nt": "Collada_dbg.dll", }[os.name] # The DLL may need other DLLs which are in its directory, so set the path to that # (Don't care about clobbering the old PATH - it doesn't have anything important) -os.environ['PATH'] = '%s/system/' % binaries +os.environ["PATH"] = "%s/system/" % binaries # Load the actual library -library = cdll.LoadLibrary('%s/system/%s' % (binaries, dll_filename)) +library = cdll.LoadLibrary("%s/system/%s" % (binaries, dll_filename)) + def log(severity, message): - print('[%s] %s' % (('INFO', 'WARNING', 'ERROR')[severity], message)) + print("[%s] %s" % (("INFO", "WARNING", "ERROR")[severity], message)) + clog = CFUNCTYPE(None, c_int, c_char_p)(log) - # (the CFUNCTYPE must not be GC'd, so try to keep a reference) +# (the CFUNCTYPE must not be GC'd, so try to keep a reference) library.set_logger(clog) -skeleton_definitions = open('%s/data/tests/collada/skeletons.xml' % binaries).read() +skeleton_definitions = open("%s/data/tests/collada/skeletons.xml" % binaries).read() library.set_skeleton_definitions(skeleton_definitions, len(skeleton_definitions)) -def _convert_dae(func, filename, expected_status=0): - output = [] - def cb(cbdata, str, len): - output.append(string_at(str, len)) - cbtype = CFUNCTYPE(None, POINTER(None), POINTER(c_char), c_uint) - status = func(filename, cbtype(cb), None) - assert(status == expected_status) - return ''.join(output) +def _convert_dae(func, filename, expected_status=0): + output = [] + + def cb(cbdata, str, len): + output.append(string_at(str, len)) + + cbtype = CFUNCTYPE(None, POINTER(None), POINTER(c_char), c_uint) + status = func(filename, cbtype(cb), None) + assert status == expected_status + return "".join(output) + def convert_dae_to_pmd(*args, **kwargs): - return _convert_dae(library.convert_dae_to_pmd, *args, **kwargs) + return _convert_dae(library.convert_dae_to_pmd, *args, **kwargs) + def convert_dae_to_psa(*args, **kwargs): - return _convert_dae(library.convert_dae_to_psa, *args, **kwargs) + return _convert_dae(library.convert_dae_to_psa, *args, **kwargs) + def clean_dir(path): - # Remove all files first - try: - for f in os.listdir(path): - os.remove(path+'/'+f) - os.rmdir(path) - except OSError: - pass # (ignore errors if files are in use) - # Make sure the directory exists - try: - os.makedirs(path) - except OSError: - pass # (ignore errors if it already exists) + # Remove all files first + try: + for f in os.listdir(path): + os.remove(path + "/" + f) + os.rmdir(path) + except OSError: + pass # (ignore errors if files are in use) + # Make sure the directory exists + try: + os.makedirs(path) + except OSError: + pass # (ignore errors if it already exists) + def create_actor(mesh, texture, anims, props_): - actor = ET.Element('actor', version='1') - ET.SubElement(actor, 'castshadow') - group = ET.SubElement(actor, 'group') - variant = ET.SubElement(group, 'variant', frequency='100', name='Base') - ET.SubElement(variant, 'mesh').text = mesh+'.pmd' - ET.SubElement(variant, 'texture').text = texture+'.dds' + actor = ET.Element("actor", version="1") + ET.SubElement(actor, "castshadow") + group = ET.SubElement(actor, "group") + variant = ET.SubElement(group, "variant", frequency="100", name="Base") + ET.SubElement(variant, "mesh").text = mesh + ".pmd" + ET.SubElement(variant, "texture").text = texture + ".dds" + + animations = ET.SubElement(variant, "animations") + for name, file in anims: + ET.SubElement(animations, "animation", file=file + ".psa", name=name, speed="100") + + props = ET.SubElement(variant, "props") + for name, file in props_: + ET.SubElement(props, "prop", actor=file + ".xml", attachpoint=name) + + return ET.tostring(actor) - animations = ET.SubElement(variant, 'animations') - for name, file in anims: - ET.SubElement(animations, 'animation', file=file+'.psa', name=name, speed='100') - - props = ET.SubElement(variant, 'props') - for name, file in props_: - ET.SubElement(props, 'prop', actor=file+'.xml', attachpoint=name) - return ET.tostring(actor) - def create_actor_static(mesh, texture): - actor = ET.Element('actor', version='1') - ET.SubElement(actor, 'castshadow') - group = ET.SubElement(actor, 'group') - variant = ET.SubElement(group, 'variant', frequency='100', name='Base') - ET.SubElement(variant, 'mesh').text = mesh+'.pmd' - ET.SubElement(variant, 'texture').text = texture+'.dds' - return ET.tostring(actor) + actor = ET.Element("actor", version="1") + ET.SubElement(actor, "castshadow") + group = ET.SubElement(actor, "group") + variant = ET.SubElement(group, "variant", frequency="100", name="Base") + ET.SubElement(variant, "mesh").text = mesh + ".pmd" + ET.SubElement(variant, "texture").text = texture + ".dds" + return ET.tostring(actor) + ################################ # Error handling if False: - convert_dae_to_pmd('This is not well-formed XML', expected_status=-2) - convert_dae_to_pmd('This is not COLLADA', expected_status=-2) - convert_dae_to_pmd('This is still not valid COLLADA', expected_status=-2) + convert_dae_to_pmd("This is not well-formed XML", expected_status=-2) + convert_dae_to_pmd("This is not COLLADA", expected_status=-2) + convert_dae_to_pmd("This is still not valid COLLADA", expected_status=-2) # Do some real conversions, so the output can be tested in the Actor Viewer -test_data = binaries + '/data/tests/collada' -test_mod = binaries + '/data/mods/_test.collada' +test_data = binaries + "/data/tests/collada" +test_mod = binaries + "/data/mods/_test.collada" -clean_dir(test_mod + '/art/meshes') -clean_dir(test_mod + '/art/actors') -clean_dir(test_mod + '/art/animation') +clean_dir(test_mod + "/art/meshes") +clean_dir(test_mod + "/art/actors") +clean_dir(test_mod + "/art/animation") -#for test_file in ['cube', 'jav2', 'jav2b', 'teapot_basic', 'teapot_skin', 'plane_skin', 'dude_skin', 'mergenonbone', 'densemesh']: -#for test_file in ['teapot_basic', 'jav2b', 'jav2d']: -for test_file in ['xsitest3c','xsitest3e','jav2d','jav2d2']: -#for test_file in ['xsitest3']: -#for test_file in []: - print("* Converting PMD %s" % (test_file)) +# for test_file in ['cube', 'jav2', 'jav2b', 'teapot_basic', 'teapot_skin', 'plane_skin', 'dude_skin', 'mergenonbone', 'densemesh']: +# for test_file in ['teapot_basic', 'jav2b', 'jav2d']: +for test_file in ["xsitest3c", "xsitest3e", "jav2d", "jav2d2"]: + # for test_file in ['xsitest3']: + # for test_file in []: + print("* Converting PMD %s" % (test_file)) - input_filename = '%s/%s.dae' % (test_data, test_file) - output_filename = '%s/art/meshes/%s.pmd' % (test_mod, test_file) - - input = open(input_filename).read() - output = convert_dae_to_pmd(input) - open(output_filename, 'wb').write(output) + input_filename = "%s/%s.dae" % (test_data, test_file) + output_filename = "%s/art/meshes/%s.pmd" % (test_mod, test_file) - xml = create_actor(test_file, 'male', [('Idle','dudeidle'),('Corpse','dudecorpse'),('attack1',test_file),('attack2','jav2d')], [('helmet','teapot_basic_static')]) - open('%s/art/actors/%s.xml' % (test_mod, test_file), 'w').write(xml) + input = open(input_filename).read() + output = convert_dae_to_pmd(input) + open(output_filename, "wb").write(output) - xml = create_actor_static(test_file, 'male') - open('%s/art/actors/%s_static.xml' % (test_mod, test_file), 'w').write(xml) + xml = create_actor( + test_file, + "male", + [ + ("Idle", "dudeidle"), + ("Corpse", "dudecorpse"), + ("attack1", test_file), + ("attack2", "jav2d"), + ], + [("helmet", "teapot_basic_static")], + ) + open("%s/art/actors/%s.xml" % (test_mod, test_file), "w").write(xml) -#for test_file in ['jav2','jav2b', 'jav2d']: -for test_file in ['xsitest3c','xsitest3e','jav2d','jav2d2']: -#for test_file in []: - print("* Converting PSA %s" % (test_file)) + xml = create_actor_static(test_file, "male") + open("%s/art/actors/%s_static.xml" % (test_mod, test_file), "w").write(xml) - input_filename = '%s/%s.dae' % (test_data, test_file) - output_filename = '%s/art/animation/%s.psa' % (test_mod, test_file) +# for test_file in ['jav2','jav2b', 'jav2d']: +for test_file in ["xsitest3c", "xsitest3e", "jav2d", "jav2d2"]: + # for test_file in []: + print("* Converting PSA %s" % (test_file)) - input = open(input_filename).read() - output = convert_dae_to_psa(input) - open(output_filename, 'wb').write(output) + input_filename = "%s/%s.dae" % (test_data, test_file) + output_filename = "%s/art/animation/%s.psa" % (test_mod, test_file) + input = open(input_filename).read() + output = convert_dae_to_psa(input) + open(output_filename, "wb").write(output) diff --git a/source/tools/entity/checkrefs.py b/source/tools/entity/checkrefs.py index e6b70d0d3c..40ee93fbe8 100644 --- a/source/tools/entity/checkrefs.py +++ b/source/tools/entity/checkrefs.py @@ -11,6 +11,7 @@ import sys from scriptlib import SimulTemplateEntity, find_files from logging import WARNING, getLogger, StreamHandler, INFO, Formatter, Filter + class SingleLevelFilter(Filter): def __init__(self, passlevel, reject): self.passlevel = passlevel @@ -18,9 +19,10 @@ class SingleLevelFilter(Filter): def filter(self, record): if self.reject: - return (record.levelno != self.passlevel) + return record.levelno != self.passlevel else: - return (record.levelno == self.passlevel) + return record.levelno == self.passlevel + class CheckRefs: def __init__(self): @@ -30,11 +32,11 @@ class CheckRefs: self.roots = [] # list of tuple (parent_file:str, dep_file:str) self.deps = [] - self.vfs_root = Path(__file__).resolve().parents[3] / 'binaries' / 'data' / 'mods' - self.supportedTextureFormats = ('dds', 'png') - self.supportedMeshesFormats = ('pmd', 'dae') - self.supportedAnimationFormats = ('psa', 'dae') - self.supportedAudioFormats = ('ogg') + self.vfs_root = Path(__file__).resolve().parents[3] / "binaries" / "data" / "mods" + self.supportedTextureFormats = ("dds", "png") + self.supportedMeshesFormats = ("pmd", "dae") + self.supportedAnimationFormats = ("psa", "dae") + self.supportedAudioFormats = "ogg" self.mods = [] self.__init_logger self.inError = False @@ -46,36 +48,63 @@ class CheckRefs: # create a console handler, seems nicer to Windows and for future uses ch = StreamHandler(sys.stdout) ch.setLevel(INFO) - ch.setFormatter(Formatter('%(levelname)s - %(message)s')) + ch.setFormatter(Formatter("%(levelname)s - %(message)s")) f1 = SingleLevelFilter(INFO, False) ch.addFilter(f1) logger.addHandler(ch) errorch = StreamHandler(sys.stderr) errorch.setLevel(WARNING) - errorch.setFormatter(Formatter('%(levelname)s - %(message)s')) + errorch.setFormatter(Formatter("%(levelname)s - %(message)s")) logger.addHandler(errorch) self.logger = logger def main(self): - ap = ArgumentParser(description="Checks the game files for missing dependencies, unused files," - " and for file integrity.") - ap.add_argument('-u', '--check-unused', action='store_true', - help="check for all the unused files in the given mods and their dependencies." - " Implies --check-map-xml. Currently yields a lot of false positives.") - ap.add_argument('-x', '--check-map-xml', action='store_true', - help="check maps for missing actor and templates.") - ap.add_argument('-a', '--validate-actors', action='store_true', - help="run the validator.py script to check if the actors files have extra or missing textures." - " This currently only works for the public mod.") - ap.add_argument('-t', '--validate-templates', action='store_true', - help="run the validator.py script to check if the xml files match their (.rng) grammar file.") - ap.add_argument('-m', '--mods', metavar="MOD", dest='mods', nargs='+', default=['public'], - help="specify which mods to check. Default to public.") + ap = ArgumentParser( + description="Checks the game files for missing dependencies, unused files," + " and for file integrity." + ) + ap.add_argument( + "-u", + "--check-unused", + action="store_true", + help="check for all the unused files in the given mods and their dependencies." + " Implies --check-map-xml. Currently yields a lot of false positives.", + ) + ap.add_argument( + "-x", + "--check-map-xml", + action="store_true", + help="check maps for missing actor and templates.", + ) + ap.add_argument( + "-a", + "--validate-actors", + action="store_true", + help="run the validator.py script to check if the actors files have extra or missing textures." + " This currently only works for the public mod.", + ) + ap.add_argument( + "-t", + "--validate-templates", + action="store_true", + help="run the validator.py script to check if the xml files match their (.rng) grammar file.", + ) + ap.add_argument( + "-m", + "--mods", + metavar="MOD", + dest="mods", + nargs="+", + default=["public"], + help="specify which mods to check. Default to public.", + ) args = ap.parse_args() # force check_map_xml if check_unused is used to avoid false positives. args.check_map_xml |= args.check_unused # ordered uniq mods (dict maintains ordered keys from python 3.6) - self.mods = list(dict.fromkeys([*args.mods, *self.get_mod_dependencies(*args.mods), 'mod']).keys()) + self.mods = list( + dict.fromkeys([*args.mods, *self.get_mod_dependencies(*args.mods), "mod"]).keys() + ) self.logger.info(f"Checking {'|'.join(args.mods)}'s integrity.") self.logger.info(f"The following mods will be loaded: {'|'.join(self.mods)}.") if args.check_map_xml: @@ -103,12 +132,14 @@ class CheckRefs: if args.validate_templates: sys.path.append("../xmlvalidator/") from validate_grammar import RelaxNGValidator + validate = RelaxNGValidator(self.vfs_root, self.mods) if not validate.run(): self.inError = True if args.validate_actors: sys.path.append("../xmlvalidator/") from validator import Validator + validator = Validator(self.vfs_root, self.mods) if not validator.run(): self.inError = True @@ -118,14 +149,16 @@ class CheckRefs: def get_mod_dependencies(self, *mods): modjsondeps = [] for mod in mods: - mod_json_path = self.vfs_root / mod / 'mod.json' + mod_json_path = self.vfs_root / mod / "mod.json" if not exists(mod_json_path): continue - with open(mod_json_path, encoding='utf-8') as f: + with open(mod_json_path, encoding="utf-8") as f: modjson = load(f) # 0ad's folder isn't named like the mod. - modjsondeps.extend(['public' if '0ad' in dep else dep for dep in modjson.get('dependencies', [])]) + modjsondeps.extend( + ["public" if "0ad" in dep else dep for dep in modjson.get("dependencies", [])] + ) return modjsondeps def vfs_to_relative_to_mods(self, vfs_path): @@ -144,71 +177,86 @@ class CheckRefs: def add_maps_xml(self): self.logger.info("Loading maps XML...") - mapfiles = self.find_files('maps/scenarios', 'xml') - mapfiles.extend(self.find_files('maps/skirmishes', 'xml')) - mapfiles.extend(self.find_files('maps/tutorials', 'xml')) - actor_prefix = 'actor|' - resource_prefix = 'resource|' - for (fp, ffp) in sorted(mapfiles): + mapfiles = self.find_files("maps/scenarios", "xml") + mapfiles.extend(self.find_files("maps/skirmishes", "xml")) + mapfiles.extend(self.find_files("maps/tutorials", "xml")) + actor_prefix = "actor|" + resource_prefix = "resource|" + for fp, ffp in sorted(mapfiles): self.files.append(str(fp)) self.roots.append(str(fp)) et_map = ElementTree.parse(ffp).getroot() - entities = et_map.find('Entities') - used = {entity.find('Template').text.strip() for entity in entities.findall('Entity')} if entities is not None else {} + entities = et_map.find("Entities") + used = ( + {entity.find("Template").text.strip() for entity in entities.findall("Entity")} + if entities is not None + else {} + ) for template in used: if template.startswith(actor_prefix): - self.deps.append((str(fp), f'art/actors/{template[len(actor_prefix):]}')) + self.deps.append((str(fp), f"art/actors/{template[len(actor_prefix):]}")) elif template.startswith(resource_prefix): - self.deps.append((str(fp), f'simulation/templates/{template[len(resource_prefix):]}.xml')) + self.deps.append( + (str(fp), f"simulation/templates/{template[len(resource_prefix):]}.xml") + ) else: - self.deps.append((str(fp), f'simulation/templates/{template}.xml')) + self.deps.append((str(fp), f"simulation/templates/{template}.xml")) # Map previews - settings = loads(et_map.find('ScriptSettings').text) - if settings.get('Preview', None): - self.deps.append((str(fp), f'art/textures/ui/session/icons/mappreview/{settings["Preview"]}')) + settings = loads(et_map.find("ScriptSettings").text) + if settings.get("Preview", None): + self.deps.append( + (str(fp), f'art/textures/ui/session/icons/mappreview/{settings["Preview"]}') + ) def add_maps_pmp(self): self.logger.info("Loading maps PMP...") # Need to generate terrain texture filename=>relative path lookup first terrains = dict() - for (fp, ffp) in self.find_files('art/terrains', 'xml'): + for fp, ffp in self.find_files("art/terrains", "xml"): name = fp.stem # ignore terrains.xml - if name != 'terrains': + if name != "terrains": if name in terrains: self.inError = True - self.logger.error(f"Duplicate terrain name '{name}' (from '{terrains[name]}' and '{ffp}')") + self.logger.error( + f"Duplicate terrain name '{name}' (from '{terrains[name]}' and '{ffp}')" + ) terrains[name] = str(fp) - mapfiles = self.find_files('maps/scenarios', 'pmp') - mapfiles.extend(self.find_files('maps/skirmishes', 'pmp')) - for (fp, ffp) in sorted(mapfiles): + mapfiles = self.find_files("maps/scenarios", "pmp") + mapfiles.extend(self.find_files("maps/skirmishes", "pmp")) + for fp, ffp in sorted(mapfiles): self.files.append(str(fp)) self.roots.append(str(fp)) - with open(ffp, 'rb') as f: - expected_header = b'PSMP' + with open(ffp, "rb") as f: + expected_header = b"PSMP" header = f.read(len(expected_header)) if header != expected_header: raise ValueError(f"Invalid PMP header {header} in '{ffp}'") - int_fmt = ' 0: for sound_group in sound_groups: if sound_group.text and sound_group.text.strip(): - if '{phenotype}' in sound_group.text: + if "{phenotype}" in sound_group.text: for phenotype in phenotypes: # see simulation/components/Sound.js and Identity.js for explanation - sound_path = sound_group.text.replace('{phenotype}', phenotype).replace('{lang}', lang) - self.deps.append((str(fp), f'audio/{sound_path}')) + sound_path = sound_group.text.replace( + "{phenotype}", phenotype + ).replace("{lang}", lang) + self.deps.append((str(fp), f"audio/{sound_path}")) else: - sound_path = sound_group.text.replace('{lang}', lang) - self.deps.append((str(fp), f'audio/{sound_path}')) - if entity.find('Identity') is not None: - icon = entity.find('Identity').find('Icon') + sound_path = sound_group.text.replace("{lang}", lang) + self.deps.append((str(fp), f"audio/{sound_path}")) + if entity.find("Identity") is not None: + icon = entity.find("Identity").find("Icon") if icon is not None and icon.text: - if entity.find('Formation') is not None: - self.deps.append((str(fp), f'art/textures/ui/session/icons/{icon.text}')) + if entity.find("Formation") is not None: + self.deps.append( + (str(fp), f"art/textures/ui/session/icons/{icon.text}") + ) else: - self.deps.append((str(fp), f'art/textures/ui/session/portraits/{icon.text}')) - if entity.find('Heal') is not None and entity.find('Heal').find('RangeOverlay') is not None: - range_overlay = entity.find('Heal').find('RangeOverlay') - for tag in ('LineTexture', 'LineTextureMask'): + self.deps.append( + (str(fp), f"art/textures/ui/session/portraits/{icon.text}") + ) + if ( + entity.find("Heal") is not None + and entity.find("Heal").find("RangeOverlay") is not None + ): + range_overlay = entity.find("Heal").find("RangeOverlay") + for tag in ("LineTexture", "LineTextureMask"): elem = range_overlay.find(tag) if elem is not None and elem.text: - self.deps.append((str(fp), f'art/textures/selection/{elem.text}')) - if entity.find('Selectable') is not None and entity.find('Selectable').find('Overlay') is not None \ - and entity.find('Selectable').find('Overlay').find('Texture') is not None: - texture = entity.find('Selectable').find('Overlay').find('Texture') - for tag in ('MainTexture', 'MainTextureMask'): + self.deps.append((str(fp), f"art/textures/selection/{elem.text}")) + if ( + entity.find("Selectable") is not None + and entity.find("Selectable").find("Overlay") is not None + and entity.find("Selectable").find("Overlay").find("Texture") is not None + ): + texture = entity.find("Selectable").find("Overlay").find("Texture") + for tag in ("MainTexture", "MainTextureMask"): elem = texture.find(tag) if elem is not None and elem.text: - self.deps.append((str(fp), f'art/textures/selection/{elem.text}')) - if entity.find('Formation') is not None: - icon = entity.find('Formation').find('Icon') + self.deps.append((str(fp), f"art/textures/selection/{elem.text}")) + if entity.find("Formation") is not None: + icon = entity.find("Formation").find("Icon") if icon is not None and icon.text: - self.deps.append((str(fp), f'art/textures/ui/session/icons/{icon.text}')) + self.deps.append((str(fp), f"art/textures/ui/session/icons/{icon.text}")) - cmp_auras = entity.find('Auras') + cmp_auras = entity.find("Auras") if cmp_auras is not None: auraString = cmp_auras.text - for aura in split(r'\s+', auraString): + for aura in split(r"\s+", auraString): if not aura: continue - if aura.startswith('-'): + if aura.startswith("-"): continue - self.deps.append((str(fp), f'simulation/data/auras/{aura}.json')) + self.deps.append((str(fp), f"simulation/data/auras/{aura}.json")) - cmp_identity = entity.find('Identity') + cmp_identity = entity.find("Identity") if cmp_identity is not None: - reqTag = cmp_identity.find('Requirements') + reqTag = cmp_identity.find("Requirements") if reqTag is not None: - def parse_requirements(req, recursionDepth = 1): - techsTag = req.find('Techs') + + def parse_requirements(req, recursionDepth=1): + techsTag = req.find("Techs") if techsTag is not None: for techTag in techsTag.text.split(): - self.deps.append((str(fp), f'simulation/data/technologies/{techTag}.json')) + self.deps.append( + (str(fp), f"simulation/data/technologies/{techTag}.json") + ) if recursionDepth > 0: recursionDepth -= 1 - allReqTag = req.find('All') + allReqTag = req.find("All") if allReqTag is not None: parse_requirements(allReqTag, recursionDepth) - anyReqTag = req.find('Any') + anyReqTag = req.find("Any") if anyReqTag is not None: parse_requirements(anyReqTag, recursionDepth) + parse_requirements(reqTag) - cmp_researcher = entity.find('Researcher') + cmp_researcher = entity.find("Researcher") if cmp_researcher is not None: - techString = cmp_researcher.find('Technologies') + techString = cmp_researcher.find("Technologies") if techString is not None: - for tech in split(r'\s+', techString.text): + for tech in split(r"\s+", techString.text): if not tech: continue - if tech.startswith('-'): + if tech.startswith("-"): continue - if '{civ}' in tech and cmp_identity is not None and cmp_identity.find('Civ') is not None: - civ = cmp_identity.find('Civ').text + if ( + "{civ}" in tech + and cmp_identity is not None + and cmp_identity.find("Civ") is not None + ): + civ = cmp_identity.find("Civ").text # Fallback for non specific phase techs. - if tech.startswith('phase') and not bool([phase_tech for phase_tech in custom_phase_techs if (tech.replace('{civ}', civ) + ".json") == phase_tech]) : - civ = 'generic' - tech = tech.replace('{civ}', civ) - self.deps.append((str(fp), f'simulation/data/technologies/{tech}.json')) + if tech.startswith("phase") and not bool( + [ + phase_tech + for phase_tech in custom_phase_techs + if (tech.replace("{civ}", civ) + ".json") == phase_tech + ] + ): + civ = "generic" + tech = tech.replace("{civ}", civ) + self.deps.append( + (str(fp), f"simulation/data/technologies/{tech}.json") + ) def append_variant_dependencies(self, variant, fp): - variant_file = variant.get('file') - mesh = variant.find('mesh') - particles = variant.find('particles') - texture_files = [tex.get('file') for tex in variant.find('textures').findall('texture')] \ - if variant.find('textures') is not None else [] - prop_actors = [prop.get('actor') for prop in variant.find('props').findall('prop')] \ - if variant.find('props') is not None else [] - animation_files = [anim.get('file') for anim in variant.find('animations').findall('animation')] \ - if variant.find('animations') is not None else [] + variant_file = variant.get("file") + mesh = variant.find("mesh") + particles = variant.find("particles") + texture_files = ( + [tex.get("file") for tex in variant.find("textures").findall("texture")] + if variant.find("textures") is not None + else [] + ) + prop_actors = ( + [prop.get("actor") for prop in variant.find("props").findall("prop")] + if variant.find("props") is not None + else [] + ) + animation_files = ( + [anim.get("file") for anim in variant.find("animations").findall("animation")] + if variant.find("animations") is not None + else [] + ) if variant_file: - self.deps.append((str(fp), f'art/variants/{variant_file}')) + self.deps.append((str(fp), f"art/variants/{variant_file}")) if mesh is not None and mesh.text: - self.deps.append((str(fp), f'art/meshes/{mesh.text}')) - if particles is not None and particles.get('file'): + self.deps.append((str(fp), f"art/meshes/{mesh.text}")) + if particles is not None and particles.get("file"): self.deps.append((str(fp), f'art/particles/{particles.get("file")}')) for texture_file in [x for x in texture_files if x]: - self.deps.append((str(fp), f'art/textures/skins/{texture_file}')) + self.deps.append((str(fp), f"art/textures/skins/{texture_file}")) for prop_actor in [x for x in prop_actors if x]: - self.deps.append((str(fp), f'art/actors/{prop_actor}')) + self.deps.append((str(fp), f"art/actors/{prop_actor}")) for animation_file in [x for x in animation_files if x]: - self.deps.append((str(fp), f'art/animation/{animation_file}')) + self.deps.append((str(fp), f"art/animation/{animation_file}")) def append_actor_dependencies(self, actor, fp): - for group in actor.findall('group'): - for variant in group.findall('variant'): + for group in actor.findall("group"): + for variant in group.findall("variant"): self.append_variant_dependencies(variant, fp) - material = actor.find('material') + material = actor.find("material") if material is not None and material.text: - self.deps.append((str(fp), f'art/materials/{material.text}')) + self.deps.append((str(fp), f"art/materials/{material.text}")) def add_actors(self): self.logger.info("Loading actors...") - for (fp, ffp) in sorted(self.find_files('art/actors', 'xml')): + for fp, ffp in sorted(self.find_files("art/actors", "xml")): self.files.append(str(fp)) self.roots.append(str(fp)) root = ElementTree.parse(ffp).getroot() - if root.tag == 'actor': + if root.tag == "actor": self.append_actor_dependencies(root, fp) # model has lods - elif root.tag == 'qualitylevels': + elif root.tag == "qualitylevels": qualitylevels = root - for actor in qualitylevels.findall('actor'): + for actor in qualitylevels.findall("actor"): self.append_actor_dependencies(actor, fp) - for actor in qualitylevels.findall('inline'): + for actor in qualitylevels.findall("inline"): self.append_actor_dependencies(actor, fp) def add_variants(self): self.logger.info("Loading variants...") - for (fp, ffp) in sorted(self.find_files('art/variants', 'xml')): + for fp, ffp in sorted(self.find_files("art/variants", "xml")): self.files.append(str(fp)) self.roots.append(str(fp)) variant = ElementTree.parse(ffp).getroot() @@ -403,235 +505,302 @@ class CheckRefs: def add_art(self): self.logger.info("Loading art files...") - self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/particles', *self.supportedTextureFormats)]) - self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/terrain', *self.supportedTextureFormats)]) - self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/skins', *self.supportedTextureFormats)]) - self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/meshes', *self.supportedMeshesFormats)]) - self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/animation', *self.supportedAnimationFormats)]) - + self.files.extend( + [ + str(fp) + for (fp, ffp) in self.find_files( + "art/textures/particles", *self.supportedTextureFormats + ) + ] + ) + self.files.extend( + [ + str(fp) + for (fp, ffp) in self.find_files( + "art/textures/terrain", *self.supportedTextureFormats + ) + ] + ) + self.files.extend( + [ + str(fp) + for (fp, ffp) in self.find_files( + "art/textures/skins", *self.supportedTextureFormats + ) + ] + ) + self.files.extend( + [str(fp) for (fp, ffp) in self.find_files("art/meshes", *self.supportedMeshesFormats)] + ) + self.files.extend( + [ + str(fp) + for (fp, ffp) in self.find_files("art/animation", *self.supportedAnimationFormats) + ] + ) def add_materials(self): self.logger.info("Loading materials...") - for (fp, ffp) in sorted(self.find_files('art/materials', 'xml')): + for fp, ffp in sorted(self.find_files("art/materials", "xml")): self.files.append(str(fp)) material_elem = ElementTree.parse(ffp).getroot() - for alternative in material_elem.findall('alternative'): - material = alternative.get('material') + for alternative in material_elem.findall("alternative"): + material = alternative.get("material") if material is not None: - self.deps.append((str(fp), f'art/materials/{material}')) + self.deps.append((str(fp), f"art/materials/{material}")) def add_particles(self): self.logger.info("Loading particles...") - for (fp, ffp) in sorted(self.find_files('art/particles', 'xml')): + for fp, ffp in sorted(self.find_files("art/particles", "xml")): self.files.append(str(fp)) self.roots.append(str(fp)) particle = ElementTree.parse(ffp).getroot() - texture = particle.find('texture') + texture = particle.find("texture") if texture is not None: self.deps.append((str(fp), texture.text)) def add_soundgroups(self): self.logger.info("Loading sound groups...") - for (fp, ffp) in sorted(self.find_files('audio', 'xml')): + for fp, ffp in sorted(self.find_files("audio", "xml")): self.files.append(str(fp)) self.roots.append(str(fp)) sound_group = ElementTree.parse(ffp).getroot() - path = sound_group.find('Path').text.rstrip('/') - for sound in sound_group.findall('Sound'): - self.deps.append((str(fp), f'{path}/{sound.text}')) + path = sound_group.find("Path").text.rstrip("/") + for sound in sound_group.findall("Sound"): + self.deps.append((str(fp), f"{path}/{sound.text}")) def add_audio(self): self.logger.info("Loading audio files...") - self.files.extend([str(fp) for (fp, ffp) in self.find_files('audio/', self.supportedAudioFormats)]) - + self.files.extend( + [str(fp) for (fp, ffp) in self.find_files("audio/", self.supportedAudioFormats)] + ) def add_gui_object_repeat(self, obj, fp): - for repeat in obj.findall('repeat'): - for sub_obj in repeat.findall('object'): + for repeat in obj.findall("repeat"): + for sub_obj in repeat.findall("object"): # TODO: look at sprites, styles, etc self.add_gui_object_include(sub_obj, fp) - for sub_obj in repeat.findall('objects'): + for sub_obj in repeat.findall("objects"): # TODO: look at sprites, styles, etc self.add_gui_object_include(sub_obj, fp) self.add_gui_object_include(repeat, fp) def add_gui_object_include(self, obj, fp): - for include in obj.findall('include'): - included_file = include.get('file') + for include in obj.findall("include"): + included_file = include.get("file") if included_file: - self.deps.append((str(fp), f'{included_file}')) + self.deps.append((str(fp), f"{included_file}")) def add_gui_object(self, parent, fp): if parent is None: return - for obj in parent.findall('object'): + for obj in parent.findall("object"): # TODO: look at sprites, styles, etc self.add_gui_object_repeat(obj, fp) self.add_gui_object_include(obj, fp) self.add_gui_object(obj, fp) - for obj in parent.findall('objects'): + for obj in parent.findall("objects"): # TODO: look at sprites, styles, etc self.add_gui_object_repeat(obj, fp) self.add_gui_object_include(obj, fp) self.add_gui_object(obj, fp) - def add_gui_xml(self): self.logger.info("Loading GUI XML...") - for (fp, ffp) in sorted(self.find_files('gui', 'xml')): + for fp, ffp in sorted(self.find_files("gui", "xml")): self.files.append(str(fp)) # GUI page definitions are assumed to be named page_[something].xml and alone in that. if match(r".*[\\\/]page(_[^.\/\\]+)?\.xml$", str(fp)): self.roots.append(str(fp)) root_xml = ElementTree.parse(ffp).getroot() - for include in root_xml.findall('include'): + for include in root_xml.findall("include"): # If including an entire directory, find all the *.xml files - if include.text.endswith('/'): - self.deps.extend([(str(fp), str(sub_fp)) for (sub_fp, sub_ffp) in self.find_files(f'gui/{include.text}', 'xml')]) + if include.text.endswith("/"): + self.deps.extend( + [ + (str(fp), str(sub_fp)) + for (sub_fp, sub_ffp) in self.find_files( + f"gui/{include.text}", "xml" + ) + ] + ) else: - self.deps.append((str(fp), f'gui/{include.text}')) + self.deps.append((str(fp), f"gui/{include.text}")) else: xml = ElementTree.parse(ffp) root_xml = xml.getroot() name = root_xml.tag self.roots.append(str(fp)) - if name in ('objects', 'object'): - for script in root_xml.findall('script'): - if script.get('file'): - self.deps.append((str(fp), script.get('file'))) - if script.get('directory'): + if name in ("objects", "object"): + for script in root_xml.findall("script"): + if script.get("file"): + self.deps.append((str(fp), script.get("file"))) + if script.get("directory"): # If including an entire directory, find all the *.js files - self.deps.extend([(str(fp), str(sub_fp)) for (sub_fp, sub_ffp) in self.find_files(script.get('directory'), 'js')]) + self.deps.extend( + [ + (str(fp), str(sub_fp)) + for (sub_fp, sub_ffp) in self.find_files( + script.get("directory"), "js" + ) + ] + ) self.add_gui_object(root_xml, fp) - elif name == 'setup': + elif name == "setup": # TODO: look at sprites, styles, etc pass - elif name == 'styles': - for style in root_xml.findall('style'): - if(style.get('sound_opened')): + elif name == "styles": + for style in root_xml.findall("style"): + if style.get("sound_opened"): self.deps.append((str(fp), f"{style.get('sound_opened')}")) - if(style.get('sound_closed')): + if style.get("sound_closed"): self.deps.append((str(fp), f"{style.get('sound_closed')}")) - if(style.get('sound_selected')): + if style.get("sound_selected"): self.deps.append((str(fp), f"{style.get('sound_selected')}")) - if(style.get('sound_disabled')): + if style.get("sound_disabled"): self.deps.append((str(fp), f"{style.get('sound_disabled')}")) # TODO: look at sprites, styles, etc pass - elif name == 'sprites': - for sprite in root_xml.findall('sprite'): - for image in sprite.findall('image'): - if image.get('texture'): - self.deps.append((str(fp), f"art/textures/ui/{image.get('texture')}")) + elif name == "sprites": + for sprite in root_xml.findall("sprite"): + for image in sprite.findall("image"): + if image.get("texture"): + self.deps.append( + (str(fp), f"art/textures/ui/{image.get('texture')}") + ) else: bio = BytesIO() xml.write(bio) bio.seek(0) - raise ValueError(f"Unexpected GUI XML root element '{name}':\n{bio.read().decode('ascii')}") + raise ValueError( + f"Unexpected GUI XML root element '{name}':\n{bio.read().decode('ascii')}" + ) def add_gui_data(self): self.logger.info("Loading GUI data...") - self.files.extend([str(fp) for (fp, ffp) in self.find_files('gui', 'js')]) - self.files.extend([str(fp) for (fp, ffp) in self.find_files('gamesettings', 'js')]) - self.files.extend([str(fp) for (fp, ffp) in self.find_files('autostart', 'js')]) - self.roots.extend([str(fp) for (fp, ffp) in self.find_files('autostart', 'js')]) - self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/ui', *self.supportedTextureFormats)]) - self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/selection', *self.supportedTextureFormats)]) + self.files.extend([str(fp) for (fp, ffp) in self.find_files("gui", "js")]) + self.files.extend([str(fp) for (fp, ffp) in self.find_files("gamesettings", "js")]) + self.files.extend([str(fp) for (fp, ffp) in self.find_files("autostart", "js")]) + self.roots.extend([str(fp) for (fp, ffp) in self.find_files("autostart", "js")]) + self.files.extend( + [ + str(fp) + for (fp, ffp) in self.find_files("art/textures/ui", *self.supportedTextureFormats) + ] + ) + self.files.extend( + [ + str(fp) + for (fp, ffp) in self.find_files( + "art/textures/selection", *self.supportedTextureFormats + ) + ] + ) def add_civs(self): self.logger.info("Loading civs...") - for (fp, ffp) in sorted(self.find_files('simulation/data/civs', 'json')): + for fp, ffp in sorted(self.find_files("simulation/data/civs", "json")): self.files.append(str(fp)) self.roots.append(str(fp)) - with open(ffp, encoding='utf-8') as f: + with open(ffp, encoding="utf-8") as f: civ = load(f) - for music in civ.get('Music', []): + for music in civ.get("Music", []): self.deps.append((str(fp), f"audio/music/{music['File']}")) def add_tips(self): self.logger.info("Loading tips...") - for (fp, ffp) in sorted(self.find_files('gui/text/tips', 'txt')): + for fp, ffp in sorted(self.find_files("gui/text/tips", "txt")): relative_path = str(fp) self.files.append(relative_path) self.roots.append(relative_path) - self.deps.append((relative_path, f"art/textures/ui/loading/tips/{basename(relative_path).split('.')[0]}.png")) - + self.deps.append( + ( + relative_path, + f"art/textures/ui/loading/tips/{basename(relative_path).split('.')[0]}.png", + ) + ) def add_rms(self): self.logger.info("Loading random maps...") - self.files.extend([str(fp) for (fp, ffp) in self.find_files('maps/random', 'js')]) - for (fp, ffp) in sorted(self.find_files('maps/random', 'json')): - if str(fp).startswith('maps/random/rmbiome'): + self.files.extend([str(fp) for (fp, ffp) in self.find_files("maps/random", "js")]) + for fp, ffp in sorted(self.find_files("maps/random", "json")): + if str(fp).startswith("maps/random/rmbiome"): continue self.files.append(str(fp)) self.roots.append(str(fp)) - with open(ffp, encoding='utf-8') as f: + with open(ffp, encoding="utf-8") as f: randmap = load(f) - settings = randmap.get('settings', {}) - if settings.get('Script', None): + settings = randmap.get("settings", {}) + if settings.get("Script", None): self.deps.append((str(fp), f"maps/random/{settings['Script']}")) # Map previews - if settings.get('Preview', None): - self.deps.append((str(fp), f'art/textures/ui/session/icons/mappreview/{settings["Preview"]}')) + if settings.get("Preview", None): + self.deps.append( + (str(fp), f'art/textures/ui/session/icons/mappreview/{settings["Preview"]}') + ) def add_techs(self): self.logger.info("Loading techs...") - for (fp, ffp) in sorted(self.find_files('simulation/data/technologies', 'json')): + for fp, ffp in sorted(self.find_files("simulation/data/technologies", "json")): self.files.append(str(fp)) - with open(ffp, encoding='utf-8') as f: + with open(ffp, encoding="utf-8") as f: tech = load(f) - if tech.get('autoResearch', None): + if tech.get("autoResearch", None): self.roots.append(str(fp)) - if tech.get('icon', None): - self.deps.append((str(fp), f"art/textures/ui/session/portraits/technologies/{tech['icon']}")) - if tech.get('supersedes', None): - self.deps.append((str(fp), f"simulation/data/technologies/{tech['supersedes']}.json")) - if tech.get('top', None): + if tech.get("icon", None): + self.deps.append( + (str(fp), f"art/textures/ui/session/portraits/technologies/{tech['icon']}") + ) + if tech.get("supersedes", None): + self.deps.append( + (str(fp), f"simulation/data/technologies/{tech['supersedes']}.json") + ) + if tech.get("top", None): self.deps.append((str(fp), f"simulation/data/technologies/{tech['top']}.json")) - if tech.get('bottom', None): + if tech.get("bottom", None): self.deps.append((str(fp), f"simulation/data/technologies/{tech['bottom']}.json")) def add_terrains(self): self.logger.info("Loading terrains...") - for (fp, ffp) in sorted(self.find_files('art/terrains', 'xml')): + for fp, ffp in sorted(self.find_files("art/terrains", "xml")): # ignore terrains.xml - if str(fp).endswith('terrains.xml'): + if str(fp).endswith("terrains.xml"): continue self.files.append(str(fp)) self.roots.append(str(fp)) terrain = ElementTree.parse(ffp).getroot() - for texture in terrain.find('textures').findall('texture'): - if texture.get('file'): + for texture in terrain.find("textures").findall("texture"): + if texture.get("file"): self.deps.append((str(fp), f"art/textures/terrain/{texture.get('file')}")) - if terrain.find('material') is not None: - material = terrain.find('material').text + if terrain.find("material") is not None: + material = terrain.find("material").text self.deps.append((str(fp), f"art/materials/{material}")) def add_auras(self): self.logger.info("Loading auras...") - for (fp, ffp) in sorted(self.find_files('simulation/data/auras', 'json')): + for fp, ffp in sorted(self.find_files("simulation/data/auras", "json")): self.files.append(str(fp)) - with open(ffp, encoding='utf-8') as f: + with open(ffp, encoding="utf-8") as f: aura = load(f) - if aura.get('overlayIcon', None): - self.deps.append((str(fp), aura['overlayIcon'])) - range_overlay = aura.get('rangeOverlay', {}) - for prop in ('lineTexture', 'lineTextureMask'): + if aura.get("overlayIcon", None): + self.deps.append((str(fp), aura["overlayIcon"])) + range_overlay = aura.get("rangeOverlay", {}) + for prop in ("lineTexture", "lineTextureMask"): if range_overlay.get(prop, None): self.deps.append((str(fp), f"art/textures/selection/{range_overlay[prop]}")) def check_deps(self): self.logger.info("Looking for missing files...") uniq_files = set(self.files) - uniq_files = [r.replace(sep, '/') for r in uniq_files] + uniq_files = [r.replace(sep, "/") for r in uniq_files] lower_case_files = {f.lower(): f for f in uniq_files} reverse_deps = dict() for parent, dep in self.deps: - if sep != '/': - parent = parent.replace(sep, '/') - dep = dep.replace(sep, '/') + if sep != "/": + parent = parent.replace(sep, "/") + dep = dep.replace(sep, "/") if dep not in reverse_deps: reverse_deps[dep] = {parent} else: @@ -639,9 +808,9 @@ class CheckRefs: for dep in sorted(reverse_deps.keys()): if "simulation/templates" in dep and ( - dep.replace("templates/", "template/special/filter/") in uniq_files or - dep.replace("templates/", "template/mixins/") in uniq_files - ): + dep.replace("templates/", "template/special/filter/") in uniq_files + or dep.replace("templates/", "template/mixins/") in uniq_files + ): continue if dep in uniq_files: @@ -651,15 +820,17 @@ class CheckRefs: self.logger.error(f"Missing file '{dep}' referenced by: {', '.join(sorted(callers))}") self.inError = True if dep.lower() in lower_case_files: - self.logger.warning(f"### Case-insensitive match (found '{lower_case_files[dep.lower()]}')") + self.logger.warning( + f"### Case-insensitive match (found '{lower_case_files[dep.lower()]}')" + ) def check_unused(self): self.logger.info("Looking for unused files...") deps = dict() for parent, dep in self.deps: - if sep != '/': - parent = parent.replace(sep, '/') - dep = dep.replace(sep, '/') + if sep != "/": + parent = parent.replace(sep, "/") + dep = dep.replace(sep, "/") if parent not in deps: deps[parent] = {dep} @@ -667,9 +838,9 @@ class CheckRefs: deps[parent].add(dep) uniq_files = set(self.files) - uniq_files = [r.replace(sep, '/') for r in uniq_files] + uniq_files = [r.replace(sep, "/") for r in uniq_files] reachable = list(set(self.roots)) - reachable = [r.replace(sep, '/') for r in reachable] + reachable = [r.replace(sep, "/") for r in reachable] while True: new_reachable = [] for r in reachable: @@ -680,16 +851,18 @@ class CheckRefs: break for f in sorted(uniq_files): - if any(( - f in reachable, - 'art/terrains/' in f, - 'maps/random/' in f, - )): + if any( + ( + f in reachable, + "art/terrains/" in f, + "maps/random/" in f, + ) + ): continue self.logger.warning(f"Unused file '{str(self.vfs_to_relative_to_mods(f))}'") -if __name__ == '__main__': +if __name__ == "__main__": check_ref = CheckRefs() if not check_ref.main(): sys.exit(1) diff --git a/source/tools/entity/creationgraph.py b/source/tools/entity/creationgraph.py index 6856cda60c..7a053df5cf 100644 --- a/source/tools/entity/creationgraph.py +++ b/source/tools/entity/creationgraph.py @@ -8,41 +8,58 @@ from scriptlib import warn, SimulTemplateEntity, find_files def find_entities(vfs_root): - base = vfs_root / 'public' / 'simulation' / 'templates' - return [str(fp.relative_to(base).with_suffix('')) for (_, fp) in find_files(vfs_root, ['public'], 'simulation/templates', 'xml')] + base = vfs_root / "public" / "simulation" / "templates" + return [ + str(fp.relative_to(base).with_suffix("")) + for (_, fp) in find_files(vfs_root, ["public"], "simulation/templates", "xml") + ] def main(): - vfs_root = Path(__file__).resolve().parents[3] / 'binaries' / 'data' / 'mods' - simul_templates_path = Path('simulation/templates') + vfs_root = Path(__file__).resolve().parents[3] / "binaries" / "data" / "mods" + simul_templates_path = Path("simulation/templates") simul_template_entity = SimulTemplateEntity(vfs_root) - with open('creation.dot', 'w') as dot_f: - dot_f.write('digraph G {\n') + with open("creation.dot", "w") as dot_f: + dot_f.write("digraph G {\n") files = sorted(find_entities(vfs_root)) for f in files: - if f.startswith('template_'): + if f.startswith("template_"): continue print(f"# {f}...") - entity = simul_template_entity.load_inherited(simul_templates_path, f, ['public']) - if entity.find('Builder') is not None and entity.find('Builder').find('Entities') is not None: - entities = entity.find('Builder').find('Entities').text.replace('{civ}', entity.find('Identity').find('Civ').text) - builders = split(r'\s+', entities.strip()) + entity = simul_template_entity.load_inherited(simul_templates_path, f, ["public"]) + if ( + entity.find("Builder") is not None + and entity.find("Builder").find("Entities") is not None + ): + entities = ( + entity.find("Builder") + .find("Entities") + .text.replace("{civ}", entity.find("Identity").find("Civ").text) + ) + builders = split(r"\s+", entities.strip()) for builder in builders: if Path(builder) in files: warn(f"Invalid Builder reference: {f} -> {builder}") dot_f.write(f'"{f}" -> "{builder}" [color=green];\n') - if entity.find('TrainingQueue') is not None and entity.find('TrainingQueue').find('Entities') is not None: - entities = entity.find('TrainingQueue').find('Entities').text.replace('{civ}', entity.find('Identity').find('Civ').text) - training_queues = split(r'\s+', entities.strip()) + if ( + entity.find("TrainingQueue") is not None + and entity.find("TrainingQueue").find("Entities") is not None + ): + entities = ( + entity.find("TrainingQueue") + .find("Entities") + .text.replace("{civ}", entity.find("Identity").find("Civ").text) + ) + training_queues = split(r"\s+", entities.strip()) for training_queue in training_queues: if Path(training_queue) in files: warn(f"Invalid TrainingQueue reference: {f} -> {training_queue}") dot_f.write(f'"{f}" -> "{training_queue}" [color=blue];\n') - dot_f.write('}\n') - if run(['dot', '-V'], capture_output=True).returncode == 0: - exit(run(['dot', '-Tpng', 'creation.dot', '-o', 'creation.png'], text=True).returncode) + dot_f.write("}\n") + if run(["dot", "-V"], capture_output=True).returncode == 0: + exit(run(["dot", "-Tpng", "creation.dot", "-o", "creation.png"], text=True).returncode) -if __name__ == '__main__': +if __name__ == "__main__": chdir(Path(__file__).resolve().parent) main() diff --git a/source/tools/entity/entvalidate.py b/source/tools/entity/entvalidate.py index ada7fc91b0..5d3b2faafc 100644 --- a/source/tools/entity/entvalidate.py +++ b/source/tools/entity/entvalidate.py @@ -17,8 +17,10 @@ RELAXNG_SCHEMA_ERROR_MSG = """Relax NG schema non existant. Please create the file: {} You can do that by running 'pyrogenesis -dumpSchema' in the 'system' directory """ -XMLLINT_ERROR_MSG = ("xmllint not found in your PATH, please install it " - "(usually in libxml2 package)") +XMLLINT_ERROR_MSG = ( + "xmllint not found in your PATH, please install it " "(usually in libxml2 package)" +) + class SingleLevelFilter(logging.Filter): def __init__(self, passlevel, reject): @@ -27,37 +29,48 @@ class SingleLevelFilter(logging.Filter): def filter(self, record): if self.reject: - return (record.levelno != self.passlevel) + return record.levelno != self.passlevel else: - return (record.levelno == self.passlevel) + return record.levelno == self.passlevel + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) # create a console handler, seems nicer to Windows and for future uses ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.INFO) -ch.setFormatter(logging.Formatter('%(levelname)s - %(message)s')) +ch.setFormatter(logging.Formatter("%(levelname)s - %(message)s")) f1 = SingleLevelFilter(logging.INFO, False) ch.addFilter(f1) logger.addHandler(ch) -errorch =logging. StreamHandler(sys.stderr) +errorch = logging.StreamHandler(sys.stderr) errorch.setLevel(logging.WARNING) -errorch.setFormatter(logging.Formatter('%(levelname)s - %(message)s')) +errorch.setFormatter(logging.Formatter("%(levelname)s - %(message)s")) logger.addHandler(errorch) + def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Validate templates") - parser.add_argument("-m", "--mod-name", required=True, - help="The name of the mod to validate.") - parser.add_argument("-r", "--root", dest="vfs_root", default=Path(), - type=Path, help="The path to mod's root location.") - parser.add_argument("-s", "--relaxng-schema", - default=Path() / ENTITY_RELAXNG_FNAME, type=Path, - help="The path to mod's root location.") - parser.add_argument("-t", "--templates", nargs="*", - help="Optionally, a list of templates to validate.") - parser.add_argument("-v", "--verbose", - help="Be verbose about the output.", default=False) + parser.add_argument("-m", "--mod-name", required=True, help="The name of the mod to validate.") + parser.add_argument( + "-r", + "--root", + dest="vfs_root", + default=Path(), + type=Path, + help="The path to mod's root location.", + ) + parser.add_argument( + "-s", + "--relaxng-schema", + default=Path() / ENTITY_RELAXNG_FNAME, + type=Path, + help="The path to mod's root location.", + ) + parser.add_argument( + "-t", "--templates", nargs="*", help="Optionally, a list of templates to validate." + ) + parser.add_argument("-v", "--verbose", help="Be verbose about the output.", default=False) args = parser.parse_args(argv) @@ -72,8 +85,9 @@ def main(argv: Sequence[str] | None = None) -> int: if args.templates: templates = sorted([(Path(t), None) for t in args.templates]) else: - templates = sorted(find_files(args.vfs_root, [args.mod_name], - SIMUL_TEMPLATES_PATH.as_posix(), "xml")) + templates = sorted( + find_files(args.vfs_root, [args.mod_name], SIMUL_TEMPLATES_PATH.as_posix(), "xml") + ) simul_template_entity = SimulTemplateEntity(args.vfs_root, logger) count, failed = 0, 0 @@ -82,29 +96,32 @@ def main(argv: Sequence[str] | None = None) -> int: continue path = fp.as_posix() - if (path.startswith(f"{SIMUL_TEMPLATES_PATH.as_posix()}/mixins/") - or path.startswith( - f"{SIMUL_TEMPLATES_PATH.as_posix()}/special/")): + if path.startswith(f"{SIMUL_TEMPLATES_PATH.as_posix()}/mixins/") or path.startswith( + f"{SIMUL_TEMPLATES_PATH.as_posix()}/special/" + ): continue - if (args.verbose): + if args.verbose: logger.info(f"Parsing {fp}...") count += 1 entity = simul_template_entity.load_inherited( - SIMUL_TEMPLATES_PATH, - str(fp.relative_to(SIMUL_TEMPLATES_PATH)), - [args.mod_name] + SIMUL_TEMPLATES_PATH, str(fp.relative_to(SIMUL_TEMPLATES_PATH)), [args.mod_name] ) xmlcontent = ElementTree.tostring(entity, encoding="unicode") try: - run(["xmllint", "--relaxng", - str(args.relaxng_schema.resolve()), "-"], - input=xmlcontent, encoding="utf-8", capture_output=True, text=True, check=True) + run( + ["xmllint", "--relaxng", str(args.relaxng_schema.resolve()), "-"], + input=xmlcontent, + encoding="utf-8", + capture_output=True, + text=True, + check=True, + ) except CalledProcessError as e: failed += 1 - if (e.stderr): + if e.stderr: logger.error(e.stderr) - if (e.stdout): + if e.stdout: logger.info(e.stdout) logger.info(f"Total: {count}; failed: {failed}") diff --git a/source/tools/entity/scriptlib/__init__.py b/source/tools/entity/scriptlib/__init__.py index b8dca9d585..d6e00a519e 100644 --- a/source/tools/entity/scriptlib/__init__.py +++ b/source/tools/entity/scriptlib/__init__.py @@ -4,6 +4,7 @@ from re import split from xml.etree import ElementTree from os.path import exists + class SimulTemplateEntity: def __init__(self, vfs_root, logger): self.vfs_root = vfs_root @@ -11,11 +12,11 @@ class SimulTemplateEntity: def get_file(self, base_path, vfs_path, mod): default_path = self.vfs_root / mod / base_path - file = (default_path / "special" / "filter" / vfs_path).with_suffix('.xml') + file = (default_path / "special" / "filter" / vfs_path).with_suffix(".xml") if not exists(file): - file = (default_path / "mixins" / vfs_path).with_suffix('.xml') + file = (default_path / "mixins" / vfs_path).with_suffix(".xml") if not exists(file): - file = (default_path / vfs_path).with_suffix('.xml') + file = (default_path / vfs_path).with_suffix(".xml") return file def get_main_mod(self, base_path, vfs_path, mods): @@ -35,52 +36,52 @@ class SimulTemplateEntity: """ apply tag layer to base_tag """ - if tag.get('datatype') == 'tokens': - base_tokens = split(r'\s+', base_tag.text or '') - tokens = split(r'\s+', tag.text or '') + if tag.get("datatype") == "tokens": + base_tokens = split(r"\s+", base_tag.text or "") + tokens = split(r"\s+", tag.text or "") final_tokens = base_tokens.copy() for token in tokens: - if token.startswith('-'): + if token.startswith("-"): token_to_remove = token[1:] if token_to_remove in final_tokens: final_tokens.remove(token_to_remove) elif token not in final_tokens: final_tokens.append(token) - base_tag.text = ' '.join(final_tokens) + base_tag.text = " ".join(final_tokens) base_tag.set("datatype", "tokens") - elif tag.get('op'): - op = tag.get('op') - op1 = Decimal(base_tag.text or '0') - op2 = Decimal(tag.text or '0') + elif tag.get("op"): + op = tag.get("op") + op1 = Decimal(base_tag.text or "0") + op2 = Decimal(tag.text or "0") # Try converting to integers if possible, to pass validation. - if op == 'add': + if op == "add": base_tag.text = str(int(op1 + op2) if int(op1 + op2) == op1 + op2 else op1 + op2) - elif op == 'mul': + elif op == "mul": base_tag.text = str(int(op1 * op2) if int(op1 * op2) == op1 * op2 else op1 * op2) - elif op == 'mul_round': + elif op == "mul_round": base_tag.text = str(round(op1 * op2)) else: raise ValueError(f"Invalid operator '{op}'") else: base_tag.text = tag.text for prop in tag.attrib: - if prop not in ('disable', 'replace', 'parent', 'merge'): + if prop not in ("disable", "replace", "parent", "merge"): base_tag.set(prop, tag.get(prop)) for child in tag: base_child = base_tag.find(child.tag) - if 'disable' in child.attrib: + if "disable" in child.attrib: if base_child is not None: base_tag.remove(base_child) - elif ('merge' not in child.attrib) or (base_child is not None): - if 'replace' in child.attrib and base_child is not None: + elif ("merge" not in child.attrib) or (base_child is not None): + if "replace" in child.attrib and base_child is not None: base_tag.remove(base_child) base_child = None if base_child is None: base_child = ElementTree.Element(child.tag) base_tag.append(base_child) self.apply_layer(base_child, child) - if 'replace' in base_child.attrib: - del base_child.attrib['replace'] + if "replace" in base_child.attrib: + del base_child.attrib["replace"] def load_inherited(self, base_path, vfs_path, mods): entity = self._load_inherited(base_path, vfs_path, mods) @@ -91,7 +92,7 @@ class SimulTemplateEntity: """ vfs_path should be relative to base_path in a mod """ - if '|' in vfs_path: + if "|" in vfs_path: paths = vfs_path.split("|", 1) base = self._load_inherited(base_path, paths[1], mods, base) base = self._load_inherited(base_path, paths[0], mods, base) @@ -106,8 +107,8 @@ class SimulTemplateEntity: if duplicates: for dup in duplicates: self.logger.warning(f"Duplicate child node '{dup}' in tag {el.tag} of {fp}") - if layer.get('parent'): - parent = self._load_inherited(base_path, layer.get('parent'), mods, base) + if layer.get("parent"): + parent = self._load_inherited(base_path, layer.get("parent"), mods, base) self.apply_layer(parent, layer) return parent else: @@ -124,15 +125,20 @@ def find_files(vfs_root, mods, vfs_path, *ext_list): - Path relative to the mod base - full Path """ - full_exts = ['.' + ext for ext in ext_list] + full_exts = ["." + ext for ext in ext_list] def find_recursive(dp, base): """(relative Path, full Path) generator""" if dp.is_dir(): - if dp.name != '.svn' and dp.name != '.git' and not dp.name.endswith('~'): + if dp.name != ".svn" and dp.name != ".git" and not dp.name.endswith("~"): for fp in dp.iterdir(): yield from find_recursive(fp, base) elif dp.suffix in full_exts: relative_file_path = dp.relative_to(base) yield (relative_file_path, dp.resolve()) - return [(rp, fp) for mod in mods for (rp, fp) in find_recursive(vfs_root / mod / vfs_path, vfs_root / mod)] + + return [ + (rp, fp) + for mod in mods + for (rp, fp) in find_recursive(vfs_root / mod / vfs_path, vfs_root / mod) + ] diff --git a/source/tools/fontbuilder2/FontLoader.py b/source/tools/fontbuilder2/FontLoader.py index 9ebb704427..35b2339fe2 100644 --- a/source/tools/fontbuilder2/FontLoader.py +++ b/source/tools/fontbuilder2/FontLoader.py @@ -20,47 +20,54 @@ else: ft_lib = "libfreetype.so.6" lc_lib = "libcairo.so.2" -_freetype_so = ctypes.CDLL (ft_lib) -_cairo_so = ctypes.CDLL (lc_lib) +_freetype_so = ctypes.CDLL(ft_lib) +_cairo_so = ctypes.CDLL(lc_lib) _cairo_so.cairo_ft_font_face_create_for_ft_face.restype = ctypes.c_void_p -_cairo_so.cairo_ft_font_face_create_for_ft_face.argtypes = [ ctypes.c_void_p, ctypes.c_int ] -_cairo_so.cairo_set_font_face.argtypes = [ ctypes.c_void_p, ctypes.c_void_p ] -_cairo_so.cairo_font_face_status.argtypes = [ ctypes.c_void_p ] -_cairo_so.cairo_status.argtypes = [ ctypes.c_void_p ] +_cairo_so.cairo_ft_font_face_create_for_ft_face.argtypes = [ctypes.c_void_p, ctypes.c_int] +_cairo_so.cairo_set_font_face.argtypes = [ctypes.c_void_p, ctypes.c_void_p] +_cairo_so.cairo_font_face_status.argtypes = [ctypes.c_void_p] +_cairo_so.cairo_status.argtypes = [ctypes.c_void_p] # initialize freetype -_ft_lib = ctypes.c_void_p () -if FT_Err_Ok != _freetype_so.FT_Init_FreeType (ctypes.byref (_ft_lib)): +_ft_lib = ctypes.c_void_p() +if FT_Err_Ok != _freetype_so.FT_Init_FreeType(ctypes.byref(_ft_lib)): raise Exception("Error initialising FreeType library.") -_surface = cairo.ImageSurface (cairo.FORMAT_A8, 0, 0) +_surface = cairo.ImageSurface(cairo.FORMAT_A8, 0, 0) + class PycairoContext(ctypes.Structure): - _fields_ = [("PyObject_HEAD", ctypes.c_byte * object.__basicsize__), + _fields_ = [ + ("PyObject_HEAD", ctypes.c_byte * object.__basicsize__), ("ctx", ctypes.c_void_p), - ("base", ctypes.c_void_p)] + ("base", ctypes.c_void_p), + ] -def create_cairo_font_face_for_file (filename, faceindex=0, loadoptions=0): + +def create_cairo_font_face_for_file(filename, faceindex=0, loadoptions=0): # create freetype face ft_face = ctypes.c_void_p() - cairo_ctx = cairo.Context (_surface) + cairo_ctx = cairo.Context(_surface) cairo_t = PycairoContext.from_address(id(cairo_ctx)).ctx - if FT_Err_Ok != _freetype_so.FT_New_Face (_ft_lib, filename.encode('ascii'), faceindex, ctypes.byref(ft_face)): + if FT_Err_Ok != _freetype_so.FT_New_Face( + _ft_lib, filename.encode("ascii"), faceindex, ctypes.byref(ft_face) + ): raise Exception("Error creating FreeType font face for " + filename) # create cairo font face for freetype face - cr_face = _cairo_so.cairo_ft_font_face_create_for_ft_face (ft_face, loadoptions) - if CAIRO_STATUS_SUCCESS != _cairo_so.cairo_font_face_status (cr_face): + cr_face = _cairo_so.cairo_ft_font_face_create_for_ft_face(ft_face, loadoptions) + if CAIRO_STATUS_SUCCESS != _cairo_so.cairo_font_face_status(cr_face): raise Exception("Error creating cairo font face for " + filename) - _cairo_so.cairo_set_font_face (cairo_t, cr_face) - if CAIRO_STATUS_SUCCESS != _cairo_so.cairo_status (cairo_t): + _cairo_so.cairo_set_font_face(cairo_t, cr_face) + if CAIRO_STATUS_SUCCESS != _cairo_so.cairo_status(cairo_t): raise Exception("Error creating cairo font face for " + filename) - face = cairo_ctx.get_font_face () + face = cairo_ctx.get_font_face() - indexes = lambda char: _freetype_so.FT_Get_Char_Index(ft_face, ord(char)) + def indexes(char): + return _freetype_so.FT_Get_Char_Index(ft_face, ord(char)) return (face, indexes) diff --git a/source/tools/fontbuilder2/Packer.py b/source/tools/fontbuilder2/Packer.py index eeffb9235a..5c2afb44f1 100644 --- a/source/tools/fontbuilder2/Packer.py +++ b/source/tools/fontbuilder2/Packer.py @@ -18,59 +18,64 @@ from bisect import bisect_left -class OutOfSpaceError(Exception): pass + +class OutOfSpaceError(Exception): + pass + class Point(object): def __init__(self, x, y): self.x = x self.y = y - + def __cmp__(self, other): """Compares the starting position of height slices""" return self.x - other.x + class RectanglePacker(object): """Base class for rectangle packing algorithms - + By uniting all rectangle packers under this common base class, you can easily switch between different algorithms to find the most efficient or performant one for a given job. - + An almost exhaustive list of packing algorithms can be found here: http://www.csc.liv.ac.uk/~epa/surveyhtml.html""" - + def __init__(self, packingAreaWidth, packingAreaHeight): """Initializes a new rectangle packer - + packingAreaWidth: Maximum width of the packing area packingAreaHeight: Maximum height of the packing area""" self.packingAreaWidth = packingAreaWidth self.packingAreaHeight = packingAreaHeight - + def Pack(self, rectangleWidth, rectangleHeight): """Allocates space for a rectangle in the packing area - + rectangleWidth: Width of the rectangle to allocate rectangleHeight: Height of the rectangle to allocate - + Returns the location at which the rectangle has been placed""" point = self.TryPack(rectangleWidth, rectangleHeight) - + if not point: raise OutOfSpaceError("Rectangle does not fit in packing area") - + return point - + def TryPack(self, rectangleWidth, rectangleHeight): """Tries to allocate space for a rectangle in the packing area - + rectangleWidth: Width of the rectangle to allocate rectangleHeight: Height of the rectangle to allocate - + Returns a Point instance if space for the rectangle could be allocated be found, otherwise returns None""" raise NotImplementedError + class DumbRectanglePacker(RectanglePacker): def __init__(self, packingAreaWidth, packingAreaHeight): RectanglePacker.__init__(self, packingAreaWidth, packingAreaHeight) @@ -91,86 +96,85 @@ class DumbRectanglePacker(RectanglePacker): self.rowh = max(self.rowh, rectangleHeight) return r + class CygonRectanglePacker(RectanglePacker): """ Packer using a custom algorithm by Markus 'Cygon' Ewald - + Algorithm conceived by Markus Ewald (cygon at nuclex dot org), though I'm quite sure I'm not the first one to come up with it :) - + The algorithm always places rectangles as low as possible in the packing area. So, for any new rectangle that is to be added, the packer has to determine the X coordinate at which the rectangle can have the lowest overall height without intersecting any other rectangles. - + To quickly discover these locations, the packer uses a sophisticated data structure that stores the upper silhouette of the packing area. When a new rectangle needs to be added, only the silouette edges need to be analyzed to find the position where the rectangle would achieve the lowest""" - + def __init__(self, packingAreaWidth, packingAreaHeight): """Initializes a new rectangle packer - + packingAreaWidth: Maximum width of the packing area packingAreaHeight: Maximum height of the packing area""" RectanglePacker.__init__(self, packingAreaWidth, packingAreaHeight) - + # Stores the height silhouette of the rectangles self.heightSlices = [] - + # At the beginning, the packing area is a single slice of height 0 - self.heightSlices.append(Point(0,0)) - + self.heightSlices.append(Point(0, 0)) + def TryPack(self, rectangleWidth, rectangleHeight): """Tries to allocate space for a rectangle in the packing area - + rectangleWidth: Width of the rectangle to allocate rectangleHeight: Height of the rectangle to allocate - + Returns a Point instance if space for the rectangle could be allocated be found, otherwise returns None""" placement = None - + # If the rectangle is larger than the packing area in any dimension, # it will never fit! - if rectangleWidth > self.packingAreaWidth or rectangleHeight > \ - self.packingAreaHeight: + if rectangleWidth > self.packingAreaWidth or rectangleHeight > self.packingAreaHeight: return None - + # Determine the placement for the new rectangle placement = self.tryFindBestPlacement(rectangleWidth, rectangleHeight) - + # If a place for the rectangle could be found, update the height slice # table to mark the region of the rectangle as being taken. if placement: - self.integrateRectangle(placement.x, rectangleWidth, placement.y \ - + rectangleHeight) - + self.integrateRectangle(placement.x, rectangleWidth, placement.y + rectangleHeight) + return placement - + def tryFindBestPlacement(self, rectangleWidth, rectangleHeight): """Finds the best position for a rectangle of the given dimensions - + rectangleWidth: Width of the rectangle to find a position for rectangleHeight: Height of the rectangle to find a position for - + Returns a Point instance if a valid placement for the rectangle could be found, otherwise returns None""" # Slice index, vertical position and score of the best placement we # could find - bestSliceIndex = -1 # Slice index where the best placement was found - bestSliceY = 0 # Y position of the best placement found + bestSliceIndex = -1 # Slice index where the best placement was found + bestSliceY = 0 # Y position of the best placement found # lower == better! - bestScore = self.packingAreaHeight + bestScore = self.packingAreaHeight # This is the counter for the currently checked position. The search # works by skipping from slice to slice, determining the suitability # of the location for the placement of the rectangle. leftSliceIndex = 0 - + # Determine the slice in which the right end of the rectangle is located rightSliceIndex = bisect_left(self.heightSlices, Point(rectangleWidth, 0)) - + while rightSliceIndex <= len(self.heightSlices): # Determine the highest slice within the slices covered by the # rectangle at its current placement. We cannot put the rectangle @@ -179,21 +183,21 @@ class CygonRectanglePacker(RectanglePacker): for index in range(leftSliceIndex + 1, rightSliceIndex): if self.heightSlices[index].y > highest: highest = self.heightSlices[index].y - + # Only process this position if it doesn't leave the packing area if highest + rectangleHeight < self.packingAreaHeight: score = highest - + if score < bestScore: bestSliceIndex = leftSliceIndex bestSliceY = highest bestScore = score - + # Advance the starting slice to the next slice start leftSliceIndex += 1 if leftSliceIndex >= len(self.heightSlices): break - + # Advance the ending slice until we're on the proper slice again, # given the new starting position of the rectangle. rightRectangleEnd = self.heightSlices[leftSliceIndex].x + rectangleWidth @@ -202,18 +206,18 @@ class CygonRectanglePacker(RectanglePacker): rightSliceStart = self.packingAreaWidth else: rightSliceStart = self.heightSlices[rightSliceIndex].x - + # Is this the slice we're looking for? if rightSliceStart > rightRectangleEnd: break - + rightSliceIndex += 1 - + # If we crossed the end of the slice array, the rectangle's right # end has left the packing area, and thus, our search ends. if rightSliceIndex > len(self.heightSlices): break - + # Return the best placement we found for this rectangle. If the # rectangle didn't fit anywhere, the slice index will still have its # initialization value of -1 and we can report that no placement @@ -222,23 +226,23 @@ class CygonRectanglePacker(RectanglePacker): return None else: return Point(self.heightSlices[bestSliceIndex].x, bestSliceY) - + def integrateRectangle(self, left, width, bottom): """Integrates a new rectangle into the height slice table - + left: Position of the rectangle's left side width: Width of the rectangle bottom: Position of the rectangle's lower side""" # Find the first slice that is touched by the rectangle startSlice = bisect_left(self.heightSlices, Point(left, 0)) - + # We scored a direct hit, so we can replace the slice we have hit firstSliceOriginalHeight = self.heightSlices[startSlice].y self.heightSlices[startSlice] = Point(left, bottom) - + right = left + width startSlice += 1 - + # Special case, the rectangle started on the last slice, so we cannot # use the start slice + 1 for the binary search and the possibly # already modified start slice height now only remains in our temporary @@ -249,21 +253,24 @@ class CygonRectanglePacker(RectanglePacker): # to return to the original height at the end of the rectangle. if right < self.packingAreaWidth: self.heightSlices.append(Point(right, firstSliceOriginalHeight)) - else: # The rectangle doesn't start on the last slice - endSlice = bisect_left(self.heightSlices, Point(right,0), \ - startSlice, len(self.heightSlices)) - + else: # The rectangle doesn't start on the last slice + endSlice = bisect_left( + self.heightSlices, Point(right, 0), startSlice, len(self.heightSlices) + ) + # Another direct hit on the final slice's end? - if endSlice < len(self.heightSlices) and not (Point(right, 0) < self.heightSlices[endSlice]): + if endSlice < len(self.heightSlices) and not ( + Point(right, 0) < self.heightSlices[endSlice] + ): del self.heightSlices[startSlice:endSlice] - else: # No direct hit, rectangle ends inside another slice + else: # No direct hit, rectangle ends inside another slice # Find out to which height we need to return at the right end of # the rectangle if endSlice == startSlice: returnHeight = firstSliceOriginalHeight else: returnHeight = self.heightSlices[endSlice - 1].y - + # Remove all slices covered by the rectangle and begin a new # slice at its end to return back to the height of the slice on # which the rectangle ends. diff --git a/source/tools/fontbuilder2/dumpfontchars.py b/source/tools/fontbuilder2/dumpfontchars.py index 482fc0f481..52f8fc04db 100644 --- a/source/tools/fontbuilder2/dumpfontchars.py +++ b/source/tools/fontbuilder2/dumpfontchars.py @@ -4,13 +4,16 @@ import FontLoader + def dump_font(ttf): + (face, indexes) = FontLoader.create_cairo_font_face_for_file( + "../../../binaries/data/tools/fontbuilder/fonts/%s" % ttf, 0, FontLoader.FT_LOAD_DEFAULT + ) - (face, indexes) = FontLoader.create_cairo_font_face_for_file("../../../binaries/data/tools/fontbuilder/fonts/%s" % ttf, 0, FontLoader.FT_LOAD_DEFAULT) + mappings = [(c, indexes(chr(c))) for c in range(1, 65535)] + print(ttf, end=" ") + print(" ".join(str(c) for (c, g) in mappings if g != 0)) - mappings = [ (c, indexes(chr(c))) for c in range(1, 65535) ] - print(ttf, end=' ') - print(' '.join(str(c) for (c, g) in mappings if g != 0)) dump_font("DejaVuSansMono.ttf") dump_font("FreeSans.ttf") diff --git a/source/tools/fontbuilder2/fontbuilder.py b/source/tools/fontbuilder2/fontbuilder.py index a112e73d92..98c81392dd 100644 --- a/source/tools/fontbuilder2/fontbuilder.py +++ b/source/tools/fontbuilder2/fontbuilder.py @@ -7,6 +7,7 @@ import math import FontLoader import Packer + # Representation of a rendered glyph class Glyph(object): def __init__(self, ctx, renderstyle, char, idx, face, size): @@ -30,7 +31,7 @@ class Glyph(object): bb = [inf, inf, -inf, -inf] if "stroke" in self.renderstyle: - for (c, w) in self.renderstyle["stroke"]: + for c, w in self.renderstyle["stroke"]: ctx.set_line_width(w) ctx.glyph_path([self.glyph]) e = ctx.stroke_extents() @@ -52,8 +53,8 @@ class Glyph(object): # Force multiple of 4, to avoid leakage across S3TC blocks # (TODO: is this useful?) - #self.w += (4 - (self.w % 4)) % 4 - #self.h += (4 - (self.h % 4)) % 4 + # self.w += (4 - (self.w % 4)) % 4 + # self.h += (4 - (self.h % 4)) % 4 def pack(self, packer): self.pos = packer.Pack(self.w, self.h) @@ -69,20 +70,21 @@ class Glyph(object): # Render each stroke, and then each fill on top of it if "stroke" in self.renderstyle: - for ((r, g, b, a), w) in self.renderstyle["stroke"]: + for (r, g, b, a), w in self.renderstyle["stroke"]: ctx.set_line_width(w) ctx.set_source_rgba(r, g, b, a) ctx.glyph_path([self.glyph]) ctx.stroke() if "fill" in self.renderstyle: - for (r, g, b, a) in self.renderstyle["fill"]: + for r, g, b, a in self.renderstyle["fill"]: ctx.set_source_rgba(r, g, b, a) ctx.glyph_path([self.glyph]) ctx.fill() ctx.restore() + # Load the set of characters contained in the given text file def load_char_list(filename): f = codecs.open(filename, "r", "utf-8") @@ -90,22 +92,25 @@ def load_char_list(filename): f.close() return set(chars) + # Construct a Cairo context and surface for rendering text with the given parameters def setup_context(width, height, renderstyle): - format = (cairo.FORMAT_ARGB32 if "colour" in renderstyle else cairo.FORMAT_A8) + format = cairo.FORMAT_ARGB32 if "colour" in renderstyle else cairo.FORMAT_A8 surface = cairo.ImageSurface(format, width, height) ctx = cairo.Context(surface) ctx.set_line_join(cairo.LINE_JOIN_ROUND) return ctx, surface -def generate_font(outname, ttfNames, loadopts, size, renderstyle, dsizes): +def generate_font(outname, ttfNames, loadopts, size, renderstyle, dsizes): faceList = [] indexList = [] for i in range(len(ttfNames)): - (face, indices) = FontLoader.create_cairo_font_face_for_file("../../../binaries/data/tools/fontbuilder/fonts/%s" % ttfNames[i], 0, loadopts) + (face, indices) = FontLoader.create_cairo_font_face_for_file( + "../../../binaries/data/tools/fontbuilder/fonts/%s" % ttfNames[i], 0, loadopts + ) faceList.append(face) - if not ttfNames[i] in dsizes: + if ttfNames[i] not in dsizes: dsizes[ttfNames[i]] = 0 indexList.append(indices) @@ -123,32 +128,36 @@ def generate_font(outname, ttfNames, loadopts, size, renderstyle, dsizes): # Translate all the characters into glyphs # (This is inefficient if multiple characters have the same glyph) glyphs = [] - #for c in chars: + # for c in chars: for c in range(0x20, 0xFFFE): for i in range(len(indexList)): idx = indexList[i](chr(c)) - if c == 0xFFFD and idx == 0: # use "?" if the missing-glyph glyph is missing + if c == 0xFFFD and idx == 0: # use "?" if the missing-glyph glyph is missing idx = indexList[i]("?") if idx: - glyphs.append(Glyph(ctx, renderstyle, chr(c), idx, faceList[i], size + dsizes[ttfNames[i]])) + glyphs.append( + Glyph(ctx, renderstyle, chr(c), idx, faceList[i], size + dsizes[ttfNames[i]]) + ) break # Sort by decreasing height (tie-break on decreasing width) - glyphs.sort(key = lambda g: (-g.h, -g.w)) + glyphs.sort(key=lambda g: (-g.h, -g.w)) # Try various sizes to pack the glyphs into sizes = [] for h in [32, 64, 128, 256, 512, 1024, 2048, 4096]: sizes.append((h, h)) - sizes.append((h*2, h)) - sizes.sort(key = lambda w_h: (w_h[0]*w_h[1], max(w_h[0], w_h[1]))) # prefer smaller and squarer + sizes.append((h * 2, h)) + sizes.sort( + key=lambda w_h: (w_h[0] * w_h[1], max(w_h[0], w_h[1])) + ) # prefer smaller and squarer for w, h in sizes: try: # Using the dump pacher usually creates bigger textures, but runs faster # In practice the size difference is so small it always ends up in the same size packer = Packer.DumbRectanglePacker(w, h) - #packer = Packer.CygonRectanglePacker(w, h) + # packer = Packer.CygonRectanglePacker(w, h) for g in glyphs: g.pack(packer) except Packer.OutOfSpaceError: @@ -168,7 +177,7 @@ def generate_font(outname, ttfNames, loadopts, size, renderstyle, dsizes): fnt.write("%d\n" % linespacing) fnt.write("%d\n" % charheight) # sorting unneeded, as glyphs are added in increasing order - #glyphs.sort(key = lambda g: ord(g.char)) + # glyphs.sort(key = lambda g: ord(g.char)) for g in glyphs: x0 = g.x0 y0 = g.y0 @@ -179,31 +188,39 @@ def generate_font(outname, ttfNames, loadopts, size, renderstyle, dsizes): # glyph by an arbitrary amount to make it roughly the right # place when used after an a-macron glyph. if ord(g.char) == 0x0301: - y0 += charheight/3 + y0 += charheight / 3 - fnt.write("%d %d %d %d %d %d %d %d\n" % (ord(g.char), g.pos.x, h-g.pos.y, g.w, g.h, -x0, y0, g.xadvance)) + fnt.write( + "%d %d %d %d %d %d %d %d\n" + % (ord(g.char), g.pos.x, h - g.pos.y, g.w, g.h, -x0, y0, g.xadvance) + ) fnt.close() return print("Failed to fit glyphs in texture") -filled = { "fill": [(1, 1, 1, 1)] } -stroked1 = { "colour": True, "stroke": [((0, 0, 0, 1), 2.0), ((0, 0, 0, 1), 2.0)], "fill": [(1, 1, 1, 1)] } -stroked2 = { "colour": True, "stroke": [((0, 0, 0, 1), 2.0)], "fill": [(1, 1, 1, 1), (1, 1, 1, 1)] } -stroked3 = { "colour": True, "stroke": [((0, 0, 0, 1), 2.5)], "fill": [(1, 1, 1, 1), (1, 1, 1, 1)] } + +filled = {"fill": [(1, 1, 1, 1)]} +stroked1 = { + "colour": True, + "stroke": [((0, 0, 0, 1), 2.0), ((0, 0, 0, 1), 2.0)], + "fill": [(1, 1, 1, 1)], +} +stroked2 = {"colour": True, "stroke": [((0, 0, 0, 1), 2.0)], "fill": [(1, 1, 1, 1), (1, 1, 1, 1)]} +stroked3 = {"colour": True, "stroke": [((0, 0, 0, 1), 2.5)], "fill": [(1, 1, 1, 1), (1, 1, 1, 1)]} # For extra glyph support, add your preferred font to the font array -Sans = (["LinBiolinum_Rah.ttf","FreeSans.ttf"], FontLoader.FT_LOAD_DEFAULT) -Sans_Bold = (["LinBiolinum_RBah.ttf","FreeSansBold.ttf"], FontLoader.FT_LOAD_DEFAULT) -Sans_Italic = (["LinBiolinum_RIah.ttf","FreeSansOblique.ttf"], FontLoader.FT_LOAD_DEFAULT) -SansMono = (["DejaVuSansMono.ttf","FreeMono.ttf"], FontLoader.FT_LOAD_DEFAULT) -Serif = (["texgyrepagella-regular.otf","FreeSerif.ttf"], FontLoader.FT_LOAD_NO_HINTING) -Serif_Bold = (["texgyrepagella-bold.otf","FreeSerifBold.ttf"], FontLoader.FT_LOAD_NO_HINTING) +Sans = (["LinBiolinum_Rah.ttf", "FreeSans.ttf"], FontLoader.FT_LOAD_DEFAULT) +Sans_Bold = (["LinBiolinum_RBah.ttf", "FreeSansBold.ttf"], FontLoader.FT_LOAD_DEFAULT) +Sans_Italic = (["LinBiolinum_RIah.ttf", "FreeSansOblique.ttf"], FontLoader.FT_LOAD_DEFAULT) +SansMono = (["DejaVuSansMono.ttf", "FreeMono.ttf"], FontLoader.FT_LOAD_DEFAULT) +Serif = (["texgyrepagella-regular.otf", "FreeSerif.ttf"], FontLoader.FT_LOAD_NO_HINTING) +Serif_Bold = (["texgyrepagella-bold.otf", "FreeSerifBold.ttf"], FontLoader.FT_LOAD_NO_HINTING) # Define the size differences used to render different fallback fonts # I.e. when adding a fallback font has smaller glyphs than the original, you can bump it -dsizes = {'HanaMinA.ttf': 2} # make the glyphs for the (chinese font 2 pts bigger) +dsizes = {"HanaMinA.ttf": 2} # make the glyphs for the (chinese font 2 pts bigger) fonts = ( ("mono-10", SansMono, 10, filled), @@ -231,6 +248,8 @@ fonts = ( ("sans-stroke-16", Sans, 16, stroked2), ) -for (name, (fontnames, loadopts), size, style) in fonts: +for name, (fontnames, loadopts), size, style in fonts: print("%s..." % name) - generate_font("../../../binaries/data/mods/mod/fonts/%s" % name, fontnames, loadopts, size, style, dsizes) + generate_font( + "../../../binaries/data/mods/mod/fonts/%s" % name, fontnames, loadopts, size, style, dsizes + ) diff --git a/source/tools/i18n/checkDiff.py b/source/tools/i18n/checkDiff.py index 727d68208d..1e7ff7a22d 100644 --- a/source/tools/i18n/checkDiff.py +++ b/source/tools/i18n/checkDiff.py @@ -16,6 +16,8 @@ # You should have received a copy of the GNU General Public License # along with 0 A.D. If not, see . +# ruff: noqa: E741 + import io import os import subprocess @@ -23,6 +25,7 @@ from typing import List from i18n_helper import projectRootDirectory + def get_diff(): """Return a diff using svn diff""" os.chdir(projectRootDirectory) @@ -31,9 +34,10 @@ def get_diff(): if diff_process.returncode != 0: print(f"Error running svn diff: {diff_process.stderr.decode('utf-8')}. Exiting.") return - return io.StringIO(diff_process.stdout.decode('utf-8')) + return io.StringIO(diff_process.stdout.decode("utf-8")) -def check_diff(diff : io.StringIO, verbose = False) -> List[str]: + +def check_diff(diff: io.StringIO, verbose=False) -> List[str]: """Run through a diff of .po files and check that some of the changes are real translations changes and not just noise (line changes....). The algorithm isn't extremely clever, but it is quite fast.""" @@ -57,13 +61,18 @@ def check_diff(diff : io.StringIO, verbose = False) -> List[str]: diff.readline() l = diff.readline() continue - if l[0] != '-' and l[0] != '+': + if l[0] != "-" and l[0] != "+": l = diff.readline() continue - if l[1:].strip() == "" or (l[1] == '#' and l[2] == ':'): + if l[1:].strip() == "" or (l[1] == "#" and l[2] == ":"): l = diff.readline() continue - if "# Copyright (C)" in l or "POT-Creation-Date:" in l or "PO-Revision-Date" in l or "Last-Translator" in l: + if ( + "# Copyright (C)" in l + or "POT-Creation-Date:" in l + or "PO-Revision-Date" in l + or "Last-Translator" in l + ): l = diff.readline() continue # We've hit a real line @@ -75,23 +84,25 @@ def check_diff(diff : io.StringIO, verbose = False) -> List[str]: return list(files.difference(keep)) -def revert_files(files: List[str], verbose = False): +def revert_files(files: List[str], verbose=False): revert_process = subprocess.run(["svn", "revert"] + files, capture_output=True) if revert_process.returncode != 0: - print(f"Warning: Some files could not be reverted. Error: {revert_process.stderr.decode('utf-8')}") + print( + f"Warning: Some files could not be reverted. Error: {revert_process.stderr.decode('utf-8')}" + ) if verbose: for file in files: print(f"Reverted {file}") -def add_untracked(verbose = False): +def add_untracked(verbose=False): """Add untracked .po files to svn""" diff_process = subprocess.run(["svn", "st", "binaries"], capture_output=True) - if diff_process.stderr != b'': + if diff_process.stderr != b"": print(f"Error running svn st: {diff_process.stderr.decode('utf-8')}. Exiting.") return - for line in diff_process.stdout.decode('utf-8').split('\n'): + for line in diff_process.stdout.decode("utf-8").split("\n"): if not line.startswith("?"): continue # Ignore non PO files. This is important so that the translator credits @@ -100,16 +111,17 @@ def add_untracked(verbose = False): if not file.endswith(".po") and not file.endswith(".pot"): continue add_process = subprocess.run(["svn", "add", file, "--parents"], capture_output=True) - if add_process.stderr != b'': + if add_process.stderr != b"": print(f"Warning: file {file} could not be added.") if verbose: print(f"Added {file}") -if __name__ == '__main__': +if __name__ == "__main__": import argparse + parser = argparse.ArgumentParser() - parser.add_argument("--verbose", help="Print reverted files.", action='store_true') + parser.add_argument("--verbose", help="Print reverted files.", action="store_true") args = parser.parse_args() need_revert = check_diff(get_diff(), args.verbose) revert_files(need_revert, args.verbose) diff --git a/source/tools/i18n/checkTranslations.py b/source/tools/i18n/checkTranslations.py index a1243f2854..94178186b3 100644 --- a/source/tools/i18n/checkTranslations.py +++ b/source/tools/i18n/checkTranslations.py @@ -16,7 +16,10 @@ # You should have received a copy of the GNU General Public License # along with 0 A.D. If not, see . -import sys, os, re, multiprocessing +import sys +import os +import re +import multiprocessing from i18n_helper import l10nFolderName, projectRootDirectory from i18n_helper.catalog import Catalog @@ -27,14 +30,17 @@ VERBOSE = 0 class MessageChecker: """Checks all messages in a catalog against a regex.""" + def __init__(self, human_name, regex): self.regex = re.compile(regex, re.IGNORECASE) self.human_name = human_name def check(self, inputFilePath, templateMessage, translatedCatalogs): - patterns = set(self.regex.findall( - templateMessage.id[0] if templateMessage.pluralizable else templateMessage.id - )) + patterns = set( + self.regex.findall( + templateMessage.id[0] if templateMessage.pluralizable else templateMessage.id + ) + ) # As a sanity check, verify that the template message is coherent. # Note that these tend to be false positives. @@ -42,23 +48,32 @@ class MessageChecker: if templateMessage.pluralizable: pluralUrls = set(self.regex.findall(templateMessage.id[1])) if pluralUrls.difference(patterns): - print(f"{inputFilePath} - Different {self.human_name} in singular and plural source strings " - f"for '{templateMessage}' in '{inputFilePath}'") + print( + f"{inputFilePath} - Different {self.human_name} in singular and plural source strings " + f"for '{templateMessage}' in '{inputFilePath}'" + ) for translationCatalog in translatedCatalogs: translationMessage = translationCatalog.get( - templateMessage.id, templateMessage.context) + templateMessage.id, templateMessage.context + ) if not translationMessage: continue - translatedPatterns = set(self.regex.findall( - translationMessage.string[0] if translationMessage.pluralizable else translationMessage.string - )) + translatedPatterns = set( + self.regex.findall( + translationMessage.string[0] + if translationMessage.pluralizable + else translationMessage.string + ) + ) unknown_patterns = translatedPatterns.difference(patterns) if unknown_patterns: - print(f'{inputFilePath} - {translationCatalog.locale}: ' - f'Found unknown {self.human_name} {", ".join(["`" + x + "`" for x in unknown_patterns])} in the translation ' - f'which do not match any of the URLs in the template: {", ".join(["`" + x + "`" for x in patterns])}') + print( + f'{inputFilePath} - {translationCatalog.locale}: ' + f'Found unknown {self.human_name} {", ".join(["`" + x + "`" for x in unknown_patterns])} in the translation ' + f'which do not match any of the URLs in the template: {", ".join(["`" + x + "`" for x in patterns])}' + ) if templateMessage.pluralizable and translationMessage.pluralizable: for indx, val in enumerate(translationMessage.string): @@ -67,9 +82,12 @@ class MessageChecker: translatedPatternsMulti = set(self.regex.findall(val)) unknown_patterns_multi = translatedPatternsMulti.difference(pluralUrls) if unknown_patterns_multi: - print(f'{inputFilePath} - {translationCatalog.locale}: ' - f'Found unknown {self.human_name} {", ".join(["`" + x + "`" for x in unknown_patterns_multi])} in the pluralised translation ' - f'which do not match any of the URLs in the template: {", ".join(["`" + x + "`" for x in pluralUrls])}') + print( + f'{inputFilePath} - {translationCatalog.locale}: ' + f'Found unknown {self.human_name} {", ".join(["`" + x + "`" for x in unknown_patterns_multi])} in the pluralised translation ' + f'which do not match any of the URLs in the template: {", ".join(["`" + x + "`" for x in pluralUrls])}' + ) + def check_translations(inputFilePath): if VERBOSE: @@ -100,23 +118,29 @@ def check_translations(inputFilePath): def main(): - print("\n\tWARNING: Remember to regenerate the POT files with “updateTemplates.py” " - "before you run this script.\n\tPOT files are not in the repository.\n") + print( + "\n\tWARNING: Remember to regenerate the POT files with “updateTemplates.py” " + "before you run this script.\n\tPOT files are not in the repository.\n" + ) foundPots = 0 for root, folders, filenames in os.walk(projectRootDirectory): for filename in filenames: - if len(filename) > 4 and filename[-4:] == ".pot" and os.path.basename(root) == l10nFolderName: + if ( + len(filename) > 4 + and filename[-4:] == ".pot" + and os.path.basename(root) == l10nFolderName + ): foundPots += 1 multiprocessing.Process( - target=check_translations, - args=(os.path.join(root, filename), ) + target=check_translations, args=(os.path.join(root, filename),) ).start() if foundPots == 0: print( "This script did not work because no '.pot' files were found. " "Please run 'updateTemplates.py' to generate the '.pot' files, " "and run 'pullTranslations.py' to pull the latest translations from Transifex. " - "Then you can run this script to check for spam in translations.") + "Then you can run this script to check for spam in translations." + ) if __name__ == "__main__": diff --git a/source/tools/i18n/cleanTranslationFiles.py b/source/tools/i18n/cleanTranslationFiles.py index 7ad3981b2a..c807d38fdf 100644 --- a/source/tools/i18n/cleanTranslationFiles.py +++ b/source/tools/i18n/cleanTranslationFiles.py @@ -26,10 +26,15 @@ However that needs to be fixed on the transifex side, see rP25896. For now strip the e-mails using this script. """ -import sys, os, glob, re, fileinput +import sys +import os +import glob +import re +import fileinput from i18n_helper import l10nFolderName, transifexClientFolder, projectRootDirectory + def main(): translatorMatch = re.compile(r"^(#\s+[^,<]*)\s+<.*>(.*)") lastTranslatorMatch = re.compile(r"^(\"Last-Translator:[^,<]*)\s+<.*>(.*)") @@ -43,7 +48,9 @@ def main(): for file in files: usernames = [] reached = False - for line in fileinput.input(file.replace("\\", "/"), inplace=True, encoding="utf-8"): + for line in fileinput.input( + file.replace("\\", "/"), inplace=True, encoding="utf-8" + ): if reached: if line == "# \n": line = "" @@ -61,5 +68,6 @@ def main(): reached = True sys.stdout.write(line) + if __name__ == "__main__": main() diff --git a/source/tools/i18n/creditTranslators.py b/source/tools/i18n/creditTranslators.py index 592d498635..0e15d9408d 100755 --- a/source/tools/i18n/creditTranslators.py +++ b/source/tools/i18n/creditTranslators.py @@ -29,7 +29,9 @@ Translatable strings will be extracted from the generated file, so this should b once before updateTemplates.py. """ -import json, os, re +import json +import os +import re from collections import defaultdict from pathlib import Path @@ -44,13 +46,23 @@ for root, folders, filenames in os.walk(projectRootDirectory): if os.path.exists(os.path.join(root, folder, transifexClientFolder)): poLocations.append(os.path.join(root, folder)) -creditsLocation = os.path.join(projectRootDirectory, 'binaries', 'data', 'mods', 'public', 'gui', 'credits', 'texts', 'translators.json') +creditsLocation = os.path.join( + projectRootDirectory, + "binaries", + "data", + "mods", + "public", + "gui", + "credits", + "texts", + "translators.json", +) # This dictionary will hold creditors lists for each language, indexed by code langsLists = defaultdict(list) # Create the new JSON data -newJSONData = {'Title': 'Translators', 'Content': []} +newJSONData = {"Title": "Translators", "Content": []} # Now go through the list of languages and search the .po files for people @@ -60,7 +72,7 @@ deletedUsernameMatch = re.compile(r"[0-9a-f]{32}(_[0-9a-f]{7})?") # Search for location in poLocations: - files = Path(location).glob('*.po') + files = Path(location).glob("*.po") for file in files: lang = file.stem.split(".")[0] @@ -69,7 +81,7 @@ for location in poLocations: if lang == "debug" or lang == "long": continue - with file.open(encoding='utf-8') as poFile: + with file.open(encoding="utf-8") as poFile: reached = False for line in poFile: if reached: @@ -80,7 +92,7 @@ for location in poLocations: username = m.group(1) if not deletedUsernameMatch.fullmatch(username): langsLists[lang].append(username) - if line.strip() == '# Translators:': + if line.strip() == "# Translators:": reached = True # Sort translator names and remove duplicates @@ -100,18 +112,18 @@ for langCode, langList in sorted(langsLists.items()): try: lang_name = Locale.parse(langCode).english_name except UnknownLocaleError: - lang_name = Locale.parse('en').languages.get(langCode) + lang_name = Locale.parse("en").languages.get(langCode) if not lang_name: raise - translators = [{'name': name} for name in langList] - newJSONData['Content'].append({'LangName': lang_name, 'List': translators}) + translators = [{"name": name} for name in langList] + newJSONData["Content"].append({"LangName": lang_name, "List": translators}) # Sort languages by their English names -newJSONData['Content'] = sorted(newJSONData['Content'], key=lambda x: x['LangName']) +newJSONData["Content"] = sorted(newJSONData["Content"], key=lambda x: x["LangName"]) # Save the JSON data to the credits file -creditsFile = open(creditsLocation, 'w', encoding='utf-8') +creditsFile = open(creditsLocation, "w", encoding="utf-8") json.dump(newJSONData, creditsFile, indent=4) creditsFile.close() diff --git a/source/tools/i18n/extractors/extractors.py b/source/tools/i18n/extractors/extractors.py index 039a00bc3c..54cffc62a6 100644 --- a/source/tools/i18n/extractors/extractors.py +++ b/source/tools/i18n/extractors/extractors.py @@ -20,14 +20,17 @@ # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import codecs, re, os, sys +import codecs +import re +import os +import sys import json as jsonParser -from tokenize import generate_tokens, COMMENT, NAME, OP, STRING from textwrap import dedent + def pathmatch(mask, path): - """ Matches paths to a mask, where the mask supports * and **. + """Matches paths to a mask, where the mask supports * and **. Paths use / as the separator * matches a sequence of characters without /. @@ -45,13 +48,11 @@ def pathmatch(mask, path): else: p = p + re.escape(s[i]) p = p + "$" - return re.match(p, path) != None + return re.match(p, path) is not None class Extractor(object): - def __init__(self, directoryPath, filemasks, options): - self.directoryPath = directoryPath self.options = options @@ -62,9 +63,8 @@ class Extractor(object): self.includeMasks = filemasks self.excludeMasks = [] - def run(self): - """ Extracts messages. + """Extracts messages. :return: An iterator over ``(message, plural, context, (location, pos), comment)`` tuples. :rtype: ``iterator`` @@ -73,12 +73,14 @@ class Extractor(object): directoryAbsolutePath = os.path.abspath(self.directoryPath) for root, folders, filenames in os.walk(directoryAbsolutePath): for subdir in folders: - if subdir.startswith('.') or subdir.startswith('_'): + if subdir.startswith(".") or subdir.startswith("_"): folders.remove(subdir) folders.sort() filenames.sort() for filename in filenames: - filename = os.path.relpath(os.path.join(root, filename), self.directoryPath).replace(os.sep, '/') + filename = os.path.relpath( + os.path.join(root, filename), self.directoryPath + ).replace(os.sep, "/") for filemask in self.excludeMasks: if pathmatch(filemask, filename): break @@ -86,7 +88,13 @@ class Extractor(object): for filemask in self.includeMasks: if pathmatch(filemask, filename): filepath = os.path.join(directoryAbsolutePath, filename) - for message, plural, context, position, comments in self.extractFromFile(filepath): + for ( + message, + plural, + context, + position, + comments, + ) in self.extractFromFile(filepath): if empty_string_pattern.match(message): continue @@ -94,9 +102,8 @@ class Extractor(object): filename = "\u2068" + filename + "\u2069" yield message, plural, context, (filename, position), comments - def extractFromFile(self, filepath): - """ Extracts messages from a specific file. + """Extracts messages from a specific file. :return: An iterator over ``(message, plural, context, position, comments)`` tuples. :rtype: ``iterator`` @@ -104,17 +111,17 @@ class Extractor(object): pass - class javascript(Extractor): - """ Extract messages from JavaScript source code. - """ + """Extract messages from JavaScript source code.""" - empty_msgid_warning = ( '%s: warning: Empty msgid. It is reserved by GNU gettext: gettext("") ' - 'returns the header entry with meta information, not the empty string.' ) + empty_msgid_warning = ( + '%s: warning: Empty msgid. It is reserved by GNU gettext: gettext("") ' + "returns the header entry with meta information, not the empty string." + ) def extractJavascriptFromFile(self, fileObject): - from babel.messages.jslexer import tokenize, unquote_string + funcname = message_lineno = None messages = [] last_argument = None @@ -122,21 +129,21 @@ class javascript(Extractor): concatenate_next = False last_token = None call_stack = -1 - comment_tags = self.options.get('commentTags', []) - keywords = self.options.get('keywords', {}).keys() + comment_tags = self.options.get("commentTags", []) + keywords = self.options.get("keywords", {}).keys() for token in tokenize(fileObject.read(), dotted=False): - if token.type == 'operator' and \ - (token.value == '(' or (call_stack != -1 and \ - (token.value == '[' or token.value == '{'))): + if token.type == "operator" and ( + token.value == "(" + or (call_stack != -1 and (token.value == "[" or token.value == "{")) + ): if funcname: message_lineno = token.lineno call_stack += 1 - elif call_stack == -1 and token.type == 'linecomment': + elif call_stack == -1 and token.type == "linecomment": value = token.value[2:].strip() - if translator_comments and \ - translator_comments[-1][0] == token.lineno - 1: + if translator_comments and translator_comments[-1][0] == token.lineno - 1: translator_comments.append((token.lineno, value)) continue @@ -145,7 +152,7 @@ class javascript(Extractor): translator_comments.append((token.lineno, value.strip())) break - elif token.type == 'multilinecomment': + elif token.type == "multilinecomment": # only one multi-line comment may preceed a translation translator_comments = [] value = token.value[2:-2].strip() @@ -154,14 +161,13 @@ class javascript(Extractor): lines = value.splitlines() if lines: lines[0] = lines[0].strip() - lines[1:] = dedent('\n'.join(lines[1:])).splitlines() + lines[1:] = dedent("\n".join(lines[1:])).splitlines() for offset, line in enumerate(lines): - translator_comments.append((token.lineno + offset, - line)) + translator_comments.append((token.lineno + offset, line)) break elif funcname and call_stack == 0: - if token.type == 'operator' and token.value == ')': + if token.type == "operator" and token.value == ")": if last_argument is not None: messages.append(last_argument) if len(messages) > 1: @@ -173,13 +179,16 @@ class javascript(Extractor): # Comments don't apply unless they immediately precede the # message - if translator_comments and \ - translator_comments[-1][0] < message_lineno - 1: + if translator_comments and translator_comments[-1][0] < message_lineno - 1: translator_comments = [] if messages is not None: - yield (message_lineno, funcname, messages, - [comment[1] for comment in translator_comments]) + yield ( + message_lineno, + funcname, + messages, + [comment[1] for comment in translator_comments], + ) funcname = message_lineno = last_argument = None concatenate_next = False @@ -187,47 +196,54 @@ class javascript(Extractor): messages = [] call_stack = -1 - elif token.type == 'string': + elif token.type == "string": new_value = unquote_string(token.value) if concatenate_next: - last_argument = (last_argument or '') + new_value + last_argument = (last_argument or "") + new_value concatenate_next = False else: last_argument = new_value - elif token.type == 'operator': - if token.value == ',': + elif token.type == "operator": + if token.value == ",": if last_argument is not None: messages.append(last_argument) last_argument = None else: messages.append(None) concatenate_next = False - elif token.value == '+': + elif token.value == "+": concatenate_next = True - elif call_stack > 0 and token.type == 'operator' and \ - (token.value == ')' or token.value == ']' or token.value == '}'): + elif ( + call_stack > 0 + and token.type == "operator" + and (token.value == ")" or token.value == "]" or token.value == "}") + ): call_stack -= 1 elif funcname and call_stack == -1: funcname = None - elif call_stack == -1 and token.type == 'name' and \ - token.value in keywords and \ - (last_token is None or last_token.type != 'name' or - last_token.value != 'function'): + elif ( + call_stack == -1 + and token.type == "name" + and token.value in keywords + and ( + last_token is None + or last_token.type != "name" + or last_token.value != "function" + ) + ): funcname = token.value last_token = token - def extractFromFile(self, filepath): - - with codecs.open(filepath, 'r', encoding='utf-8-sig') as fileObject: + with codecs.open(filepath, "r", encoding="utf-8-sig") as fileObject: for lineno, funcname, messages, comments in self.extractJavascriptFromFile(fileObject): if funcname: - spec = self.options.get('keywords', {})[funcname] or (1,) + spec = self.options.get("keywords", {})[funcname] or (1,) else: spec = (1,) if not isinstance(messages, (list, tuple)): @@ -265,8 +281,10 @@ class javascript(Extractor): first_msg_index = spec[0] - 1 if not messages[first_msg_index]: # An empty string msgid isn't valid, emit a warning - where = '%s:%i' % (hasattr(fileObject, 'name') and \ - fileObject.name or '(unknown)', lineno) + where = "%s:%i" % ( + hasattr(fileObject, "name") and fileObject.name or "(unknown)", + lineno, + ) print(self.empty_msgid_warning % where, file=sys.stderr) continue @@ -279,20 +297,17 @@ class javascript(Extractor): yield message, plural, context, lineno, comments - class cpp(javascript): - """ Extract messages from C++ source code. - """ + """Extract messages from C++ source code.""" + pass - class txt(Extractor): - """ Extract messages from plain text files. - """ + """Extract messages from plain text files.""" def extractFromFile(self, filepath): - with codecs.open(filepath, "r", encoding='utf-8-sig') as fileObject: + with codecs.open(filepath, "r", encoding="utf-8-sig") as fileObject: lineno = 0 for line in [line.strip("\n\r") for line in fileObject.readlines()]: lineno += 1 @@ -300,10 +315,8 @@ class txt(Extractor): yield line, None, None, lineno, [] - class json(Extractor): - """ Extract messages from JSON files. - """ + """Extract messages from JSON files.""" def __init__(self, directoryPath=None, filemasks=[], options={}): super(json, self).__init__(directoryPath, filemasks, options) @@ -318,7 +331,7 @@ class json(Extractor): self.comments = self.options.get("comments", []) def extractFromFile(self, filepath): - with codecs.open(filepath, "r", 'utf-8') as fileObject: + with codecs.open(filepath, "r", "utf-8") as fileObject: for message, context in self.extractFromString(fileObject.read()): yield message, None, context, None, self.comments @@ -326,14 +339,16 @@ class json(Extractor): jsonDocument = jsonParser.loads(string) if isinstance(jsonDocument, list): for message, context in self.parseList(jsonDocument): - if message: # Skip empty strings. + if message: # Skip empty strings. yield message, context elif isinstance(jsonDocument, dict): for message, context in self.parseDictionary(jsonDocument): - if message: # Skip empty strings. + if message: # Skip empty strings. yield message, context else: - raise Exception("Unexpected JSON document parent structure (not a list or a dictionary). You must extend the JSON extractor to support it.") + raise Exception( + "Unexpected JSON document parent structure (not a list or a dictionary). You must extend the JSON extractor to support it." + ) def parseList(self, itemsList): index = 0 @@ -356,8 +371,13 @@ class json(Extractor): yield message, context elif isinstance(dictionary[keyword], dict): extract = None - if "extractFromInnerKeys" in self.keywords[keyword] and self.keywords[keyword]["extractFromInnerKeys"]: - for message, context in self.extractDictionaryInnerKeys(dictionary[keyword], keyword): + if ( + "extractFromInnerKeys" in self.keywords[keyword] + and self.keywords[keyword]["extractFromInnerKeys"] + ): + for message, context in self.extractDictionaryInnerKeys( + dictionary[keyword], keyword + ): yield message, context else: extract = self.extractDictionary(dictionary[keyword], keyword) @@ -386,7 +406,7 @@ class json(Extractor): if isinstance(listItem, str): yield self.extractString(listItem, keyword) elif isinstance(listItem, dict): - extract = self.extractDictionary(dictionary[keyword], keyword) + extract = self.extractDictionary(listItem[keyword], keyword) if extract: yield extract index += 1 @@ -420,8 +440,7 @@ class json(Extractor): class xml(Extractor): - """ Extract messages from XML files. - """ + """Extract messages from XML files.""" def __init__(self, directoryPath, filemasks, options): super(xml, self).__init__(directoryPath, filemasks, options) @@ -435,7 +454,8 @@ class xml(Extractor): def extractFromFile(self, filepath): from lxml import etree - with codecs.open(filepath, "r", encoding='utf-8-sig') as fileObject: + + with codecs.open(filepath, "r", encoding="utf-8-sig") as fileObject: xmlDocument = etree.parse(fileObject) for keyword in self.keywords: for element in xmlDocument.iter(keyword): @@ -457,7 +477,9 @@ class xml(Extractor): context = self.keywords[keyword]["customContext"] if "comment" in element.attrib: comment = element.get("comment") - comment = u" ".join(comment.split()) # Remove tabs, line breaks and unecessary spaces. + comment = " ".join( + comment.split() + ) # Remove tabs, line breaks and unecessary spaces. comments.append(comment) if "splitOnWhitespace" in self.keywords[keyword]: for splitText in element.text.split(): @@ -470,21 +492,22 @@ class xml(Extractor): # Hack from http://stackoverflow.com/a/2819788 class FakeSectionHeader(object): - def __init__(self, fp): self.fp = fp - self.sechead = '[root]\n' + self.sechead = "[root]\n" def readline(self): if self.sechead: - try: return self.sechead - finally: self.sechead = None - else: return self.fp.readline() + try: + return self.sechead + finally: + self.sechead = None + else: + return self.fp.readline() class ini(Extractor): - """ Extract messages from INI files. - """ + """Extract messages from INI files.""" def __init__(self, directoryPath, filemasks, options): super(ini, self).__init__(directoryPath, filemasks, options) @@ -492,6 +515,7 @@ class ini(Extractor): def extractFromFile(self, filepath): import ConfigParser + config = ConfigParser.RawConfigParser() config.readfp(FakeSectionHeader(open(filepath))) for keyword in self.keywords: diff --git a/source/tools/i18n/generateDebugTranslation.py b/source/tools/i18n/generateDebugTranslation.py index 61eb6b8317..dc642209cb 100644 --- a/source/tools/i18n/generateDebugTranslation.py +++ b/source/tools/i18n/generateDebugTranslation.py @@ -26,16 +26,16 @@ from i18n_helper.catalog import Catalog from i18n_helper.globber import getCatalogs -DEBUG_PREFIX = 'X_X ' +DEBUG_PREFIX = "X_X " def generate_long_strings(root_path, input_file_name, output_file_name, languages=None): """ - Generate the 'long strings' debug catalog. - This catalog contains the longest singular and plural string, - found amongst all translated languages or a filtered subset. - It can be used to check if GUI elements are large enough. - The catalog is long.*.po + Generate the 'long strings' debug catalog. + This catalog contains the longest singular and plural string, + found amongst all translated languages or a filtered subset. + It can be used to check if GUI elements are large enough. + The catalog is long.*.po """ print("Generating", output_file_name) input_file_path = os.path.join(root_path, input_file_name) @@ -48,8 +48,11 @@ def generate_long_strings(root_path, input_file_name, output_file_name, language # Fill catalog with English strings. for message in template_catalog: long_string_catalog.add( - id=message.id, string=message.id, context=message.context, - auto_comments=message.auto_comments) + id=message.id, + string=message.id, + context=message.context, + auto_comments=message.auto_comments, + ) # Load existing translation catalogs. existing_translation_catalogs = getCatalogs(input_file_path, languages) @@ -58,18 +61,23 @@ def generate_long_strings(root_path, input_file_name, output_file_name, language for translation_catalog in existing_translation_catalogs: for long_string_catalog_message in long_string_catalog: translation_message = translation_catalog.get( - long_string_catalog_message.id, long_string_catalog_message.context) + long_string_catalog_message.id, long_string_catalog_message.context + ) if not translation_message or not translation_message.string: continue - if not long_string_catalog_message.pluralizable or not translation_message.pluralizable: + if ( + not long_string_catalog_message.pluralizable + or not translation_message.pluralizable + ): if len(translation_message.string) > len(long_string_catalog_message.string): long_string_catalog_message.string = translation_message.string continue longest_singular_string = translation_message.string[0] - longest_plural_string = translation_message.string[1 if len( - translation_message.string) > 1 else 0] + longest_plural_string = translation_message.string[ + 1 if len(translation_message.string) > 1 else 0 + ] candidate_singular_string = long_string_catalog_message.string[0] # There might be between 0 and infinite plural forms. @@ -88,17 +96,19 @@ def generate_long_strings(root_path, input_file_name, output_file_name, language if changed: long_string_catalog_message.string = [ - longest_singular_string, longest_plural_string] + longest_singular_string, + longest_plural_string, + ] translation_message = long_string_catalog_message long_string_catalog.writeTo(output_file_path) def generate_debug(root_path, input_file_name, output_file_name): """ - Generate a debug catalog to identify untranslated strings. - This prefixes all strings with DEBUG_PREFIX, to easily identify - untranslated strings while still making the game navigable. - The catalog is debug.*.po + Generate a debug catalog to identify untranslated strings. + This prefixes all strings with DEBUG_PREFIX, to easily identify + untranslated strings while still making the game navigable. + The catalog is debug.*.po """ print("Generating", output_file_name) input_file_path = os.path.join(root_path, input_file_name) @@ -114,28 +124,34 @@ def generate_debug(root_path, input_file_name, output_file_name): id=message.id, string=(DEBUG_PREFIX + message.id[0],), context=message.context, - auto_comments=message.auto_comments) + auto_comments=message.auto_comments, + ) else: out_catalog.add( id=message.id, string=DEBUG_PREFIX + message.id, context=message.context, - auto_comments=message.auto_comments) + auto_comments=message.auto_comments, + ) out_catalog.writeTo(output_file_path) def main(): parser = argparse.ArgumentParser() - parser.add_argument("--debug", - help="Generate debug localisation to identify non-translated strings.", - action="store_true") - parser.add_argument("--long", - help="Generate 'long strings' localisation to identify GUI elements too small.", - action="store_true") - parser.add_argument("--languages", - nargs="+", - help="For long strings, restrict to these languages") + parser.add_argument( + "--debug", + help="Generate debug localisation to identify non-translated strings.", + action="store_true", + ) + parser.add_argument( + "--long", + help="Generate 'long strings' localisation to identify GUI elements too small.", + action="store_true", + ) + parser.add_argument( + "--languages", nargs="+", help="For long strings, restrict to these languages" + ) args = parser.parse_args() if not args.debug and not args.long: @@ -145,24 +161,28 @@ def main(): found_pot_files = 0 for root, _, filenames in os.walk(projectRootDirectory): for filename in filenames: - if len(filename) > 4 and filename[-4:] == ".pot" and os.path.basename(root) == l10nFolderName: + if ( + len(filename) > 4 + and filename[-4:] == ".pot" + and os.path.basename(root) == l10nFolderName + ): found_pot_files += 1 if args.debug: multiprocessing.Process( - target=generate_debug, - args=(root, filename, "debug." + filename[:-1]) + target=generate_debug, args=(root, filename, "debug." + filename[:-1]) ).start() if args.long: multiprocessing.Process( target=generate_long_strings, - args=(root, filename, "long." + - filename[:-1], args.languages) + args=(root, filename, "long." + filename[:-1], args.languages), ).start() if found_pot_files == 0: - print("This script did not work because no ‘.pot’ files were found. " - "Please, run ‘updateTemplates.py’ to generate the ‘.pot’ files, and run ‘pullTranslations.py’ to pull the latest translations from Transifex. " - "Then you can run this script to generate ‘.po’ files with obvious debug strings.") + print( + "This script did not work because no ‘.pot’ files were found. " + "Please, run ‘updateTemplates.py’ to generate the ‘.pot’ files, and run ‘pullTranslations.py’ to pull the latest translations from Transifex. " + "Then you can run this script to generate ‘.po’ files with obvious debug strings." + ) if __name__ == "__main__": diff --git a/source/tools/i18n/i18n_helper/__init__.py b/source/tools/i18n/i18n_helper/__init__.py index 6280442d49..66d3f60c92 100644 --- a/source/tools/i18n/i18n_helper/__init__.py +++ b/source/tools/i18n/i18n_helper/__init__.py @@ -3,4 +3,6 @@ import os l10nFolderName = "l10n" transifexClientFolder = ".tx" l10nToolsDirectory = os.path.dirname(os.path.realpath(__file__)) -projectRootDirectory = os.path.abspath(os.path.join(l10nToolsDirectory, os.pardir, os.pardir, os.pardir, os.pardir)) +projectRootDirectory = os.path.abspath( + os.path.join(l10nToolsDirectory, os.pardir, os.pardir, os.pardir, os.pardir) +) diff --git a/source/tools/i18n/i18n_helper/catalog.py b/source/tools/i18n/i18n_helper/catalog.py index f963c8dc21..6410685fd7 100644 --- a/source/tools/i18n/i18n_helper/catalog.py +++ b/source/tools/i18n/i18n_helper/catalog.py @@ -1,14 +1,19 @@ """Wrapper around babel Catalog / .po handling""" + from datetime import datetime from babel.messages.catalog import Catalog as BabelCatalog from babel.messages.pofile import read_po, write_po + class Catalog(BabelCatalog): """Wraps a BabelCatalog for convenience.""" + def __init__(self, *args, project=None, copyright_holder=None, **other_kwargs): date = datetime.now() - super().__init__(*args, header_comment=( + super().__init__( + *args, + header_comment=( f"# Translation template for {project}.\n" f"# Copyright (C) {date.year} {copyright_holder}\n" f"# This file is distributed under the same license as the {project} project." @@ -18,7 +23,8 @@ class Catalog(BabelCatalog): charset="utf-8", creation_date=date, revision_date=date, - **other_kwargs) + **other_kwargs, + ) self._project = project @BabelCatalog.mime_headers.getter @@ -31,14 +37,15 @@ class Catalog(BabelCatalog): "MIME-Version", "Content-Type", "Content-Transfer-Encoding", - "Plural-Forms"}: + "Plural-Forms", + }: headers.append((name, value)) - return [('Project-Id-Version', self._project)] + headers + return [("Project-Id-Version", self._project)] + headers @staticmethod - def readFrom(file_path, locale = None): - return read_po(open(file_path, "r+",encoding="utf-8"), locale=locale) + def readFrom(file_path, locale=None): + return read_po(open(file_path, "r+", encoding="utf-8"), locale=locale) def writeTo(self, file_path): return write_po( diff --git a/source/tools/i18n/i18n_helper/globber.py b/source/tools/i18n/i18n_helper/globber.py index ab311b70ca..2108ee4b5b 100644 --- a/source/tools/i18n/i18n_helper/globber.py +++ b/source/tools/i18n/i18n_helper/globber.py @@ -1,10 +1,12 @@ """Utils to list .po""" + import os from typing import List from i18n_helper.catalog import Catalog -def getCatalogs(inputFilePath, filters : List[str] = None) -> List[Catalog]: + +def getCatalogs(inputFilePath, filters: List[str] = None) -> List[Catalog]: """Returns a list of "real" catalogs (.po) in the given folder.""" existingTranslationCatalogs = [] l10nFolderPath = os.path.dirname(inputFilePath) @@ -17,6 +19,9 @@ def getCatalogs(inputFilePath, filters : List[str] = None) -> List[Catalog]: continue if not filters or filename.split(".")[0] in filters: existingTranslationCatalogs.append( - Catalog.readFrom(os.path.join(l10nFolderPath, filename), locale=filename.split('.')[0])) + Catalog.readFrom( + os.path.join(l10nFolderPath, filename), locale=filename.split(".")[0] + ) + ) return existingTranslationCatalogs diff --git a/source/tools/i18n/pullTranslations.py b/source/tools/i18n/pullTranslations.py index 1fe80fec97..7a85711c8f 100644 --- a/source/tools/i18n/pullTranslations.py +++ b/source/tools/i18n/pullTranslations.py @@ -21,6 +21,7 @@ import subprocess from i18n_helper import l10nFolderName, transifexClientFolder, projectRootDirectory + def main(): for root, folders, _ in os.walk(projectRootDirectory): for folder in folders: diff --git a/source/tools/i18n/tests/test_checkDiff.py b/source/tools/i18n/tests/test_checkDiff.py index d2901a4d52..217bce45cc 100644 --- a/source/tools/i18n/tests/test_checkDiff.py +++ b/source/tools/i18n/tests/test_checkDiff.py @@ -1,11 +1,9 @@ import io import pytest from checkDiff import check_diff -from unittest import mock -from types import SimpleNamespace PATCHES = [ -""" + """ Index: binaries/data/l10n/en_GB.engine.po =================================================================== --- binaries/data/l10n/en_GB.engine.po @@ -21,7 +19,7 @@ Index: binaries/data/l10n/en_GB.engine.po msgid "The incoming stream version is unsupported" """, -""" + """ Index: binaries/data/l10n/en_GB.engine.po =================================================================== --- binaries/data/l10n/en_GB.engine.po @@ -33,7 +31,7 @@ Index: binaries/data/l10n/en_GB.engine.po msgid "Stream error" msgstr "Stream error" """, -""" + """ Index: binaries/data/l10n/en_GB.engine.po =================================================================== --- binaries/data/l10n/en_GB.engine.po @@ -65,7 +63,7 @@ Index: binaries/data/l10n/en_GB_3.engine.po msgid "Stream error" msgstr "Stream error" """, -""" + """ Index: binaries/data/l10n/bar.engine.po =================================================================== --- binaries/data/l10n/bar.engine.po @@ -86,16 +84,17 @@ Index: binaries/data/l10n/bar.engine.po "Language-Team: Bavarian (http://www.transifex.com/wildfire-games/0ad/language/bar/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" -""" +""", ] PATCHES_EXPECT_REVERT = [ set(), {"binaries/data/l10n/en_GB.engine.po"}, {"binaries/data/l10n/en_GB.engine.po", "binaries/data/l10n/en_GB_3.engine.po"}, - {"binaries/data/l10n/bar.engine.po"} + {"binaries/data/l10n/bar.engine.po"}, ] + @pytest.fixture(params=zip(PATCHES, PATCHES_EXPECT_REVERT)) def patch(request): return [io.StringIO(request.param[0]), request.param[1]] diff --git a/source/tools/i18n/updateTemplates.py b/source/tools/i18n/updateTemplates.py index cd09e084e2..83e8a33d43 100644 --- a/source/tools/i18n/updateTemplates.py +++ b/source/tools/i18n/updateTemplates.py @@ -16,42 +16,56 @@ # You should have received a copy of the GNU General Public License # along with 0 A.D. If not, see . -import json, os +import json +import os import multiprocessing from importlib import import_module -from lxml import etree from i18n_helper import l10nFolderName, projectRootDirectory from i18n_helper.catalog import Catalog -from extractors import extractors + messagesFilename = "messages.json" def warnAboutUntouchedMods(): """ - Warn about mods that are not properly configured to get their messages extracted. + Warn about mods that are not properly configured to get their messages extracted. """ modsRootFolder = os.path.join(projectRootDirectory, "binaries", "data", "mods") untouchedMods = {} for modFolder in os.listdir(modsRootFolder): - if modFolder[0] != "_" and modFolder[0] != '.': + if modFolder[0] != "_" and modFolder[0] != ".": if not os.path.exists(os.path.join(modsRootFolder, modFolder, l10nFolderName)): - untouchedMods[modFolder] = "There is no '{folderName}' folder in the root folder of this mod.".format(folderName=l10nFolderName) - elif not os.path.exists(os.path.join(modsRootFolder, modFolder, l10nFolderName, messagesFilename)): - untouchedMods[modFolder] = "There is no '{filename}' file within the '{folderName}' folder in the root folder of this mod.".format(folderName=l10nFolderName, filename=messagesFilename) + untouchedMods[modFolder] = ( + "There is no '{folderName}' folder in the root folder of this mod.".format( + folderName=l10nFolderName + ) + ) + elif not os.path.exists( + os.path.join(modsRootFolder, modFolder, l10nFolderName, messagesFilename) + ): + untouchedMods[modFolder] = ( + "There is no '{filename}' file within the '{folderName}' folder in the root folder of this mod.".format( + folderName=l10nFolderName, filename=messagesFilename + ) + ) if untouchedMods: - print("" - "Warning: No messages were extracted from the following mods:" - "") + print("" "Warning: No messages were extracted from the following mods:" "") for mod in untouchedMods: - print("• {modName}: {warningMessage}".format(modName=mod, warningMessage=untouchedMods[mod])) - print("" + print( + "• {modName}: {warningMessage}".format( + modName=mod, warningMessage=untouchedMods[mod] + ) + ) + print( + "" f"For this script to extract messages from a mod folder, this mod folder must contain a '{l10nFolderName}' " f"folder, and this folder must contain a '{messagesFilename}' file that describes how to extract messages for the " f"mod. See the folder of the main mod ('public') for an example, and see the documentation for more " f"information." - ) + ) + def generatePOT(templateSettings, rootPath): if "skip" in templateSettings and templateSettings["skip"] == "yes": @@ -64,7 +78,7 @@ def generatePOT(templateSettings, rootPath): template = Catalog( project=templateSettings["project"], copyright_holder=templateSettings["copyrightHolder"], - locale='en', + locale="en", ) for rule in templateSettings["rules"]: @@ -72,7 +86,7 @@ def generatePOT(templateSettings, rootPath): return options = rule.get("options", {}) - extractorClass = getattr(import_module("extractors.extractors"), rule['extractor']) + extractorClass = getattr(import_module("extractors.extractors"), rule["extractor"]) extractor = extractorClass(inputRootPath, rule["filemasks"], options) formatFlag = None if "format" in options: @@ -84,31 +98,34 @@ def generatePOT(templateSettings, rootPath): id=message_id, context=context, auto_comments=comments, - flags=[formatFlag] if formatFlag and message.find("%") != -1 else [] + flags=[formatFlag] if formatFlag and message.find("%") != -1 else [], ) saved_message.locations.append(location) - saved_message.flags.discard('python-format') + saved_message.flags.discard("python-format") template.writeTo(os.path.join(rootPath, templateSettings["output"])) - print(u"Generated \"{}\" with {} messages.".format(templateSettings["output"], len(template))) + print('Generated "{}" with {} messages.'.format(templateSettings["output"], len(template))) + def generateTemplatesForMessagesFile(messagesFilePath): - - with open(messagesFilePath, 'r') as fileObject: + with open(messagesFilePath, "r") as fileObject: settings = json.load(fileObject) for templateSettings in settings: multiprocessing.Process( - target=generatePOT, - args=(templateSettings, os.path.dirname(messagesFilePath)) + target=generatePOT, args=(templateSettings, os.path.dirname(messagesFilePath)) ).start() def main(): import argparse + parser = argparse.ArgumentParser() - parser.add_argument("--scandir", help="Directory to start scanning for l10n folders in. " - "Type '.' for current working directory") + parser.add_argument( + "--scandir", + help="Directory to start scanning for l10n folders in. " + "Type '.' for current working directory", + ) args = parser.parse_args() for root, folders, filenames in os.walk(args.scandir or projectRootDirectory): for folder in folders: diff --git a/source/tools/mapcompatibility/a18_to_a19.py b/source/tools/mapcompatibility/a18_to_a19.py index 24fe1c22dc..5ea5a06df8 100755 --- a/source/tools/mapcompatibility/a18_to_a19.py +++ b/source/tools/mapcompatibility/a18_to_a19.py @@ -23,118 +23,148 @@ # THE SOFTWARE. import argparse -import io import os import struct import sys -parser = argparse.ArgumentParser(description="Convert maps compatible with 0 A.D. version Alpha XVIII (A18) to maps compatible with version Alpha XIX (A19), or the other way around.") +parser = argparse.ArgumentParser( + description="Convert maps compatible with 0 A.D. version Alpha XVIII (A18) to maps compatible with version Alpha XIX (A19), or the other way around." +) -parser.add_argument("--reverse", action="store_true", help="Make an A19 map compatible with A18 (note that conversion will fail if mountains are too high)") -parser.add_argument("--no-version-bump", action="store_true", help="Don't change the version number of the map") -parser.add_argument("--no-color-spelling", action="store_true", help="Don't change the spelling of color and colour") +parser.add_argument( + "--reverse", + action="store_true", + help="Make an A19 map compatible with A18 (note that conversion will fail if mountains are too high)", +) +parser.add_argument( + "--no-version-bump", action="store_true", help="Don't change the version number of the map" +) +parser.add_argument( + "--no-color-spelling", + action="store_true", + help="Don't change the spelling of color and colour", +) parser.add_argument("--no-height-change", action="store_true", help="Don't change the heightmap") -parser.add_argument("files", nargs="+", help="XML file to process (use wildcards '*' to select multiple files)") +parser.add_argument( + "files", nargs="+", help="XML file to process (use wildcards '*' to select multiple files)" +) args = parser.parse_args() HEIGHTMAP_BIT_SHIFT = 3 for xmlFile in args.files: - pmpFile = xmlFile[:-3] + "pmp" + pmpFile = xmlFile[:-3] + "pmp" - print("Processing " + xmlFile + " ...") + print("Processing " + xmlFile + " ...") - if os.path.isfile(pmpFile): - with open(pmpFile, "rb") as f1, open(pmpFile + "~", "wb") as f2: - # 4 bytes PSMP to start the file - f2.write(f1.read(4)) + if os.path.isfile(pmpFile): + with open(pmpFile, "rb") as f1, open(pmpFile + "~", "wb") as f2: + # 4 bytes PSMP to start the file + f2.write(f1.read(4)) - # 4 bytes to encode the version of the file format - version = struct.unpack(">' operator - if args.no_height_change: - def height_transform(h): - return h - else: - if args.reverse: - def height_transform(h): - return h << HEIGHTMAP_BIT_SHIFT - else: - def height_transform(h): - return h >> HEIGHTMAP_BIT_SHIFT - - for i in range(0, (map_size*16+1)*(map_size*16+1)): - height = struct.unpack(">' operator + if args.no_height_change: - f2.close() - f1.close() + def height_transform(h): + return h + else: + if args.reverse: - # replace the old file, comment to see both files - os.remove(pmpFile) - os.rename(pmpFile + "~", pmpFile) + def height_transform(h): + return h << HEIGHTMAP_BIT_SHIFT + else: + def height_transform(h): + return h >> HEIGHTMAP_BIT_SHIFT - if os.path.isfile(xmlFile): - with open(xmlFile, "r") as f1, open(xmlFile + "~", "w") as f2: - data = f1.read() + for i in range(0, (map_size * 16 + 1) * (map_size * 16 + 1)): + height = struct.unpack("') == -1: - print("Warning: File " + xmlFile + " was not at version 6, while a negative version bump was requested.\nABORTING ...") - sys.exit() - else: - data = data.replace('', '') - else: - if data.find('') == -1: - print("Warning: File " + xmlFile + " was not at version 5, while a version bump was requested.\nABORTING ...") - sys.exit() - else: - data = data.replace('', '') - + # copy the rest of the file + byte = f1.read(1) + while byte != b"": + f2.write(byte) + byte = f1.read(1) - # transform the color keys - if not args.no_color_spelling: - if args.reverse: - data = data.replace("color", "colour").replace("Color", "Colour") - else: - data = data.replace("colour", "color").replace("Colour", "Color") - - f2.write(data) - f1.close() - f2.close() + f2.close() + f1.close() - # replace the old file, comment to see both files - os.remove(xmlFile) - os.rename(xmlFile + "~", xmlFile) + # replace the old file, comment to see both files + os.remove(pmpFile) + os.rename(pmpFile + "~", pmpFile) + + if os.path.isfile(xmlFile): + with open(xmlFile, "r") as f1, open(xmlFile + "~", "w") as f2: + data = f1.read() + + # bump version number (rely on how Atlas formats the XML) + if not args.no_version_bump: + if args.reverse: + if data.find('') == -1: + print( + "Warning: File " + + xmlFile + + " was not at version 6, while a negative version bump was requested.\nABORTING ..." + ) + sys.exit() + else: + data = data.replace('', '') + else: + if data.find('') == -1: + print( + "Warning: File " + + xmlFile + + " was not at version 5, while a version bump was requested.\nABORTING ..." + ) + sys.exit() + else: + data = data.replace('', '') + + # transform the color keys + if not args.no_color_spelling: + if args.reverse: + data = data.replace("color", "colour").replace("Color", "Colour") + else: + data = data.replace("colour", "color").replace("Colour", "Color") + + f2.write(data) + f1.close() + f2.close() + + # replace the old file, comment to see both files + os.remove(xmlFile) + os.rename(xmlFile + "~", xmlFile) diff --git a/source/tools/rlclient/python/samples/simple-example.py b/source/tools/rlclient/python/samples/simple-example.py index 44b3fa6f41..76d20bfd79 100644 --- a/source/tools/rlclient/python/samples/simple-example.py +++ b/source/tools/rlclient/python/samples/simple-example.py @@ -4,31 +4,36 @@ import zero_ad # First, we will define some helper functions we will use later. import math -def dist (p1, p2): + + +def dist(p1, p2): return math.sqrt(sum((math.pow(x2 - x1, 2) for (x1, x2) in zip(p1, p2)))) + def center(units): sum_position = map(sum, zip(*map(lambda u: u.position(), units))) - return [x/len(units) for x in sum_position] + return [x / len(units) for x in sum_position] + def closest(units, position): dists = (dist(unit.position(), position) for unit in units) index = 0 min_dist = next(dists) - for (i, d) in enumerate(dists): + for i, d in enumerate(dists): if d < min_dist: index = i min_dist = d return units[index] + # Connect to a 0 AD game server listening at localhost:6000 -game = zero_ad.ZeroAD('http://localhost:6000') +game = zero_ad.ZeroAD("http://localhost:6000") # Load the Arcadia map samples_dir = path.dirname(path.realpath(__file__)) -scenario_config_path = path.join(samples_dir, 'arcadia.json') -with open(scenario_config_path, 'r') as f: +scenario_config_path = path.join(samples_dir, "arcadia.json") +with open(scenario_config_path, "r") as f: arcadia_config = f.read() state = game.reset(arcadia_config) @@ -37,15 +42,15 @@ state = game.reset(arcadia_config) state = game.step() # Units can be queried from the game state -citizen_soldiers = state.units(owner=1, type='infantry') +citizen_soldiers = state.units(owner=1, type="infantry") # (including gaia units like trees or other resources) -nearby_tree = closest(state.units(owner=0, type='tree'), center(citizen_soldiers)) +nearby_tree = closest(state.units(owner=0, type="tree"), center(citizen_soldiers)) # Action commands can be created using zero_ad.actions collect_wood = zero_ad.actions.gather(citizen_soldiers, nearby_tree) -female_citizens = state.units(owner=1, type='female_citizen') -house_tpl = 'structures/spart/house' +female_citizens = state.units(owner=1, type="female_citizen") +house_tpl = "structures/spart/house" x = 680 z = 640 build_house = zero_ad.actions.construct(female_citizens, house_tpl, x, z, autocontinue=True) @@ -58,20 +63,24 @@ female_id = female_citizens[0].id() female_citizen = state.unit(female_id) # A variety of unit information can be queried from the unit: -print('female citizen\'s max health is', female_citizen.max_health()) +print("female citizen's max health is", female_citizen.max_health()) # Raw data for units and game states are available via the data attribute print(female_citizen.data) # Units can be built using the "train action" civic_center = state.units(owner=1, type="civil_centre")[0] -spearman_type = 'units/spart/infantry_spearman_b' +spearman_type = "units/spart/infantry_spearman_b" train_spearmen = zero_ad.actions.train([civic_center], spearman_type) state = game.step([train_spearmen]) + # Let's step the engine until the house has been built -is_unit_busy = lambda state, unit_id: len(state.unit(unit_id).data['unitAIOrderData']) > 0 +def is_unit_busy(state, unit_id): + return len(state.unit(unit_id).data["unitAIOrderData"]) > 0 + + while is_unit_busy(state, female_id): state = game.step() @@ -85,14 +94,16 @@ for _ in range(150): state = game.step() # Let's attack with our entire military -state = game.step([zero_ad.actions.chat('An attack is coming!')]) +state = game.step([zero_ad.actions.chat("An attack is coming!")]) -while len(state.units(owner=2, type='unit')) > 0: - attack_units = [ unit for unit in state.units(owner=1, type='unit') if 'female' not in unit.type() ] - target = closest(state.units(owner=2, type='unit'), center(attack_units)) +while len(state.units(owner=2, type="unit")) > 0: + attack_units = [ + unit for unit in state.units(owner=1, type="unit") if "female" not in unit.type() + ] + target = closest(state.units(owner=2, type="unit"), center(attack_units)) state = game.step([zero_ad.actions.attack(attack_units, target)]) while state.unit(target.id()): state = game.step() -game.step([zero_ad.actions.chat('The enemies have been vanquished. Our home is safe again.')]) +game.step([zero_ad.actions.chat("The enemies have been vanquished. Our home is safe again.")]) diff --git a/source/tools/rlclient/python/setup.py b/source/tools/rlclient/python/setup.py index 72285e7844..43d4dd82ef 100644 --- a/source/tools/rlclient/python/setup.py +++ b/source/tools/rlclient/python/setup.py @@ -1,13 +1,14 @@ -import os from setuptools import setup -setup(name='zero_ad', - version='0.0.1', - description='Python client for 0 AD', - url='https://code.wildfiregames.com', - author='Brian Broll', - author_email='brian.broll@gmail.com', - install_requires=[], - license='MIT', - packages=['zero_ad'], - zip_safe=False) +setup( + name="zero_ad", + version="0.0.1", + description="Python client for 0 AD", + url="https://code.wildfiregames.com", + author="Brian Broll", + author_email="brian.broll@gmail.com", + install_requires=[], + license="MIT", + packages=["zero_ad"], + zip_safe=False, +) diff --git a/source/tools/rlclient/python/tests/test_actions.py b/source/tools/rlclient/python/tests/test_actions.py index 06b4890d24..c59e513438 100644 --- a/source/tools/rlclient/python/tests/test_actions.py +++ b/source/tools/rlclient/python/tests/test_actions.py @@ -1,35 +1,38 @@ import zero_ad -import json import math from os import path -game = zero_ad.ZeroAD('http://localhost:6000') +game = zero_ad.ZeroAD("http://localhost:6000") scriptdir = path.dirname(path.realpath(__file__)) -with open(path.join(scriptdir, '..', 'samples', 'arcadia.json'), 'r') as f: +with open(path.join(scriptdir, "..", "samples", "arcadia.json"), "r") as f: config = f.read() -def dist (p1, p2): + +def dist(p1, p2): return math.sqrt(sum((math.pow(x2 - x1, 2) for (x1, x2) in zip(p1, p2)))) + def center(units): sum_position = map(sum, zip(*map(lambda u: u.position(), units))) - return [x/len(units) for x in sum_position] + return [x / len(units) for x in sum_position] + def closest(units, position): dists = (dist(unit.position(), position) for unit in units) index = 0 min_dist = next(dists) - for (i, d) in enumerate(dists): + for i, d in enumerate(dists): if d < min_dist: index = i min_dist = d return units[index] + def test_construct(): state = game.reset(config) - female_citizens = state.units(owner=1, type='female_citizen') - house_tpl = 'structures/spart/house' + female_citizens = state.units(owner=1, type="female_citizen") + house_tpl = "structures/spart/house" house_count = len(state.units(owner=1, type=house_tpl)) x = 680 z = 640 @@ -39,21 +42,23 @@ def test_construct(): while len(state.units(owner=1, type=house_tpl)) == house_count: state = game.step() + def test_gather(): state = game.reset(config) - female_citizen = state.units(owner=1, type='female_citizen')[0] - trees = state.units(owner=0, type='tree') - nearby_tree = closest(state.units(owner=0, type='tree'), female_citizen.position()) + female_citizen = state.units(owner=1, type="female_citizen")[0] + state.units(owner=0, type="tree") + nearby_tree = closest(state.units(owner=0, type="tree"), female_citizen.position()) collect_wood = zero_ad.actions.gather([female_citizen], nearby_tree) state = game.step([collect_wood]) - while len(state.unit(female_citizen.id()).data['resourceCarrying']) == 0: + while len(state.unit(female_citizen.id()).data["resourceCarrying"]) == 0: state = game.step() + def test_train(): state = game.reset(config) civic_centers = state.units(owner=1, type="civil_centre") - spearman_type = 'units/spart/infantry_spearman_b' + spearman_type = "units/spart/infantry_spearman_b" spearman_count = len(state.units(owner=1, type=spearman_type)) train_spearmen = zero_ad.actions.train(civic_centers, spearman_type) @@ -61,9 +66,10 @@ def test_train(): while len(state.units(owner=1, type=spearman_type)) == spearman_count: state = game.step() + def test_walk(): state = game.reset(config) - female_citizens = state.units(owner=1, type='female_citizen') + female_citizens = state.units(owner=1, type="female_citizen") x = 680 z = 640 initial_distance = dist(center(female_citizens), [x, z]) @@ -73,13 +79,14 @@ def test_walk(): distance = initial_distance while distance >= initial_distance: state = game.step() - female_citizens = state.units(owner=1, type='female_citizen') + female_citizens = state.units(owner=1, type="female_citizen") distance = dist(center(female_citizens), [x, z]) + def test_attack(): state = game.reset(config) - unit = state.units(owner=1, type='cavalry')[0] - target = state.units(owner=2, type='female_citizen')[0] + unit = state.units(owner=1, type="cavalry")[0] + target = state.units(owner=2, type="female_citizen")[0] initial_health_target = target.health() initial_health_unit = unit.health() @@ -87,11 +94,13 @@ def test_attack(): attack = zero_ad.actions.attack([unit], target) state = game.step([attack]) - while (state.unit(target.id()).health() >= initial_health_target - ) and (state.unit(unit.id()).health() >= initial_health_unit): + while (state.unit(target.id()).health() >= initial_health_target) and ( + state.unit(unit.id()).health() >= initial_health_unit + ): state = game.step() + def test_chat(): - state = game.reset(config) - chat = zero_ad.actions.chat('hello world!!') - state = game.step([chat]) + game.reset(config) + chat = zero_ad.actions.chat("hello world!!") + game.step([chat]) diff --git a/source/tools/rlclient/python/tests/test_evaluate.py b/source/tools/rlclient/python/tests/test_evaluate.py index 3a88171839..508bb8c8c7 100644 --- a/source/tools/rlclient/python/tests/test_evaluate.py +++ b/source/tools/rlclient/python/tests/test_evaluate.py @@ -1,44 +1,48 @@ import zero_ad -import json -import math from os import path -game = zero_ad.ZeroAD('http://localhost:6000') +game = zero_ad.ZeroAD("http://localhost:6000") scriptdir = path.dirname(path.realpath(__file__)) -with open(path.join(scriptdir, '..', 'samples', 'arcadia.json'), 'r') as f: +with open(path.join(scriptdir, "..", "samples", "arcadia.json"), "r") as f: config = f.read() -with open(path.join(scriptdir, 'fastactions.js'), 'r') as f: +with open(path.join(scriptdir, "fastactions.js"), "r") as f: fastactions = f.read() + def test_return_object(): - state = game.reset(config) + game.reset(config) result = game.evaluate('({"hello": "world"})') assert type(result) is dict - assert result['hello'] == 'world' + assert result["hello"] == "world" + def test_return_null(): - result = game.evaluate('null') - assert result == None + result = game.evaluate("null") + assert result is None + def test_return_string(): - state = game.reset(config) + game.reset(config) result = game.evaluate('"cat"') - assert result == 'cat' + assert result == "cat" + def test_fastactions(): state = game.reset(config) game.evaluate(fastactions) - female_citizens = state.units(owner=1, type='female_citizen') - house_tpl = 'structures/spart/house' - house_count = len(state.units(owner=1, type=house_tpl)) + female_citizens = state.units(owner=1, type="female_citizen") + house_tpl = "structures/spart/house" + len(state.units(owner=1, type=house_tpl)) x = 680 z = 640 build_house = zero_ad.actions.construct(female_citizens, house_tpl, x, z, autocontinue=True) # Check that they start building the house state = game.step([build_house]) - step_count = 0 - new_house = lambda _=None: state.units(owner=1, type=house_tpl)[0] + + def new_house(_=None): + return state.units(owner=1, type=house_tpl)[0] + initial_health = new_house().health(ratio=True) while new_house().health(ratio=True) == initial_health: state = game.step() diff --git a/source/tools/rlclient/python/zero_ad/__init__.py b/source/tools/rlclient/python/zero_ad/__init__.py index 3c610ea3a7..868f89a842 100644 --- a/source/tools/rlclient/python/zero_ad/__init__.py +++ b/source/tools/rlclient/python/zero_ad/__init__.py @@ -1,4 +1,5 @@ -from . import actions +from . import actions # noqa: F401 from . import environment + ZeroAD = environment.ZeroAD GameState = environment.GameState diff --git a/source/tools/rlclient/python/zero_ad/actions.py b/source/tools/rlclient/python/zero_ad/actions.py index b8314150f5..2e7ddafe53 100644 --- a/source/tools/rlclient/python/zero_ad/actions.py +++ b/source/tools/rlclient/python/zero_ad/actions.py @@ -1,63 +1,57 @@ def construct(units, template, x, z, angle=0, autorepair=True, autocontinue=True, queued=False): - unit_ids = [ unit.id() for unit in units ] + unit_ids = [unit.id() for unit in units] return { - 'type': 'construct', - 'entities': unit_ids, - 'template': template, - 'x': x, - 'z': z, - 'angle': angle, - 'autorepair': autorepair, - 'autocontinue': autocontinue, - 'queued': queued, + "type": "construct", + "entities": unit_ids, + "template": template, + "x": x, + "z": z, + "angle": angle, + "autorepair": autorepair, + "autocontinue": autocontinue, + "queued": queued, } + def gather(units, target, queued=False): - unit_ids = [ unit.id() for unit in units ] + unit_ids = [unit.id() for unit in units] return { - 'type': 'gather', - 'entities': unit_ids, - 'target': target.id(), - 'queued': queued, + "type": "gather", + "entities": unit_ids, + "target": target.id(), + "queued": queued, } + def train(entities, unit_type, count=1): - entity_ids = [ unit.id() for unit in entities ] + entity_ids = [unit.id() for unit in entities] return { - 'type': 'train', - 'entities': entity_ids, - 'template': unit_type, - 'count': count, + "type": "train", + "entities": entity_ids, + "template": unit_type, + "count": count, } + def chat(message): - return { - 'type': 'aichat', - 'message': message - } + return {"type": "aichat", "message": message} + def reveal_map(): - return { - 'type': 'reveal-map', - 'enable': True - } + return {"type": "reveal-map", "enable": True} + def walk(units, x, z, queued=False): - ids = [ unit.id() for unit in units ] - return { - 'type': 'walk', - 'entities': ids, - 'x': x, - 'z': z, - 'queued': queued - } + ids = [unit.id() for unit in units] + return {"type": "walk", "entities": ids, "x": x, "z": z, "queued": queued} + def attack(units, target, queued=False, allow_capture=True): - unit_ids = [ unit.id() for unit in units ] + unit_ids = [unit.id() for unit in units] return { - 'type': 'attack', - 'entities': unit_ids, - 'target': target.id(), - 'allowCapture': allow_capture, - 'queued': queued + "type": "attack", + "entities": unit_ids, + "target": target.id(), + "allowCapture": allow_capture, + "queued": queued, } diff --git a/source/tools/rlclient/python/zero_ad/api.py b/source/tools/rlclient/python/zero_ad/api.py index 8dea9a6f20..88cc3a2904 100644 --- a/source/tools/rlclient/python/zero_ad/api.py +++ b/source/tools/rlclient/python/zero_ad/api.py @@ -1,33 +1,33 @@ -import urllib from urllib import request import json -class RLAPI(): + +class RLAPI: def __init__(self, url): self.url = url def post(self, route, data): - response = request.urlopen(url=f'{self.url}/{route}', data=bytes(data, 'utf8')) + response = request.urlopen(url=f"{self.url}/{route}", data=bytes(data, "utf8")) return response.read() def step(self, commands): - post_data = '\n'.join((f'{player};{json.dumps(action)}' for (player, action) in commands)) - return self.post('step', post_data) + post_data = "\n".join((f"{player};{json.dumps(action)}" for (player, action) in commands)) + return self.post("step", post_data) def reset(self, scenario_config, player_id, save_replay): - path = 'reset?' + path = "reset?" if save_replay: - path += 'saveReplay=1&' + path += "saveReplay=1&" if player_id: - path += f'playerID={player_id}&' + path += f"playerID={player_id}&" return self.post(path, scenario_config) def get_templates(self, names): - post_data = '\n'.join(names) - response = self.post('templates', post_data) - return zip(names, response.decode().split('\n')) + post_data = "\n".join(names) + response = self.post("templates", post_data) + return zip(names, response.decode().split("\n")) def evaluate(self, code): - response = self.post('evaluate', code) + response = self.post("evaluate", code) return json.loads(response.decode()) diff --git a/source/tools/rlclient/python/zero_ad/environment.py b/source/tools/rlclient/python/zero_ad/environment.py index 5b26aac62f..bb3e2fcbde 100644 --- a/source/tools/rlclient/python/zero_ad/environment.py +++ b/source/tools/rlclient/python/zero_ad/environment.py @@ -1,11 +1,11 @@ from .api import RLAPI import json -import math from xml.etree import ElementTree from itertools import cycle -class ZeroAD(): - def __init__(self, uri='http://localhost:6000'): + +class ZeroAD: + def __init__(self, uri="http://localhost:6000"): self.api = RLAPI(uri) self.current_state = None self.cache = {} @@ -20,7 +20,7 @@ class ZeroAD(): self.current_state = GameState(json.loads(state_json), self) return self.current_state - def reset(self, config='', save_replay=False, player_id=1): + def reset(self, config="", save_replay=False, player_id=1): state_json = self.api.reset(config, player_id, save_replay) self.current_state = GameState(json.loads(state_json), self) return self.current_state @@ -33,7 +33,7 @@ class ZeroAD(): def get_templates(self, names): templates = self.api.get_templates(names) - return [ (name, EntityTemplate(content)) for (name, content) in templates ] + return [(name, EntityTemplate(content)) for (name, content) in templates] def update_templates(self, types=[]): all_types = list(set([unit.type() for unit in self.current_state.units()])) @@ -41,54 +41,60 @@ class ZeroAD(): template_pairs = self.get_templates(all_types) self.cache = {} - for (name, tpl) in template_pairs: + for name, tpl in template_pairs: self.cache[name] = tpl return template_pairs -class GameState(): + +class GameState: def __init__(self, data, game): self.data = data self.game = game - self.mapSize = self.data['mapSize'] + self.mapSize = self.data["mapSize"] def units(self, owner=None, type=None): - filter_fn = lambda e: (owner is None or e['owner'] == owner) and \ - (type is None or type in e['template']) - return [ Entity(e, self.game) for e in self.data['entities'].values() if filter_fn(e) ] + def filter_fn(e): + return (owner is None or e["owner"] == owner) and ( + type is None or type in e["template"] + ) + + return [Entity(e, self.game) for e in self.data["entities"].values() if filter_fn(e)] def unit(self, id): id = str(id) - return Entity(self.data['entities'][id], self.game) if id in self.data['entities'] else None + return ( + Entity(self.data["entities"][id], self.game) if id in self.data["entities"] else None + ) -class Entity(): +class Entity: def __init__(self, data, game): self.data = data self.game = game self.template = self.game.cache.get(self.type(), None) def type(self): - return self.data['template'] + return self.data["template"] def id(self): - return self.data['id'] + return self.data["id"] def owner(self): - return self.data['owner'] + return self.data["owner"] def max_health(self): template = self.get_template() - return float(template.get('Health/Max')) + return float(template.get("Health/Max")) def health(self, ratio=False): if ratio: - return self.data['hitpoints']/self.max_health() + return self.data["hitpoints"] / self.max_health() - return self.data['hitpoints'] + return self.data["hitpoints"] def position(self): - return self.data['position'] + return self.data["position"] def get_template(self): if self.template is None: @@ -97,9 +103,10 @@ class Entity(): return self.template -class EntityTemplate(): + +class EntityTemplate: def __init__(self, xml): - self.data = ElementTree.fromstring(f'{xml}') + self.data = ElementTree.fromstring(f"{xml}") def get(self, path): node = self.data.find(path) @@ -113,4 +120,4 @@ class EntityTemplate(): return node is not None def __str__(self): - return ElementTree.tostring(self.data).decode('utf-8') + return ElementTree.tostring(self.data).decode("utf-8") diff --git a/source/tools/spirv/compile.py b/source/tools/spirv/compile.py index 49af63b44a..31d2078c1a 100644 --- a/source/tools/spirv/compile.py +++ b/source/tools/spirv/compile.py @@ -22,14 +22,12 @@ # THE SOFTWARE. import argparse -import datetime import hashlib import itertools import json import os import subprocess import sys -import time import yaml import xml.etree.ElementTree as ET @@ -40,29 +38,32 @@ def execute(command): process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() except: - sys.stderr.write('Failed to run command: {}\n'.format(' '.join(command))) + sys.stderr.write("Failed to run command: {}\n".format(" ".join(command))) raise return process.returncode, out, err + def calculate_hash(path): assert os.path.isfile(path) - with open(path, 'rb') as handle: + with open(path, "rb") as handle: return hashlib.sha1(handle.read()).hexdigest() + def compare_spirv(path1, path2): - with open(path1, 'rb') as handle: + with open(path1, "rb") as handle: spirv1 = handle.read() - with open(path2, 'rb') as handle: + with open(path2, "rb") as handle: spirv2 = handle.read() return spirv1 == spirv2 + def resolve_if(defines, expression): - for item in expression.strip().split('||'): + for item in expression.strip().split("||"): item = item.strip() assert len(item) > 1 name = item invert = False - if name[0] == '!': + if name[0] == "!": invert = True name = item[1:] assert item[1].isalpha() @@ -70,210 +71,267 @@ def resolve_if(defines, expression): assert item[0].isalpha() found_define = False for define in defines: - if define['name'] == name: - assert define['value'] == 'UNDEFINED' or define['value'] == '0' or define['value'] == '1' + if define["name"] == name: + assert ( + define["value"] == "UNDEFINED" + or define["value"] == "0" + or define["value"] == "1" + ) if invert: - if define['value'] != '1': + if define["value"] != "1": return True found_define = True else: - if define['value'] == '1': + if define["value"] == "1": return True if invert and not found_define: return True return False -def compile_and_reflect(input_mod_path, output_mod_path, dependencies, stage, path, out_path, defines): + +def compile_and_reflect( + input_mod_path, output_mod_path, dependencies, stage, path, out_path, defines +): keep_debug = False input_path = os.path.normpath(path) output_path = os.path.normpath(out_path) command = [ - 'glslc', '-x', 'glsl', '--target-env=vulkan1.1', '-std=450core', - '-I', os.path.join(input_mod_path, 'shaders', 'glsl'), + "glslc", + "-x", + "glsl", + "--target-env=vulkan1.1", + "-std=450core", + "-I", + os.path.join(input_mod_path, "shaders", "glsl"), ] for dependency in dependencies: if dependency != input_mod_path: - command += ['-I', os.path.join(dependency, 'shaders', 'glsl')] + command += ["-I", os.path.join(dependency, "shaders", "glsl")] command += [ - '-fshader-stage=' + stage, '-O', input_path, + "-fshader-stage=" + stage, + "-O", + input_path, ] use_descriptor_indexing = False for define in defines: - if define['value'] == 'UNDEFINED': + if define["value"] == "UNDEFINED": continue - assert ' ' not in define['value'] - command.append('-D{}={}'.format(define['name'], define['value'])) - if define['name'] == 'USE_DESCRIPTOR_INDEXING': + assert " " not in define["value"] + command.append("-D{}={}".format(define["name"], define["value"])) + if define["name"] == "USE_DESCRIPTOR_INDEXING": use_descriptor_indexing = True - command.append('-D{}={}'.format('USE_SPIRV', '1')) - command.append('-DSTAGE_{}={}'.format(stage.upper(), '1')) - command += ['-o', output_path] + command.append("-D{}={}".format("USE_SPIRV", "1")) + command.append("-DSTAGE_{}={}".format(stage.upper(), "1")) + command += ["-o", output_path] # Compile the shader with debug information to see names in reflection. - ret, out, err = execute(command + ['-g']) + ret, out, err = execute(command + ["-g"]) if ret: - sys.stderr.write('Command returned {}:\nCommand: {}\nInput path: {}\nOutput path: {}\nError: {}\n'.format( - ret, ' '.join(command), input_path, output_path, err)) - preprocessor_output_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'preprocessed_file.glsl')) - execute(command[:-2] + ['-g', '-E', '-o', preprocessor_output_path]) + sys.stderr.write( + "Command returned {}:\nCommand: {}\nInput path: {}\nOutput path: {}\nError: {}\n".format( + ret, " ".join(command), input_path, output_path, err + ) + ) + preprocessor_output_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "preprocessed_file.glsl") + ) + execute(command[:-2] + ["-g", "-E", "-o", preprocessor_output_path]) raise ValueError(err) - ret, out, err = execute(['spirv-reflect', '-y','-v', '1', output_path]) + ret, out, err = execute(["spirv-reflect", "-y", "-v", "1", output_path]) if ret: - sys.stderr.write('Command returned {}:\nCommand: {}\nInput path: {}\nOutput path: {}\nError: {}\n'.format( - ret, ' '.join(command), input_path, output_path, err)) + sys.stderr.write( + "Command returned {}:\nCommand: {}\nInput path: {}\nOutput path: {}\nError: {}\n".format( + ret, " ".join(command), input_path, output_path, err + ) + ) raise ValueError(err) # Reflect the result SPIRV. data = yaml.safe_load(out) - module = data['module'] + module = data["module"] interface_variables = [] - if 'all_interface_variables' in data and data['all_interface_variables']: - interface_variables = data['all_interface_variables'] + if "all_interface_variables" in data and data["all_interface_variables"]: + interface_variables = data["all_interface_variables"] push_constants = [] vertex_attributes = [] - if 'push_constants' in module and module['push_constants']: - assert len(module['push_constants']) == 1 + if "push_constants" in module and module["push_constants"]: + assert len(module["push_constants"]) == 1 + def add_push_constants(node, push_constants): - if ('members' in node) and node['members']: - for member in node['members']: + if ("members" in node) and node["members"]: + for member in node["members"]: add_push_constants(member, push_constants) else: - assert node['absolute_offset'] + node['size'] <= 128 - push_constants.append({ - 'name': node['name'], - 'offset': node['absolute_offset'], - 'size': node['size'], - }) - assert module['push_constants'][0]['type_description']['type_name'] == 'DrawUniforms' - assert module['push_constants'][0]['size'] <= 128 - add_push_constants(module['push_constants'][0], push_constants) + assert node["absolute_offset"] + node["size"] <= 128 + push_constants.append( + { + "name": node["name"], + "offset": node["absolute_offset"], + "size": node["size"], + } + ) + + assert module["push_constants"][0]["type_description"]["type_name"] == "DrawUniforms" + assert module["push_constants"][0]["size"] <= 128 + add_push_constants(module["push_constants"][0], push_constants) descriptor_sets = [] - if 'descriptor_sets' in module and module['descriptor_sets']: + if "descriptor_sets" in module and module["descriptor_sets"]: VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER = 1 VK_DESCRIPTOR_TYPE_STORAGE_IMAGE = 3 VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER = 6 VK_DESCRIPTOR_TYPE_STORAGE_BUFFER = 7 - for descriptor_set in module['descriptor_sets']: + for descriptor_set in module["descriptor_sets"]: UNIFORM_SET = 1 if use_descriptor_indexing else 0 STORAGE_SET = 2 bindings = [] - if descriptor_set['set'] == UNIFORM_SET: - assert descriptor_set['binding_count'] > 0 - for binding in descriptor_set['bindings']: - assert binding['set'] == UNIFORM_SET - block = binding['block'] + if descriptor_set["set"] == UNIFORM_SET: + assert descriptor_set["binding_count"] > 0 + for binding in descriptor_set["bindings"]: + assert binding["set"] == UNIFORM_SET + block = binding["block"] members = [] - for member in block['members']: - members.append({ - 'name': member['name'], - 'offset': member['absolute_offset'], - 'size': member['size'], - }) - bindings.append({ - 'binding': binding['binding'], - 'type': 'uniform', - 'size': block['size'], - 'members': members - }) - binding = descriptor_set['bindings'][0] - assert binding['descriptor_type'] == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER - elif descriptor_set['set'] == STORAGE_SET: - assert descriptor_set['binding_count'] > 0 - for binding in descriptor_set['bindings']: - is_storage_image = binding['descriptor_type'] == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE - is_storage_buffer = binding['descriptor_type'] == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER + for member in block["members"]: + members.append( + { + "name": member["name"], + "offset": member["absolute_offset"], + "size": member["size"], + } + ) + bindings.append( + { + "binding": binding["binding"], + "type": "uniform", + "size": block["size"], + "members": members, + } + ) + binding = descriptor_set["bindings"][0] + assert binding["descriptor_type"] == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER + elif descriptor_set["set"] == STORAGE_SET: + assert descriptor_set["binding_count"] > 0 + for binding in descriptor_set["bindings"]: + is_storage_image = ( + binding["descriptor_type"] == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE + ) + is_storage_buffer = ( + binding["descriptor_type"] == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER + ) assert is_storage_image or is_storage_buffer - assert binding['descriptor_type'] == descriptor_set['bindings'][0]['descriptor_type'] - assert binding['image']['arrayed'] == 0 - assert binding['image']['ms'] == 0 - bindingType = 'storageImage' + assert ( + binding["descriptor_type"] + == descriptor_set["bindings"][0]["descriptor_type"] + ) + assert binding["image"]["arrayed"] == 0 + assert binding["image"]["ms"] == 0 + bindingType = "storageImage" if is_storage_buffer: - bindingType = 'storageBuffer' - bindings.append({ - 'binding': binding['binding'], - 'type': bindingType, - 'name': binding['name'], - }) + bindingType = "storageBuffer" + bindings.append( + { + "binding": binding["binding"], + "type": bindingType, + "name": binding["name"], + } + ) else: if use_descriptor_indexing: - if descriptor_set['set'] == 0: - assert descriptor_set['binding_count'] >= 1 - for binding in descriptor_set['bindings']: - assert binding['descriptor_type'] == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER - assert binding['array']['dims'][0] == 16384 - if binding['binding'] == 0: - assert binding['name'] == 'textures2D' - elif binding['binding'] == 1: - assert binding['name'] == 'texturesCube' - elif binding['binding'] == 2: - assert binding['name'] == 'texturesShadow' + if descriptor_set["set"] == 0: + assert descriptor_set["binding_count"] >= 1 + for binding in descriptor_set["bindings"]: + assert ( + binding["descriptor_type"] + == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER + ) + assert binding["array"]["dims"][0] == 16384 + if binding["binding"] == 0: + assert binding["name"] == "textures2D" + elif binding["binding"] == 1: + assert binding["name"] == "texturesCube" + elif binding["binding"] == 2: + assert binding["name"] == "texturesShadow" else: assert False else: - assert descriptor_set['binding_count'] > 0 - for binding in descriptor_set['bindings']: - assert binding['descriptor_type'] == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER - assert binding['image']['sampled'] == 1 - assert binding['image']['arrayed'] == 0 - assert binding['image']['ms'] == 0 - sampler_type = 'sampler{}D'.format(binding['image']['dim'] + 1) - if binding['image']['dim'] == 3: - sampler_type = 'samplerCube' - bindings.append({ - 'binding': binding['binding'], - 'type': sampler_type, - 'name': binding['name'], - }) - descriptor_sets.append({ - 'set': descriptor_set['set'], - 'bindings': bindings, - }) - if stage == 'vertex': + assert descriptor_set["binding_count"] > 0 + for binding in descriptor_set["bindings"]: + assert ( + binding["descriptor_type"] == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER + ) + assert binding["image"]["sampled"] == 1 + assert binding["image"]["arrayed"] == 0 + assert binding["image"]["ms"] == 0 + sampler_type = "sampler{}D".format(binding["image"]["dim"] + 1) + if binding["image"]["dim"] == 3: + sampler_type = "samplerCube" + bindings.append( + { + "binding": binding["binding"], + "type": sampler_type, + "name": binding["name"], + } + ) + descriptor_sets.append( + { + "set": descriptor_set["set"], + "bindings": bindings, + } + ) + if stage == "vertex": for variable in interface_variables: - if variable['storage_class'] == 1: + if variable["storage_class"] == 1: # Input. - vertex_attributes.append({ - 'name': variable['name'], - 'location': variable['location'], - }) + vertex_attributes.append( + { + "name": variable["name"], + "location": variable["location"], + } + ) # Compile the final version without debug information. if not keep_debug: ret, out, err = execute(command) if ret: - sys.stderr.write('Command returned {}:\nCommand: {}\nInput path: {}\nOutput path: {}\nError: {}\n'.format( - ret, ' '.join(command), input_path, output_path, err)) + sys.stderr.write( + "Command returned {}:\nCommand: {}\nInput path: {}\nOutput path: {}\nError: {}\n".format( + ret, " ".join(command), input_path, output_path, err + ) + ) raise ValueError(err) return { - 'push_constants': push_constants, - 'vertex_attributes': vertex_attributes, - 'descriptor_sets': descriptor_sets, + "push_constants": push_constants, + "vertex_attributes": vertex_attributes, + "descriptor_sets": descriptor_sets, } def output_xml_tree(tree, path): - ''' We use a simple custom printer to have the same output for all platforms.''' - with open(path, 'wt') as handle: + """We use a simple custom printer to have the same output for all platforms.""" + with open(path, "wt") as handle: handle.write('\n') - handle.write('\n'.format(os.path.basename(__file__))) + handle.write( + "\n".format(os.path.basename(__file__)) + ) + def output_xml_node(node, handle, depth): - indent = '\t' * depth - attributes = '' + indent = "\t" * depth + attributes = "" for attribute_name in sorted(node.attrib.keys()): attributes += ' {}="{}"'.format(attribute_name, node.attrib[attribute_name]) if len(node) > 0: - handle.write('{}<{}{}>\n'.format(indent, node.tag, attributes)) + handle.write("{}<{}{}>\n".format(indent, node.tag, attributes)) for child in node: output_xml_node(child, handle, depth + 1) - handle.write('{}\n'.format(indent, node.tag)) + handle.write("{}\n".format(indent, node.tag)) else: - handle.write('{}<{}{}/>\n'.format(indent, node.tag, attributes)) + handle.write("{}<{}{}/>\n".format(indent, node.tag, attributes)) + output_xml_node(tree.getroot(), handle, 0) def build(rules, input_mod_path, output_mod_path, dependencies, program_name): sys.stdout.write('Program "{}"\n'.format(program_name)) if rules and program_name not in rules: - sys.stdout.write(' Skip.\n') + sys.stdout.write(" Skip.\n") return - sys.stdout.write(' Building.\n') + sys.stdout.write(" Building.\n") rebuild = False @@ -281,64 +339,76 @@ def build(rules, input_mod_path, output_mod_path, dependencies, program_name): program_defines = [] shaders = [] - tree = ET.parse(os.path.join(input_mod_path, 'shaders', 'glsl', program_name + '.xml')) + tree = ET.parse(os.path.join(input_mod_path, "shaders", "glsl", program_name + ".xml")) root = tree.getroot() for element in root: element_tag = element.tag - if element_tag == 'defines': + if element_tag == "defines": for child in element: values = [] for value in child: - values.append({ - 'name': child.attrib['name'], - 'value': value.text, - }) + values.append( + { + "name": child.attrib["name"], + "value": value.text, + } + ) defines.append(values) - elif element_tag == 'define': - program_defines.append({'name': element.attrib['name'], 'value': element.attrib['value']}) - elif element_tag == 'vertex': + elif element_tag == "define": + program_defines.append( + {"name": element.attrib["name"], "value": element.attrib["value"]} + ) + elif element_tag == "vertex": streams = [] for shader_child in element: - assert shader_child.tag == 'stream' - streams.append({ - 'name': shader_child.attrib['name'], - 'attribute': shader_child.attrib['attribute'], - }) - if 'if' in shader_child.attrib: - streams[-1]['if'] = shader_child.attrib['if'] - shaders.append({ - 'type': 'vertex', - 'file': element.attrib['file'], - 'streams': streams, - }) - elif element_tag == 'fragment': - shaders.append({ - 'type': 'fragment', - 'file': element.attrib['file'], - }) - elif element_tag == 'compute': - shaders.append({ - 'type': 'compute', - 'file': element.attrib['file'], - }) + assert shader_child.tag == "stream" + streams.append( + { + "name": shader_child.attrib["name"], + "attribute": shader_child.attrib["attribute"], + } + ) + if "if" in shader_child.attrib: + streams[-1]["if"] = shader_child.attrib["if"] + shaders.append( + { + "type": "vertex", + "file": element.attrib["file"], + "streams": streams, + } + ) + elif element_tag == "fragment": + shaders.append( + { + "type": "fragment", + "file": element.attrib["file"], + } + ) + elif element_tag == "compute": + shaders.append( + { + "type": "compute", + "file": element.attrib["file"], + } + ) else: raise ValueError('Unsupported element tag: "{}"'.format(element_tag)) stage_extension = { - 'vertex': '.vs', - 'fragment': '.fs', - 'geometry': '.gs', - 'compute': '.cs', + "vertex": ".vs", + "fragment": ".fs", + "geometry": ".gs", + "compute": ".cs", } - output_spirv_mod_path = os.path.join(output_mod_path, 'shaders', 'spirv') + output_spirv_mod_path = os.path.join(output_mod_path, "shaders", "spirv") if not os.path.isdir(output_spirv_mod_path): os.mkdir(output_spirv_mod_path) - root = ET.Element('programs') + root = ET.Element("programs") - if 'combinations' in rules[program_name]: - combinations = rules[program_name]['combinations'] + if "combinations" in rules[program_name]: + combinations = rules[program_name]["combinations"] else: combinations = list(itertools.product(*defines)) @@ -346,36 +416,36 @@ def build(rules, input_mod_path, output_mod_path, dependencies, program_name): for index, combination in enumerate(combinations): assert index < 10000 - program_path = 'spirv/' + program_name + ('_%04d' % index) + '.xml' + program_path = "spirv/" + program_name + ("_%04d" % index) + ".xml" - programs_element = ET.SubElement(root, 'program') - programs_element.set('type', 'spirv') - programs_element.set('file', program_path) + programs_element = ET.SubElement(root, "program") + programs_element.set("type", "spirv") + programs_element.set("file", program_path) - defines_element = ET.SubElement(programs_element, 'defines') + defines_element = ET.SubElement(programs_element, "defines") for define in combination: - if define['value'] == 'UNDEFINED': + if define["value"] == "UNDEFINED": continue - define_element = ET.SubElement(defines_element, 'define') - define_element.set('name', define['name']) - define_element.set('value', define['value']) + define_element = ET.SubElement(defines_element, "define") + define_element.set("name", define["name"]) + define_element.set("value", define["value"]) - if not rebuild and os.path.isfile(os.path.join(output_mod_path, 'shaders', program_path)): + if not rebuild and os.path.isfile(os.path.join(output_mod_path, "shaders", program_path)): continue - program_root = ET.Element('program') - program_root.set('type', 'spirv') + program_root = ET.Element("program") + program_root.set("type", "spirv") for shader in shaders: - extension = stage_extension[shader['type']] - file_name = program_name + ('_%04d' % index) + extension + '.spv' + extension = stage_extension[shader["type"]] + file_name = program_name + ("_%04d" % index) + extension + ".spv" output_spirv_path = os.path.join(output_spirv_mod_path, file_name) - input_glsl_path = os.path.join(input_mod_path, 'shaders', shader['file']) + input_glsl_path = os.path.join(input_mod_path, "shaders", shader["file"]) # Some shader programs might use vs and fs shaders from different mods. if not os.path.isfile(input_glsl_path): input_glsl_path = None for dependency in dependencies: - fallback_input_path = os.path.join(dependency, 'shaders', shader['file']) + fallback_input_path = os.path.join(dependency, "shaders", shader["file"]) if os.path.isfile(fallback_input_path): input_glsl_path = fallback_input_path break @@ -385,10 +455,11 @@ def build(rules, input_mod_path, output_mod_path, dependencies, program_name): input_mod_path, output_mod_path, dependencies, - shader['type'], + shader["type"], input_glsl_path, output_spirv_path, - combination + program_defines) + combination + program_defines, + ) spirv_hash = calculate_hash(output_spirv_path) if spirv_hash not in hashed_cache: @@ -406,77 +477,95 @@ def build(rules, input_mod_path, output_mod_path, dependencies, program_name): else: hashed_cache[spirv_hash].append(file_name) - shader_element = ET.SubElement(program_root, shader['type']) - shader_element.set('file', 'spirv/' + file_name) - if shader['type'] == 'vertex': - for stream in shader['streams']: - if 'if' in stream and not resolve_if(combination, stream['if']): + shader_element = ET.SubElement(program_root, shader["type"]) + shader_element.set("file", "spirv/" + file_name) + if shader["type"] == "vertex": + for stream in shader["streams"]: + if "if" in stream and not resolve_if(combination, stream["if"]): continue found_vertex_attribute = False - for vertex_attribute in reflection['vertex_attributes']: - if vertex_attribute['name'] == stream['attribute']: + for vertex_attribute in reflection["vertex_attributes"]: + if vertex_attribute["name"] == stream["attribute"]: found_vertex_attribute = True break - if not found_vertex_attribute and stream['attribute'] == 'a_tangent': + if not found_vertex_attribute and stream["attribute"] == "a_tangent": continue if not found_vertex_attribute: - sys.stderr.write('Vertex attribute not found: {}\n'.format(stream['attribute'])) + sys.stderr.write( + "Vertex attribute not found: {}\n".format(stream["attribute"]) + ) assert found_vertex_attribute - stream_element = ET.SubElement(shader_element, 'stream') - stream_element.set('name', stream['name']) - stream_element.set('attribute', stream['attribute']) - for vertex_attribute in reflection['vertex_attributes']: - if vertex_attribute['name'] == stream['attribute']: - stream_element.set('location', vertex_attribute['location']) + stream_element = ET.SubElement(shader_element, "stream") + stream_element.set("name", stream["name"]) + stream_element.set("attribute", stream["attribute"]) + for vertex_attribute in reflection["vertex_attributes"]: + if vertex_attribute["name"] == stream["attribute"]: + stream_element.set("location", vertex_attribute["location"]) break - for push_constant in reflection['push_constants']: - push_constant_element = ET.SubElement(shader_element, 'push_constant') - push_constant_element.set('name', push_constant['name']) - push_constant_element.set('size', push_constant['size']) - push_constant_element.set('offset', push_constant['offset']) - descriptor_sets_element = ET.SubElement(shader_element, 'descriptor_sets') - for descriptor_set in reflection['descriptor_sets']: - descriptor_set_element = ET.SubElement(descriptor_sets_element, 'descriptor_set') - descriptor_set_element.set('set', descriptor_set['set']) - for binding in descriptor_set['bindings']: - binding_element = ET.SubElement(descriptor_set_element, 'binding') - binding_element.set('type', binding['type']) - binding_element.set('binding', binding['binding']) - if binding['type'] == 'uniform': - binding_element.set('size', binding['size']) - for member in binding['members']: - member_element = ET.SubElement(binding_element, 'member') - member_element.set('name', member['name']) - member_element.set('size', member['size']) - member_element.set('offset', member['offset']) - elif binding['type'].startswith('sampler'): - binding_element.set('name', binding['name']) - elif binding['type'].startswith('storage'): - binding_element.set('name', binding['name']) + for push_constant in reflection["push_constants"]: + push_constant_element = ET.SubElement(shader_element, "push_constant") + push_constant_element.set("name", push_constant["name"]) + push_constant_element.set("size", push_constant["size"]) + push_constant_element.set("offset", push_constant["offset"]) + descriptor_sets_element = ET.SubElement(shader_element, "descriptor_sets") + for descriptor_set in reflection["descriptor_sets"]: + descriptor_set_element = ET.SubElement(descriptor_sets_element, "descriptor_set") + descriptor_set_element.set("set", descriptor_set["set"]) + for binding in descriptor_set["bindings"]: + binding_element = ET.SubElement(descriptor_set_element, "binding") + binding_element.set("type", binding["type"]) + binding_element.set("binding", binding["binding"]) + if binding["type"] == "uniform": + binding_element.set("size", binding["size"]) + for member in binding["members"]: + member_element = ET.SubElement(binding_element, "member") + member_element.set("name", member["name"]) + member_element.set("size", member["size"]) + member_element.set("offset", member["offset"]) + elif binding["type"].startswith("sampler"): + binding_element.set("name", binding["name"]) + elif binding["type"].startswith("storage"): + binding_element.set("name", binding["name"]) program_tree = ET.ElementTree(program_root) - output_xml_tree(program_tree, os.path.join(output_mod_path, 'shaders', program_path)) - + output_xml_tree(program_tree, os.path.join(output_mod_path, "shaders", program_path)) tree = ET.ElementTree(root) - output_xml_tree(tree, os.path.join(output_mod_path, 'shaders', 'spirv', program_name + '.xml')) + output_xml_tree(tree, os.path.join(output_mod_path, "shaders", "spirv", program_name + ".xml")) def run(): parser = argparse.ArgumentParser() - parser.add_argument('input_mod_path', help='a path to a directory with input mod with GLSL shaders like binaries/data/mods/public') - parser.add_argument('rules_path', help='a path to JSON with rules') - parser.add_argument('output_mod_path', help='a path to a directory with mod to store SPIR-V shaders like binaries/data/mods/spirv') - parser.add_argument('-d', '--dependency', action='append', help='a path to a directory with a dependency mod (at least modmod should present as dependency)', required=True) - parser.add_argument('-p', '--program_name', help='a shader program name (in case of presence the only program will be compiled)', default=None) + parser.add_argument( + "input_mod_path", + help="a path to a directory with input mod with GLSL shaders like binaries/data/mods/public", + ) + parser.add_argument("rules_path", help="a path to JSON with rules") + parser.add_argument( + "output_mod_path", + help="a path to a directory with mod to store SPIR-V shaders like binaries/data/mods/spirv", + ) + parser.add_argument( + "-d", + "--dependency", + action="append", + help="a path to a directory with a dependency mod (at least modmod should present as dependency)", + required=True, + ) + parser.add_argument( + "-p", + "--program_name", + help="a shader program name (in case of presence the only program will be compiled)", + default=None, + ) args = parser.parse_args() if not os.path.isfile(args.rules_path): sys.stderr.write('Rules "{}" are not found\n'.format(args.rules_path)) return - with open(args.rules_path, 'rt') as handle: + with open(args.rules_path, "rt") as handle: rules = json.load(handle) if not os.path.isdir(args.input_mod_path): @@ -487,7 +576,7 @@ def run(): sys.stderr.write('Output mod path "{}" is not a directory\n'.format(args.output_mod_path)) return - mod_shaders_path = os.path.join(args.input_mod_path, 'shaders', 'glsl') + mod_shaders_path = os.path.join(args.input_mod_path, "shaders", "glsl") if not os.path.isdir(mod_shaders_path): sys.stderr.write('Directory "{}" was not found\n'.format(mod_shaders_path)) return @@ -497,11 +586,11 @@ def run(): if not args.program_name: for file_name in os.listdir(mod_shaders_path): name, ext = os.path.splitext(file_name) - if ext.lower() == '.xml': + if ext.lower() == ".xml": build(rules, args.input_mod_path, args.output_mod_path, args.dependency, name) else: build(rules, args.input_mod_path, args.output_mod_path, args.dependency, args.program_name) -if __name__ == '__main__': - run() +if __name__ == "__main__": + run() diff --git a/source/tools/templatesanalyzer/unitTables.py b/source/tools/templatesanalyzer/unitTables.py index 6094327f3b..3606e36d3b 100644 --- a/source/tools/templatesanalyzer/unitTables.py +++ b/source/tools/templatesanalyzer/unitTables.py @@ -22,15 +22,16 @@ # THE SOFTWARE. import sys -sys.path -sys.path.append('../entity') -from scriptlib import SimulTemplateEntity import xml.etree.ElementTree as ET from pathlib import Path import os import glob +sys.path.append("../entity") +from scriptlib import SimulTemplateEntity # noqa: E402 + + AttackTypes = ["Hack", "Pierce", "Crush", "Poison", "Fire"] Resources = ["food", "wood", "stone", "metal"] @@ -93,13 +94,14 @@ AddSortingOverlay = True # This is the path to the /templates/ folder to consider. Change this for mod # support. -modsFolder = Path(__file__).resolve().parents[3] / 'binaries' / 'data' / 'mods' -basePath = modsFolder / 'public' / 'simulation' / 'templates' +modsFolder = Path(__file__).resolve().parents[3] / "binaries" / "data" / "mods" +basePath = modsFolder / "public" / "simulation" / "templates" # For performance purposes, cache opened templates files. globalTemplatesList = {} sim_entity = SimulTemplateEntity(modsFolder, None) + def htbout(file, balise, value): file.write("<" + balise + ">" + value + "\n") @@ -113,7 +115,9 @@ def fastParse(template_name): if template_name in globalTemplatesList: return globalTemplatesList[template_name] parent_string = ET.parse(template_name).getroot().get("parent") - globalTemplatesList[template_name] = sim_entity.load_inherited('simulation/templates/', str(template_name), ['public']) + globalTemplatesList[template_name] = sim_entity.load_inherited( + "simulation/templates/", str(template_name), ["public"] + ) globalTemplatesList[template_name].set("parent", parent_string) return globalTemplatesList[template_name] @@ -126,7 +130,9 @@ def getParents(template_name): parents = set() for parent in parents_string.split("|"): parents.add(parent) - for element in getParents(sim_entity.get_file('simulation/templates/', parent + ".xml", 'public')): + for element in getParents( + sim_entity.get_file("simulation/templates/", parent + ".xml", "public") + ): parents.add(element) return parents @@ -135,13 +141,14 @@ def getParents(template_name): def ExtractValue(value): return float(value.text) if value is not None else 0.0 + # This function checks that a template has the given parent. def hasParentTemplate(template_name, parentName): - return any(parentName == parent + '.xml' for parent in getParents(template_name)) + return any(parentName == parent + ".xml" for parent in getParents(template_name)) def CalcUnit(UnitName, existingUnit=None): - if existingUnit != None: + if existingUnit is not None: unit = existingUnit else: unit = { @@ -188,23 +195,23 @@ def CalcUnit(UnitName, existingUnit=None): for type in list(resource_cost): unit["Cost"][type.tag] = ExtractValue(type) - - if Template.find("./Attack/Melee") != None: + if Template.find("./Attack/Melee") is not None: unit["RepeatRate"]["Melee"] = ExtractValue(Template.find("./Attack/Melee/RepeatTime")) unit["PrepRate"]["Melee"] = ExtractValue(Template.find("./Attack/Melee/PrepareTime")) for atttype in AttackTypes: - unit["Attack"]["Melee"][atttype] = ExtractValue( Template.find("./Attack/Melee/Damage/" + atttype)) + unit["Attack"]["Melee"][atttype] = ExtractValue( + Template.find("./Attack/Melee/Damage/" + atttype) + ) attack_melee_bonus = Template.find("./Attack/Melee/Bonuses") if attack_melee_bonus is not None: for Bonus in attack_melee_bonus: Against = [] CivAg = [] - if Bonus.find("Classes") != None \ - and Bonus.find("Classes").text != None: + if Bonus.find("Classes") is not None and Bonus.find("Classes").text is not None: Against = Bonus.find("Classes").text.split(" ") - if Bonus.find("Civ") != None and Bonus.find("Civ").text != None: + if Bonus.find("Civ") is not None and Bonus.find("Civ").text is not None: CivAg = Bonus.find("Civ").text.split(" ") Val = float(Bonus.find("Multiplier").text) unit["AttackBonuses"][Bonus.tag] = { @@ -223,7 +230,7 @@ def CalcUnit(UnitName, existingUnit=None): unit["Restricted"].pop(newClasses.index(elem)) unit["Restricted"] += newClasses - elif Template.find("./Attack/Ranged") != None: + elif Template.find("./Attack/Ranged") is not None: unit["Ranged"] = True unit["Range"] = ExtractValue(Template.find("./Attack/Ranged/MaxRange")) unit["Spread"] = ExtractValue(Template.find("./Attack/Ranged/Projectile/Spread")) @@ -231,16 +238,17 @@ def CalcUnit(UnitName, existingUnit=None): unit["PrepRate"]["Ranged"] = ExtractValue(Template.find("./Attack/Ranged/PrepareTime")) for atttype in AttackTypes: - unit["Attack"]["Ranged"][atttype] = ExtractValue(Template.find("./Attack/Ranged/Damage/" + atttype) ) + unit["Attack"]["Ranged"][atttype] = ExtractValue( + Template.find("./Attack/Ranged/Damage/" + atttype) + ) - if Template.find("./Attack/Ranged/Bonuses") != None: + if Template.find("./Attack/Ranged/Bonuses") is not None: for Bonus in Template.find("./Attack/Ranged/Bonuses"): Against = [] CivAg = [] - if Bonus.find("Classes") != None \ - and Bonus.find("Classes").text != None: + if Bonus.find("Classes") is not None and Bonus.find("Classes").text is not None: Against = Bonus.find("Classes").text.split(" ") - if Bonus.find("Civ") != None and Bonus.find("Civ").text != None: + if Bonus.find("Civ") is not None and Bonus.find("Civ").text is not None: CivAg = Bonus.find("Civ").text.split(" ") Val = float(Bonus.find("Multiplier").text) unit["AttackBonuses"][Bonus.tag] = { @@ -248,9 +256,8 @@ def CalcUnit(UnitName, existingUnit=None): "Civs": CivAg, "Multiplier": Val, } - if Template.find("./Attack/Melee/RestrictedClasses") != None: - newClasses = Template.find("./Attack/Melee/RestrictedClasses")\ - .text.split(" ") + if Template.find("./Attack/Melee/RestrictedClasses") is not None: + newClasses = Template.find("./Attack/Melee/RestrictedClasses").text.split(" ") for elem in newClasses: if elem.find("-") != -1: newClasses.pop(newClasses.index(elem)) @@ -258,19 +265,17 @@ def CalcUnit(UnitName, existingUnit=None): unit["Restricted"].pop(newClasses.index(elem)) unit["Restricted"] += newClasses - if Template.find("Resistance") != None: + if Template.find("Resistance") is not None: for atttype in AttackTypes: - unit["Resistance"][atttype] = ExtractValue(Template.find( - "./Resistance/Entity/Damage/" + atttype - )) + unit["Resistance"][atttype] = ExtractValue( + Template.find("./Resistance/Entity/Damage/" + atttype) + ) - - - if Template.find("./UnitMotion") != None: - if Template.find("./UnitMotion/WalkSpeed") != None: + if Template.find("./UnitMotion") is not None: + if Template.find("./UnitMotion/WalkSpeed") is not None: unit["WalkSpeed"] = ExtractValue(Template.find("./UnitMotion/WalkSpeed")) - if Template.find("./Identity/VisibleClasses") != None: + if Template.find("./Identity/VisibleClasses") is not None: newClasses = Template.find("./Identity/VisibleClasses").text.split(" ") for elem in newClasses: if elem.find("-") != -1: @@ -279,7 +284,7 @@ def CalcUnit(UnitName, existingUnit=None): unit["Classes"].pop(newClasses.index(elem)) unit["Classes"] += newClasses - if Template.find("./Identity/Classes") != None: + if Template.find("./Identity/Classes") is not None: newClasses = Template.find("./Identity/Classes").text.split(" ") for elem in newClasses: if elem.find("-") != -1: @@ -308,28 +313,23 @@ def WriteUnit(Name, UnitDict): + "%" ) - attType = "Ranged" if UnitDict["Ranged"] == True else "Melee" + attType = "Ranged" if UnitDict["Ranged"] is True else "Melee" if UnitDict["RepeatRate"][attType] != "0": for atype in AttackTypes: repeatTime = float(UnitDict["RepeatRate"][attType]) / 1000.0 ret += ( "" - + str("%.1f" % ( - float(UnitDict["Attack"][attType][atype]) / repeatTime - )) + "" + + str("%.1f" % (float(UnitDict["Attack"][attType][atype]) / repeatTime)) + + "" ) - ret += ( - "" - + str("%.1f" % (float(UnitDict["RepeatRate"][attType]) / 1000.0)) - + "" - ) + ret += "" + str("%.1f" % (float(UnitDict["RepeatRate"][attType]) / 1000.0)) + "" else: for atype in AttackTypes: ret += " - " ret += " - " - if UnitDict["Ranged"] == True and UnitDict["Range"] > 0: + if UnitDict["Ranged"] is True and UnitDict["Range"] > 0: ret += "" + str("%.1f" % float(UnitDict["Range"])) + "" spread = float(UnitDict["Spread"]) ret += "" + str("%.1f" % spread) + "" @@ -337,11 +337,9 @@ def WriteUnit(Name, UnitDict): ret += " - - " for rtype in Resources: - ret += "" + str("%.0f" % - float(UnitDict["Cost"][rtype])) + "" + ret += "" + str("%.0f" % float(UnitDict["Cost"][rtype])) + "" - ret += "" + str("%.0f" % - float(UnitDict["Cost"]["population"])) + "" + ret += "" + str("%.0f" % float(UnitDict["Cost"]["population"])) + "" ret += '' for Bonus in UnitDict["AttackBonuses"]: @@ -362,11 +360,11 @@ def SortFn(A): sortVal += 1 if classe in A[1]["Classes"]: break - if ComparativeSortByChamp == True and A[0].find("champion") == -1: + if ComparativeSortByChamp is True and A[0].find("champion") == -1: sortVal -= 20 - if ComparativeSortByCav == True and A[0].find("cavalry") == -1: + if ComparativeSortByCav is True and A[0].find("cavalry") == -1: sortVal -= 10 - if A[1]["Civ"] != None and A[1]["Civ"] in Civs: + if A[1]["Civ"] is not None and A[1]["Civ"] in Civs: sortVal += 100 * Civs.index(A[1]["Civ"]) return sortVal @@ -403,9 +401,7 @@ def WriteColouredDiff(file, diff, isChanged): file.write( """{} - """.format( - rgb_str, cleverParse(diff) - ) + """.format(rgb_str, cleverParse(diff)) ) return isChanged @@ -413,10 +409,14 @@ def WriteColouredDiff(file, diff, isChanged): def computeUnitEfficiencyDiff(TemplatesByParent, Civs): efficiency_table = {} for parent in TemplatesByParent: - for template in [template for template in TemplatesByParent[parent] if template[1]["Civ"] not in Civs]: + for template in [ + template for template in TemplatesByParent[parent] if template[1]["Civ"] not in Civs + ]: print(template) - TemplatesByParent[parent] = [template for template in TemplatesByParent[parent] if template[1]["Civ"] in Civs] + TemplatesByParent[parent] = [ + template for template in TemplatesByParent[parent] if template[1]["Civ"] in Civs + ] TemplatesByParent[parent].sort(key=lambda x: Civs.index(x[1]["Civ"])) for tp in TemplatesByParent[parent]: @@ -426,15 +426,11 @@ def computeUnitEfficiencyDiff(TemplatesByParent, Civs): efficiency_table[(parent, tp[0], "HP")] = diff # Build Time - diff = +1j + (int(tp[1]["BuildTime"]) - - int(templates[parent]["BuildTime"])) + diff = +1j + (int(tp[1]["BuildTime"]) - int(templates[parent]["BuildTime"])) efficiency_table[(parent, tp[0], "BuildTime")] = diff # walk speed - diff = -1j + ( - float(tp[1]["WalkSpeed"]) - - float(templates[parent]["WalkSpeed"]) - ) + diff = -1j + (float(tp[1]["WalkSpeed"]) - float(templates[parent]["WalkSpeed"])) efficiency_table[(parent, tp[0], "WalkSpeed")] = diff # Resistance @@ -446,54 +442,42 @@ def computeUnitEfficiencyDiff(TemplatesByParent, Civs): efficiency_table[(parent, tp[0], "Resistance/" + atype)] = diff # Attack types (DPS) and rate. - attType = "Ranged" if tp[1]["Ranged"] == True else "Melee" + attType = "Ranged" if tp[1]["Ranged"] is True else "Melee" if tp[1]["RepeatRate"][attType] != "0": for atype in AttackTypes: myDPS = float(tp[1]["Attack"][attType][atype]) / ( float(tp[1]["RepeatRate"][attType]) / 1000.0 ) - parentDPS = float( - templates[parent]["Attack"][attType][atype]) / ( + parentDPS = float(templates[parent]["Attack"][attType][atype]) / ( float(templates[parent]["RepeatRate"][attType]) / 1000.0 ) diff = -1j + (myDPS - parentDPS) - efficiency_table[ - (parent, tp[0], "Attack/" + attType + "/" + atype) - ] = diff + efficiency_table[(parent, tp[0], "Attack/" + attType + "/" + atype)] = diff diff = -1j + ( float(tp[1]["RepeatRate"][attType]) / 1000.0 - float(templates[parent]["RepeatRate"][attType]) / 1000.0 ) efficiency_table[ - (parent, tp[0], "Attack/" + attType + "/" + atype + - "/RepeatRate") + (parent, tp[0], "Attack/" + attType + "/" + atype + "/RepeatRate") ] = diff # range and spread - if tp[1]["Ranged"] == True: - diff = -1j + ( - float(tp[1]["Range"]) - - float(templates[parent]["Range"]) - ) - efficiency_table[ - (parent, tp[0], "Attack/" + attType + "/Ranged/Range") - ] = diff + if tp[1]["Ranged"] is True: + diff = -1j + (float(tp[1]["Range"]) - float(templates[parent]["Range"])) + efficiency_table[(parent, tp[0], "Attack/" + attType + "/Ranged/Range")] = diff - diff = (float(tp[1]["Spread"]) - - float(templates[parent]["Spread"])) - efficiency_table[ - (parent, tp[0], "Attack/" + attType + "/Ranged/Spread") - ] = diff + diff = float(tp[1]["Spread"]) - float(templates[parent]["Spread"]) + efficiency_table[(parent, tp[0], "Attack/" + attType + "/Ranged/Spread")] = ( + diff + ) for rtype in Resources: diff = +1j + ( - float(tp[1]["Cost"][rtype]) - - float(templates[parent]["Cost"][rtype]) + float(tp[1]["Cost"][rtype]) - float(templates[parent]["Cost"][rtype]) ) efficiency_table[(parent, tp[0], "Resources/" + rtype)] = diff diff = +1j + ( - float(tp[1]["Cost"]["population"]) - - float(templates[parent]["Cost"]["population"]) + float(tp[1]["Cost"]["population"]) - float(templates[parent]["Cost"]["population"]) ) efficiency_table[(parent, tp[0], "Population")] = diff @@ -512,7 +496,7 @@ def computeTemplates(LoadTemplatesIfParent): if hasParentTemplate(template, possParent): found = True break - if found == True: + if found is True: templates[template] = CalcUnit(template) os.chdir(pwd) return templates @@ -541,7 +525,6 @@ def computeCivTemplates(template: dict, Civs: list): civ_list = list(glob.glob("units/" + Civ + "/*.xml")) for template in civ_list: if os.path.isfile(template): - # filter based on FilterOut breakIt = False for filter in FilterOut: @@ -601,17 +584,14 @@ CivTemplates = computeCivTemplates(templates, Civs) TemplatesByParent = computeTemplatesByParent(templates, Civs, CivTemplates) # Not used; use it for your own custom analysis -efficiencyTable = computeUnitEfficiencyDiff( - TemplatesByParent, Civs -) +efficiencyTable = computeUnitEfficiencyDiff(TemplatesByParent, Civs) ############################################################ def writeHTML(): """Create the HTML file""" f = open( - os.path.realpath(__file__).replace("unitTables.py", "") - + "unit_summary_table.html", + os.path.realpath(__file__).replace("unitTables.py", "") + "unit_summary_table.html", "w", ) @@ -699,10 +679,7 @@ differences between the two. TemplatesByParent[parent].sort(key=lambda x: Civs.index(x[1]["Civ"])) for tp in TemplatesByParent[parent]: isChanged = False - ff = open( - os.path.realpath(__file__).replace("unitTables.py", "") + - ".cache", "w" - ) + ff = open(os.path.realpath(__file__).replace("unitTables.py", "") + ".cache", "w") ff.write("") ff.write( @@ -711,9 +688,7 @@ differences between the two. + "" ) ff.write( - '' - + tp[0].replace(".xml", "").replace("units/", "") - + "" + '' + tp[0].replace(".xml", "").replace("units/", "") + "" ) # HP @@ -721,15 +696,11 @@ differences between the two. isChanged = WriteColouredDiff(ff, diff, isChanged) # Build Time - diff = +1j + (int(tp[1]["BuildTime"]) - - int(templates[parent]["BuildTime"])) + diff = +1j + (int(tp[1]["BuildTime"]) - int(templates[parent]["BuildTime"])) isChanged = WriteColouredDiff(ff, diff, isChanged) # walk speed - diff = -1j + ( - float(tp[1]["WalkSpeed"]) - - float(templates[parent]["WalkSpeed"]) - ) + diff = -1j + (float(tp[1]["WalkSpeed"]) - float(templates[parent]["WalkSpeed"])) isChanged = WriteColouredDiff(ff, diff, isChanged) # Resistance @@ -741,19 +712,16 @@ differences between the two. isChanged = WriteColouredDiff(ff, diff, isChanged) # Attack types (DPS) and rate. - attType = "Ranged" if tp[1]["Ranged"] == True else "Melee" + attType = "Ranged" if tp[1]["Ranged"] is True else "Melee" if tp[1]["RepeatRate"][attType] != "0": for atype in AttackTypes: myDPS = float(tp[1]["Attack"][attType][atype]) / ( float(tp[1]["RepeatRate"][attType]) / 1000.0 ) - parentDPS = float( - templates[parent]["Attack"][attType][atype]) / ( + parentDPS = float(templates[parent]["Attack"][attType][atype]) / ( float(templates[parent]["RepeatRate"][attType]) / 1000.0 ) - isChanged = WriteColouredDiff( - ff, -1j + (myDPS - parentDPS), isChanged - ) + isChanged = WriteColouredDiff(ff, -1j + (myDPS - parentDPS), isChanged) isChanged = WriteColouredDiff( ff, -1j @@ -764,32 +732,26 @@ differences between the two. isChanged, ) # range and spread - if tp[1]["Ranged"] == True: + if tp[1]["Ranged"] is True: isChanged = WriteColouredDiff( ff, - -1j - + (float(tp[1]["Range"]) - - float(templates[parent]["Range"])), + -1j + (float(tp[1]["Range"]) - float(templates[parent]["Range"])), isChanged, ) mySpread = float(tp[1]["Spread"]) parentSpread = float(templates[parent]["Spread"]) - isChanged = WriteColouredDiff( - ff, +1j + (mySpread - parentSpread), isChanged - ) + isChanged = WriteColouredDiff(ff, +1j + (mySpread - parentSpread), isChanged) else: - ff.write("--") + ff.write( + "--" + ) else: ff.write("") for rtype in Resources: isChanged = WriteColouredDiff( ff, - +1j - + ( - float(tp[1]["Cost"][rtype]) - - float(templates[parent]["Cost"][rtype]) - ), + +1j + (float(tp[1]["Cost"][rtype]) - float(templates[parent]["Cost"][rtype])), isChanged, ) @@ -808,8 +770,7 @@ differences between the two. ff.close() # to actually write into the file with open( - os.path.realpath(__file__).replace("unitTables.py", "") + - ".cache", "r" + os.path.realpath(__file__).replace("unitTables.py", "") + ".cache", "r" ) as ff: unitStr = ff.read() diff --git a/source/tools/xmlvalidator/validate_grammar.py b/source/tools/xmlvalidator/validate_grammar.py index 12476b0c6d..6b830fb883 100644 --- a/source/tools/xmlvalidator/validate_grammar.py +++ b/source/tools/xmlvalidator/validate_grammar.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 from argparse import ArgumentParser from pathlib import Path -from os.path import sep, join, realpath, exists, basename, dirname -from json import load, loads -from re import split, match +from os.path import join, realpath, exists, dirname +from json import load +from re import match from logging import getLogger, StreamHandler, INFO, WARNING, Filter, Formatter import lxml.etree import sys + class SingleLevelFilter(Filter): def __init__(self, passlevel, reject): self.passlevel = passlevel @@ -15,15 +16,17 @@ class SingleLevelFilter(Filter): def filter(self, record): if self.reject: - return (record.levelno != self.passlevel) + return record.levelno != self.passlevel else: - return (record.levelno == self.passlevel) + return record.levelno == self.passlevel + class VFS_File: def __init__(self, mod_name, vfs_path): self.mod_name = mod_name self.vfs_path = vfs_path + class RelaxNGValidator: def __init__(self, vfs_root, mods=None, verbose=False): self.mods = mods if mods is not None else [] @@ -38,18 +41,18 @@ class RelaxNGValidator: # create a console handler, seems nicer to Windows and for future uses ch = StreamHandler(sys.stdout) ch.setLevel(INFO) - ch.setFormatter(Formatter('%(levelname)s - %(message)s')) + ch.setFormatter(Formatter("%(levelname)s - %(message)s")) f1 = SingleLevelFilter(INFO, False) ch.addFilter(f1) logger.addHandler(ch) errorch = StreamHandler(sys.stderr) errorch.setLevel(WARNING) - errorch.setFormatter(Formatter('%(levelname)s - %(message)s')) + errorch.setFormatter(Formatter("%(levelname)s - %(message)s")) logger.addHandler(errorch) self.logger = logger self.inError = False - def run (self): + def run(self): self.validate_actors() self.validate_variants() self.validate_guis() @@ -63,7 +66,7 @@ class RelaxNGValidator: return self.inError def main(self): - """ Program entry point, parses command line arguments and launches the validation """ + """Program entry point, parses command line arguments and launches the validation""" # ordered uniq mods (dict maintains ordered keys from python 3.6) self.logger.info(f"Checking {'|'.join(self.mods)}'s integrity.") self.logger.info(f"The following mods will be loaded: {'|'.join(self.mods)}.") @@ -75,88 +78,115 @@ class RelaxNGValidator: - Path relative to the mod base - full Path """ - full_exts = ['.' + ext for ext in ext_list] + full_exts = ["." + ext for ext in ext_list] def find_recursive(dp, base): """(relative Path, full Path) generator""" if dp.is_dir(): - if dp.name != '.svn' and dp.name != '.git' and not dp.name.endswith('~'): + if dp.name != ".svn" and dp.name != ".git" and not dp.name.endswith("~"): for fp in dp.iterdir(): yield from find_recursive(fp, base) elif dp.suffix in full_exts: relative_file_path = dp.relative_to(base) yield (relative_file_path, dp.resolve()) - return [(rp, fp) for mod in mods for (rp, fp) in find_recursive(vfs_root / mod / vfs_path, vfs_root / mod)] + + return [ + (rp, fp) + for mod in mods + for (rp, fp) in find_recursive(vfs_root / mod / vfs_path, vfs_root / mod) + ] def validate_actors(self): - self.logger.info('Validating actors...') - files = self.find_files(self.vfs_root, self.mods, 'art/actors/', 'xml') - self.validate_files('actors', files, 'art/actors/actor.rng') + self.logger.info("Validating actors...") + files = self.find_files(self.vfs_root, self.mods, "art/actors/", "xml") + self.validate_files("actors", files, "art/actors/actor.rng") def validate_variants(self): self.logger.info("Validating variants...") - files = self.find_files(self.vfs_root, self.mods, 'art/variants/', 'xml') - self.validate_files('variant', files, 'art/variants/variant.rng') + files = self.find_files(self.vfs_root, self.mods, "art/variants/", "xml") + self.validate_files("variant", files, "art/variants/variant.rng") def validate_guis(self): self.logger.info("Validating gui files...") - pages = [file for file in self.find_files(self.vfs_root, self.mods, 'gui/', 'xml') if match(r".*[\\\/]page(_[^.\/\\]+)?\.xml$", str(file[0]))] - self.validate_files('gui page', pages, 'gui/gui_page.rng') - xmls = [file for file in self.find_files(self.vfs_root, self.mods, 'gui/', 'xml') if not match(r".*[\\\/]page(_[^.\/\\]+)?\.xml$", str(file[0]))] - self.validate_files('gui xml', xmls, 'gui/gui.rng') + pages = [ + file + for file in self.find_files(self.vfs_root, self.mods, "gui/", "xml") + if match(r".*[\\\/]page(_[^.\/\\]+)?\.xml$", str(file[0])) + ] + self.validate_files("gui page", pages, "gui/gui_page.rng") + xmls = [ + file + for file in self.find_files(self.vfs_root, self.mods, "gui/", "xml") + if not match(r".*[\\\/]page(_[^.\/\\]+)?\.xml$", str(file[0])) + ] + self.validate_files("gui xml", xmls, "gui/gui.rng") def validate_maps(self): self.logger.info("Validating maps...") - files = self.find_files(self.vfs_root, self.mods, 'maps/scenarios/', 'xml') - self.validate_files('map', files, 'maps/scenario.rng') - files = self.find_files(self.vfs_root, self.mods, 'maps/skirmishes/', 'xml') - self.validate_files('map', files, 'maps/scenario.rng') + files = self.find_files(self.vfs_root, self.mods, "maps/scenarios/", "xml") + self.validate_files("map", files, "maps/scenario.rng") + files = self.find_files(self.vfs_root, self.mods, "maps/skirmishes/", "xml") + self.validate_files("map", files, "maps/scenario.rng") def validate_materials(self): self.logger.info("Validating materials...") - files = self.find_files(self.vfs_root, self.mods, 'art/materials/', 'xml') - self.validate_files('material', files, 'art/materials/material.rng') + files = self.find_files(self.vfs_root, self.mods, "art/materials/", "xml") + self.validate_files("material", files, "art/materials/material.rng") def validate_particles(self): self.logger.info("Validating particles...") - files = self.find_files(self.vfs_root, self.mods, 'art/particles/', 'xml') - self.validate_files('particle', files, 'art/particles/particle.rng') + files = self.find_files(self.vfs_root, self.mods, "art/particles/", "xml") + self.validate_files("particle", files, "art/particles/particle.rng") def validate_simulation(self): self.logger.info("Validating simulation...") - file = self.find_files(self.vfs_root, self.mods, 'simulation/data/pathfinder', 'xml') - self.validate_files('pathfinder', file, 'simulation/data/pathfinder.rng') - file = self.find_files(self.vfs_root, self.mods, 'simulation/data/territorymanager', 'xml') - self.validate_files('territory manager', file, 'simulation/data/territorymanager.rng') + file = self.find_files(self.vfs_root, self.mods, "simulation/data/pathfinder", "xml") + self.validate_files("pathfinder", file, "simulation/data/pathfinder.rng") + file = self.find_files(self.vfs_root, self.mods, "simulation/data/territorymanager", "xml") + self.validate_files("territory manager", file, "simulation/data/territorymanager.rng") def validate_soundgroups(self): self.logger.info("Validating soundgroups...") - files = self.find_files(self.vfs_root, self.mods, 'audio/', 'xml') - self.validate_files('sound group', files, 'audio/sound_group.rng') + files = self.find_files(self.vfs_root, self.mods, "audio/", "xml") + self.validate_files("sound group", files, "audio/sound_group.rng") def validate_terrains(self): self.logger.info("Validating terrains...") - terrains = [file for file in self.find_files(self.vfs_root, self.mods, 'art/terrains/', 'xml') if 'terrains.xml' in str(file[0])] - self.validate_files('terrain', terrains, 'art/terrains/terrain.rng') - terrains_textures = [file for file in self.find_files(self.vfs_root, self.mods, 'art/terrains/', 'xml') if 'terrains.xml' not in str(file[0])] - self.validate_files('terrain texture', terrains_textures, 'art/terrains/terrain_texture.rng') + terrains = [ + file + for file in self.find_files(self.vfs_root, self.mods, "art/terrains/", "xml") + if "terrains.xml" in str(file[0]) + ] + self.validate_files("terrain", terrains, "art/terrains/terrain.rng") + terrains_textures = [ + file + for file in self.find_files(self.vfs_root, self.mods, "art/terrains/", "xml") + if "terrains.xml" not in str(file[0]) + ] + self.validate_files( + "terrain texture", terrains_textures, "art/terrains/terrain_texture.rng" + ) def validate_textures(self): self.logger.info("Validating textures...") - files = [file for file in self.find_files(self.vfs_root, self.mods, 'art/textures/', 'xml') if 'textures.xml' in str(file[0])] - self.validate_files('texture', files, 'art/textures/texture.rng') + files = [ + file + for file in self.find_files(self.vfs_root, self.mods, "art/textures/", "xml") + if "textures.xml" in str(file[0]) + ] + self.validate_files("texture", files, "art/textures/texture.rng") def get_physical_path(self, mod_name, vfs_path): return realpath(join(self.vfs_root, mod_name, vfs_path)) def get_relaxng_file(self, schemapath): - """We look for the highest priority mod relax NG file""" - for mod in self.mods: - relax_ng_path = self.get_physical_path(mod, schemapath) - if exists(relax_ng_path): - return relax_ng_path + """We look for the highest priority mod relax NG file""" + for mod in self.mods: + relax_ng_path = self.get_physical_path(mod, schemapath) + if exists(relax_ng_path): + return relax_ng_path - return "" + return "" def validate_files(self, name, files, schemapath): relax_ng_path = self.get_relaxng_file(schemapath) @@ -185,27 +215,40 @@ class RelaxNGValidator: def get_mod_dependencies(vfs_root, *mods): modjsondeps = [] for mod in mods: - mod_json_path = Path(vfs_root) / mod / 'mod.json' + mod_json_path = Path(vfs_root) / mod / "mod.json" if not exists(mod_json_path): continue - with open(mod_json_path, encoding='utf-8') as f: + with open(mod_json_path, encoding="utf-8") as f: modjson = load(f) # 0ad's folder isn't named like the mod. - modjsondeps.extend(['public' if '0ad' in dep else dep for dep in modjson.get('dependencies', [])]) + modjsondeps.extend( + ["public" if "0ad" in dep else dep for dep in modjson.get("dependencies", [])] + ) return modjsondeps -if __name__ == '__main__': + +if __name__ == "__main__": script_dir = dirname(realpath(__file__)) - default_root = join(script_dir, '..', '..', '..', 'binaries', 'data', 'mods') + default_root = join(script_dir, "..", "..", "..", "binaries", "data", "mods") ap = ArgumentParser(description="Validates XML files againt their Relax NG schemas") - ap.add_argument('-r', '--root', action='store', dest='root', default=default_root) - ap.add_argument('-v', '--verbose', action='store_true', default=True, - help="Log validation errors.") - ap.add_argument('-m', '--mods', metavar="MOD", dest='mods', nargs='+', default=['public'], - help="specify which mods to check. Default to public and mod.") + ap.add_argument("-r", "--root", action="store", dest="root", default=default_root) + ap.add_argument( + "-v", "--verbose", action="store_true", default=True, help="Log validation errors." + ) + ap.add_argument( + "-m", + "--mods", + metavar="MOD", + dest="mods", + nargs="+", + default=["public"], + help="specify which mods to check. Default to public and mod.", + ) args = ap.parse_args() - mods = list(dict.fromkeys([*args.mods, *get_mod_dependencies(args.root, *args.mods), 'mod']).keys()) + mods = list( + dict.fromkeys([*args.mods, *get_mod_dependencies(args.root, *args.mods), "mod"]).keys() + ) relax_ng_validator = RelaxNGValidator(args.root, mods=mods, verbose=args.verbose) if not relax_ng_validator.main(): sys.exit(1) diff --git a/source/tools/xmlvalidator/validator.py b/source/tools/xmlvalidator/validator.py index f1e0675a26..cd354fac48 100644 --- a/source/tools/xmlvalidator/validator.py +++ b/source/tools/xmlvalidator/validator.py @@ -6,6 +6,7 @@ import re import xml.etree.ElementTree from logging import getLogger, StreamHandler, INFO, WARNING, Formatter, Filter + class SingleLevelFilter(Filter): def __init__(self, passlevel, reject): self.passlevel = passlevel @@ -13,9 +14,10 @@ class SingleLevelFilter(Filter): def filter(self, record): if self.reject: - return (record.levelno != self.passlevel) + return record.levelno != self.passlevel else: - return (record.levelno == self.passlevel) + return record.levelno == self.passlevel + class Actor: def __init__(self, mod_name, vfs_path): @@ -23,7 +25,7 @@ class Actor: self.vfs_path = vfs_path self.name = os.path.basename(vfs_path) self.textures = [] - self.material = '' + self.material = "" self.logger = getLogger(__name__) def read(self, physical_path): @@ -34,17 +36,17 @@ class Actor: return False root = tree.getroot() # Special case: particles don't need a diffuse texture. - if len(root.findall('.//particles')) > 0: + if len(root.findall(".//particles")) > 0: self.textures.append("baseTex") - for element in root.findall('.//material'): + for element in root.findall(".//material"): self.material = element.text - for element in root.findall('.//texture'): - self.textures.append(element.get('name')) - for element in root.findall('.//variant'): - file = element.get('file') + for element in root.findall(".//texture"): + self.textures.append(element.get("name")) + for element in root.findall(".//variant"): + file = element.get("file") if file: - self.read_variant(physical_path, os.path.join('art', 'variants', file)) + self.read_variant(physical_path, os.path.join("art", "variants", file)) return True def read_variant(self, actor_physical_path, relative_path): @@ -56,12 +58,12 @@ class Actor: return False root = tree.getroot() - file = root.get('file') + file = root.get("file") if file: - self.read_variant(actor_physical_path, os.path.join('art', 'variants', file)) + self.read_variant(actor_physical_path, os.path.join("art", "variants", file)) - for element in root.findall('.//texture'): - self.textures.append(element.get('name')) + for element in root.findall(".//texture"): + self.textures.append(element.get("name")) class Material: @@ -77,8 +79,8 @@ class Material: except xml.etree.ElementTree.ParseError as err: self.logger.error('"%s": %s' % (physical_path, err.msg)) return False - for element in root.findall('.//required_texture'): - texture_name = element.get('name') + for element in root.findall(".//required_texture"): + texture_name = element.get("name") self.required_textures.append(texture_name) return True @@ -86,7 +88,7 @@ class Material: class Validator: def __init__(self, vfs_root, mods=None): if mods is None: - mods = ['mod', 'public'] + mods = ["mod", "public"] self.vfs_root = vfs_root self.mods = mods @@ -102,13 +104,13 @@ class Validator: # create a console handler, seems nicer to Windows and for future uses ch = StreamHandler(sys.stdout) ch.setLevel(INFO) - ch.setFormatter(Formatter('%(levelname)s - %(message)s')) + ch.setFormatter(Formatter("%(levelname)s - %(message)s")) f1 = SingleLevelFilter(INFO, False) ch.addFilter(f1) logger.addHandler(ch) errorch = StreamHandler(sys.stderr) errorch.setLevel(WARNING) - errorch.setFormatter(Formatter('%(levelname)s - %(message)s')) + errorch.setFormatter(Formatter("%(levelname)s - %(message)s")) logger.addHandler(errorch) self.logger = logger self.inError = False @@ -125,17 +127,14 @@ class Validator: if not os.path.isdir(physical_path): return result for file_name in os.listdir(physical_path): - if file_name == '.git' or file_name == '.svn': + if file_name == ".git" or file_name == ".svn": continue vfs_file_path = os.path.join(vfs_path, file_name) physical_file_path = os.path.join(physical_path, file_name) if os.path.isdir(physical_file_path): result += self.find_mod_files(mod_name, vfs_file_path, pattern) elif os.path.isfile(physical_file_path) and pattern.match(file_name): - result.append({ - 'mod_name': mod_name, - 'vfs_path': vfs_file_path - }) + result.append({"mod_name": mod_name, "vfs_path": vfs_file_path}) return result def find_all_mods_files(self, vfs_path, pattern): @@ -145,72 +144,100 @@ class Validator: return result def find_materials(self, vfs_path): - self.logger.info('Collecting materials...') - material_files = self.find_all_mods_files(vfs_path, re.compile(r'.*\.xml')) + self.logger.info("Collecting materials...") + material_files = self.find_all_mods_files(vfs_path, re.compile(r".*\.xml")) for material_file in material_files: - material_name = os.path.basename(material_file['vfs_path']) + material_name = os.path.basename(material_file["vfs_path"]) if material_name in self.materials: continue - material = Material(material_file['mod_name'], material_file['vfs_path']) - if material.read(self.get_physical_path(material_file['mod_name'], material_file['vfs_path'])): + material = Material(material_file["mod_name"], material_file["vfs_path"]) + if material.read( + self.get_physical_path(material_file["mod_name"], material_file["vfs_path"]) + ): self.materials[material_name] = material else: self.invalid_materials[material_name] = material def find_actors(self, vfs_path): - self.logger.info('Collecting actors...') + self.logger.info("Collecting actors...") - actor_files = self.find_all_mods_files(vfs_path, re.compile(r'.*\.xml')) + actor_files = self.find_all_mods_files(vfs_path, re.compile(r".*\.xml")) for actor_file in actor_files: - actor = Actor(actor_file['mod_name'], actor_file['vfs_path']) - if actor.read(self.get_physical_path(actor_file['mod_name'], actor_file['vfs_path'])): + actor = Actor(actor_file["mod_name"], actor_file["vfs_path"]) + if actor.read(self.get_physical_path(actor_file["mod_name"], actor_file["vfs_path"])): self.actors.append(actor) def run(self): - self.find_materials(os.path.join('art', 'materials')) - self.find_actors(os.path.join('art', 'actors')) - self.logger.info('Validating textures...') + self.find_materials(os.path.join("art", "materials")) + self.find_actors(os.path.join("art", "actors")) + self.logger.info("Validating textures...") for actor in self.actors: if not actor.material: continue - if actor.material not in self.materials and actor.material not in self.invalid_materials: - self.logger.error('"%s": unknown material "%s"' % ( - self.get_mod_path(actor.mod_name, actor.vfs_path), - actor.material - )) + if ( + actor.material not in self.materials + and actor.material not in self.invalid_materials + ): + self.logger.error( + '"%s": unknown material "%s"' + % (self.get_mod_path(actor.mod_name, actor.vfs_path), actor.material) + ) self.inError = True if actor.material not in self.materials: continue material = self.materials[actor.material] - missing_textures = ', '.join(set([required_texture for required_texture in material.required_textures if required_texture not in actor.textures])) + missing_textures = ", ".join( + set( + [ + required_texture + for required_texture in material.required_textures + if required_texture not in actor.textures + ] + ) + ) if len(missing_textures) > 0: - self.logger.error('"%s": actor does not contain required texture(s) "%s" from "%s"' % ( - self.get_mod_path(actor.mod_name, actor.vfs_path), - missing_textures, - material.name - )) + self.logger.error( + '"%s": actor does not contain required texture(s) "%s" from "%s"' + % ( + self.get_mod_path(actor.mod_name, actor.vfs_path), + missing_textures, + material.name, + ) + ) self.inError = True - extra_textures = ', '.join(set([extra_texture for extra_texture in actor.textures if extra_texture not in material.required_textures])) + extra_textures = ", ".join( + set( + [ + extra_texture + for extra_texture in actor.textures + if extra_texture not in material.required_textures + ] + ) + ) if len(extra_textures) > 0: - self.logger.warning('"%s": actor contains unnecessary texture(s) "%s" from "%s"' % ( - self.get_mod_path(actor.mod_name, actor.vfs_path), - extra_textures, - material.name - )) + self.logger.warning( + '"%s": actor contains unnecessary texture(s) "%s" from "%s"' + % ( + self.get_mod_path(actor.mod_name, actor.vfs_path), + extra_textures, + material.name, + ) + ) self.inError = True return self.inError -if __name__ == '__main__': + +if __name__ == "__main__": script_dir = os.path.dirname(os.path.realpath(__file__)) - default_root = os.path.join(script_dir, '..', '..', '..', 'binaries', 'data', 'mods') - parser = argparse.ArgumentParser(description='Actors/materials validator.') - parser.add_argument('-r', '--root', action='store', dest='root', default=default_root) - parser.add_argument('-m', '--mods', action='store', dest='mods', default='mod,public') + default_root = os.path.join(script_dir, "..", "..", "..", "binaries", "data", "mods") + parser = argparse.ArgumentParser(description="Actors/materials validator.") + parser.add_argument("-r", "--root", action="store", dest="root", default=default_root) + parser.add_argument("-m", "--mods", action="store", dest="mods", default="mod,public") args = parser.parse_args() - validator = Validator(args.root, args.mods.split(',')) + validator = Validator(args.root, args.mods.split(",")) if not validator.run(): sys.exit(1)