From f2bef8388a0180119cd0466b6da30f62f744d233 Mon Sep 17 00:00:00 2001 From: Dunedan Date: Thu, 29 Aug 2024 07:17:10 +0200 Subject: [PATCH] Use UTF-8 as encoding when working with files This explicitly uses UTF-8 encoding when reading or writing files with Python. This is necessary as the default locale varies between operating systems. --- source/collada/tests/tests.py | 16 +++++++++++----- source/tools/entity/creationgraph.py | 2 +- source/tools/fontbuilder2/fontbuilder.py | 4 ++-- source/tools/i18n/extractors/extractors.py | 2 +- source/tools/i18n/updateTemplates.py | 2 +- source/tools/mapcompatibility/a18_to_a19.py | 5 ++++- .../rlclient/python/samples/simple-example.py | 2 +- .../tools/rlclient/python/tests/test_actions.py | 2 +- .../tools/rlclient/python/tests/test_evaluate.py | 4 ++-- source/tools/spirv/compile.py | 4 ++-- source/tools/templatesanalyzer/unitTables.py | 12 ++++++++++-- 11 files changed, 36 insertions(+), 19 deletions(-) diff --git a/source/collada/tests/tests.py b/source/collada/tests/tests.py index 154666bf67..80955d8c0f 100755 --- a/source/collada/tests/tests.py +++ b/source/collada/tests/tests.py @@ -31,7 +31,7 @@ def log(severity, message): clog = CFUNCTYPE(None, c_int, c_char_p)(log) # (the CFUNCTYPE must not be GC'd, so try to keep a reference) library.set_logger(clog) -with open(f"{binaries}/data/tests/collada/skeletons.xml") as fd: +with open(f"{binaries}/data/tests/collada/skeletons.xml", encoding="utf-8") as fd: skeleton_definitions = fd.read() library.set_skeleton_definitions(skeleton_definitions, len(skeleton_definitions)) @@ -127,7 +127,10 @@ for test_file in ["xsitest3c", "xsitest3e", "jav2d", "jav2d2"]: input_filename = f"{test_data}/{test_file}.dae" output_filename = f"{test_mod}/art/meshes/{test_file}.pmd" - with open(input_filename) as input_fd, open(output_filename, "wb") as output_fd: + with ( + open(input_filename, encoding="utf-8") as input_fd, + open(output_filename, "wb") as output_fd, + ): file_input = input_fd.read() file_output = convert_dae_to_pmd(file_input) output_fd.write(file_output) @@ -143,11 +146,11 @@ for test_file in ["xsitest3c", "xsitest3e", "jav2d", "jav2d2"]: ], [("helmet", "teapot_basic_static")], ) - with open(f"{test_mod}/art/actors/{test_file}.xml", "w") as fd: + with open(f"{test_mod}/art/actors/{test_file}.xml", "w", encoding="utf-8") as fd: fd.write(xml) xml = create_actor_static(test_file, "male") - with open(f"{test_mod}/art/actors/{test_file}_static.xml", "w") as fd: + with open(f"{test_mod}/art/actors/{test_file}_static.xml", "w", encoding="utf-8") as fd: fd.write(xml) # for test_file in ['jav2','jav2b', 'jav2d']: @@ -158,7 +161,10 @@ for test_file in ["xsitest3c", "xsitest3e", "jav2d", "jav2d2"]: input_filename = f"{test_data}/{test_file}.dae" output_filename = f"{test_mod}/art/animation/{test_file}.psa" - with open(input_filename) as input_fd, open(output_filename, "wb") as output_fd: + with ( + open(input_filename, encoding="utf-8") as input_fd, + open(output_filename, "wb") as output_fd, + ): file_input = input_fd.read() file_output = convert_dae_to_psa(file_input) output_fd.write(file_output) diff --git a/source/tools/entity/creationgraph.py b/source/tools/entity/creationgraph.py index 180c937ea1..6c9bb81a60 100755 --- a/source/tools/entity/creationgraph.py +++ b/source/tools/entity/creationgraph.py @@ -20,7 +20,7 @@ def main(): vfs_root = Path(__file__).resolve().parents[3] / "binaries" / "data" / "mods" simul_templates_path = Path("simulation/templates") simul_template_entity = SimulTemplateEntity(vfs_root) - with open("creation.dot", "w") as dot_f: + with open("creation.dot", "w", encoding="utf-8") as dot_f: dot_f.write("digraph G {\n") files = sorted(find_entities(vfs_root)) for f in files: diff --git a/source/tools/fontbuilder2/fontbuilder.py b/source/tools/fontbuilder2/fontbuilder.py index 31be741402..5d2a2ebb47 100755 --- a/source/tools/fontbuilder2/fontbuilder.py +++ b/source/tools/fontbuilder2/fontbuilder.py @@ -86,7 +86,7 @@ class Glyph: # Load the set of characters contained in the given text file def load_char_list(filename): - with open(filename) as f: + with open(filename, encoding="utf-8") as f: chars = f.read() return set(chars) @@ -167,7 +167,7 @@ def generate_font(outname, ttfNames, loadopts, size, renderstyle, dsizes): surface.write_to_png(f"{outname}.png") # Output the .fnt file with all the glyph positions etc - with open(f"{outname}.fnt", "w") as fnt: + with open(f"{outname}.fnt", "w", encoding="utf-8") as fnt: fnt.write("101\n") fnt.write("%d %d\n" % (w, h)) fnt.write("%s\n" % ("rgba" if "colour" in renderstyle else "a")) diff --git a/source/tools/i18n/extractors/extractors.py b/source/tools/i18n/extractors/extractors.py index baf9707252..c2154c746a 100644 --- a/source/tools/i18n/extractors/extractors.py +++ b/source/tools/i18n/extractors/extractors.py @@ -511,7 +511,7 @@ class ini(Extractor): import ConfigParser config = ConfigParser.RawConfigParser() - with open(filepath) as fd: + with open(filepath, encoding="utf-8") as fd: config.read_file(FakeSectionHeader(fd)) for keyword in self.keywords: message = config.get("root", keyword).strip('"').strip("'") diff --git a/source/tools/i18n/updateTemplates.py b/source/tools/i18n/updateTemplates.py index 84f4b61236..680d14168a 100755 --- a/source/tools/i18n/updateTemplates.py +++ b/source/tools/i18n/updateTemplates.py @@ -101,7 +101,7 @@ def generatePOT(templateSettings, rootPath): def generateTemplatesForMessagesFile(messagesFilePath): - with open(messagesFilePath) as fileObject: + with open(messagesFilePath, encoding="utf-8") as fileObject: settings = json.load(fileObject) for templateSettings in settings: diff --git a/source/tools/mapcompatibility/a18_to_a19.py b/source/tools/mapcompatibility/a18_to_a19.py index c94347de52..1a6ab8a71e 100755 --- a/source/tools/mapcompatibility/a18_to_a19.py +++ b/source/tools/mapcompatibility/a18_to_a19.py @@ -126,7 +126,10 @@ for xmlFile in args.files: os.rename(pmpFile + "~", pmpFile) if os.path.isfile(xmlFile): - with open(xmlFile) as f1, open(xmlFile + "~", "w") as f2: + with ( + open(xmlFile, encoding="utf-8") as f1, + open(xmlFile + "~", "w", encoding="utf-8") as f2, + ): data = f1.read() # bump version number (rely on how Atlas formats the XML) diff --git a/source/tools/rlclient/python/samples/simple-example.py b/source/tools/rlclient/python/samples/simple-example.py index 920ed805ab..26ae80117d 100644 --- a/source/tools/rlclient/python/samples/simple-example.py +++ b/source/tools/rlclient/python/samples/simple-example.py @@ -33,7 +33,7 @@ game = zero_ad.ZeroAD("http://localhost:6000") # Load the Arcadia map samples_dir = path.dirname(path.realpath(__file__)) scenario_config_path = path.join(samples_dir, "arcadia.json") -with open(scenario_config_path, encoding="utf8") as f: +with open(scenario_config_path, encoding="utf-8") as f: arcadia_config = f.read() state = game.reset(arcadia_config) diff --git a/source/tools/rlclient/python/tests/test_actions.py b/source/tools/rlclient/python/tests/test_actions.py index 7e0617d232..f9c6742d01 100644 --- a/source/tools/rlclient/python/tests/test_actions.py +++ b/source/tools/rlclient/python/tests/test_actions.py @@ -6,7 +6,7 @@ import zero_ad game = zero_ad.ZeroAD("http://localhost:6000") scriptdir = path.dirname(path.realpath(__file__)) -with open(path.join(scriptdir, "..", "samples", "arcadia.json"), encoding="utf8") as f: +with open(path.join(scriptdir, "..", "samples", "arcadia.json"), encoding="utf-8") as f: config = f.read() diff --git a/source/tools/rlclient/python/tests/test_evaluate.py b/source/tools/rlclient/python/tests/test_evaluate.py index 6ebf8e328a..55b0bcdcef 100644 --- a/source/tools/rlclient/python/tests/test_evaluate.py +++ b/source/tools/rlclient/python/tests/test_evaluate.py @@ -5,10 +5,10 @@ import zero_ad game = zero_ad.ZeroAD("http://localhost:6000") scriptdir = path.dirname(path.realpath(__file__)) -with open(path.join(scriptdir, "..", "samples", "arcadia.json")) as f: +with open(path.join(scriptdir, "..", "samples", "arcadia.json"), encoding="utf-8") as f: config = f.read() -with open(path.join(scriptdir, "fastactions.js")) as f: +with open(path.join(scriptdir, "fastactions.js"), encoding="utf-8") as f: fastactions = f.read() diff --git a/source/tools/spirv/compile.py b/source/tools/spirv/compile.py index 3b27ffcf1d..386c312ec2 100755 --- a/source/tools/spirv/compile.py +++ b/source/tools/spirv/compile.py @@ -294,7 +294,7 @@ def compile_and_reflect(input_mod_path, dependencies, stage, path, out_path, def def output_xml_tree(tree, path): """We use a simple custom printer to have the same output for all platforms.""" - with open(path, "w") as handle: + with open(path, "w", encoding="utf-8") as handle: handle.write('\n') handle.write(f"\n") @@ -555,7 +555,7 @@ def run(): if not os.path.isfile(args.rules_path): sys.stderr.write(f'Rules "{args.rules_path}" are not found\n') return - with open(args.rules_path) as handle: + with open(args.rules_path, encoding="utf-8") as handle: rules = json.load(handle) if not os.path.isdir(args.input_mod_path): diff --git a/source/tools/templatesanalyzer/unitTables.py b/source/tools/templatesanalyzer/unitTables.py index 1f365d76eb..dbda80ed65 100755 --- a/source/tools/templatesanalyzer/unitTables.py +++ b/source/tools/templatesanalyzer/unitTables.py @@ -594,6 +594,7 @@ def writeHTML(): f = open( os.path.realpath(__file__).replace("unitTables.py", "") + "unit_summary_table.html", "w", + encoding="utf-8", ) f.write( @@ -680,7 +681,11 @@ differences between the two. TemplatesByParent[parent].sort(key=lambda x: Civs.index(x[1]["Civ"])) for tp in TemplatesByParent[parent]: isChanged = False - ff = open(os.path.realpath(__file__).replace("unitTables.py", "") + ".cache", "w") + ff = open( + os.path.realpath(__file__).replace("unitTables.py", "") + ".cache", + "w", + encoding="utf-8", + ) ff.write("") ff.write( @@ -771,7 +776,10 @@ differences between the two. ff.write("\n") ff.close() # to actually write into the file - with open(os.path.realpath(__file__).replace("unitTables.py", "") + ".cache") as ff: + with open( + os.path.realpath(__file__).replace("unitTables.py", "") + ".cache", + encoding="utf-8", + ) as ff: unitStr = ff.read() if showChangedOnly: