Use UTF-8 as encoding when working with files

This explicitly uses UTF-8 encoding when reading or writing files with
Python. This is necessary as the default locale varies between
operating systems.

source/collada/tests/tests.py

@@ -31,7 +31,7 @@ def log(severity, message):
 clog = CFUNCTYPE(None, c_int, c_char_p)(log)
 # (the CFUNCTYPE must not be GC'd, so try to keep a reference)
 library.set_logger(clog)
-with open(f"{binaries}/data/tests/collada/skeletons.xml") as fd:
+with open(f"{binaries}/data/tests/collada/skeletons.xml", encoding="utf-8") as fd:
     skeleton_definitions = fd.read()
 library.set_skeleton_definitions(skeleton_definitions, len(skeleton_definitions))
 
@@ -127,7 +127,10 @@ for test_file in ["xsitest3c", "xsitest3e", "jav2d", "jav2d2"]:
     input_filename = f"{test_data}/{test_file}.dae"
     output_filename = f"{test_mod}/art/meshes/{test_file}.pmd"
 
-    with open(input_filename) as input_fd, open(output_filename, "wb") as output_fd:
+    with (
+        open(input_filename, encoding="utf-8") as input_fd,
+        open(output_filename, "wb") as output_fd,
+    ):
         file_input = input_fd.read()
         file_output = convert_dae_to_pmd(file_input)
         output_fd.write(file_output)
@@ -143,11 +146,11 @@ for test_file in ["xsitest3c", "xsitest3e", "jav2d", "jav2d2"]:
         ],
         [("helmet", "teapot_basic_static")],
     )
-    with open(f"{test_mod}/art/actors/{test_file}.xml", "w") as fd:
+    with open(f"{test_mod}/art/actors/{test_file}.xml", "w", encoding="utf-8") as fd:
         fd.write(xml)
 
     xml = create_actor_static(test_file, "male")
-    with open(f"{test_mod}/art/actors/{test_file}_static.xml", "w") as fd:
+    with open(f"{test_mod}/art/actors/{test_file}_static.xml", "w", encoding="utf-8") as fd:
         fd.write(xml)
 
 # for test_file in ['jav2','jav2b', 'jav2d']:
@@ -158,7 +161,10 @@ for test_file in ["xsitest3c", "xsitest3e", "jav2d", "jav2d2"]:
     input_filename = f"{test_data}/{test_file}.dae"
     output_filename = f"{test_mod}/art/animation/{test_file}.psa"
 
-    with open(input_filename) as input_fd, open(output_filename, "wb") as output_fd:
+    with (
+        open(input_filename, encoding="utf-8") as input_fd,
+        open(output_filename, "wb") as output_fd,
+    ):
         file_input = input_fd.read()
         file_output = convert_dae_to_psa(file_input)
         output_fd.write(file_output)

source/tools/entity/creationgraph.py

@@ -20,7 +20,7 @@ def main():
     vfs_root = Path(__file__).resolve().parents[3] / "binaries" / "data" / "mods"
     simul_templates_path = Path("simulation/templates")
     simul_template_entity = SimulTemplateEntity(vfs_root)
-    with open("creation.dot", "w") as dot_f:
+    with open("creation.dot", "w", encoding="utf-8") as dot_f:
         dot_f.write("digraph G {\n")
         files = sorted(find_entities(vfs_root))
         for f in files:

source/tools/fontbuilder2/fontbuilder.py

@@ -86,7 +86,7 @@ class Glyph:
 
 # Load the set of characters contained in the given text file
 def load_char_list(filename):
-    with open(filename) as f:
+    with open(filename, encoding="utf-8") as f:
         chars = f.read()
     return set(chars)
 
@@ -167,7 +167,7 @@ def generate_font(outname, ttfNames, loadopts, size, renderstyle, dsizes):
         surface.write_to_png(f"{outname}.png")
 
         # Output the .fnt file with all the glyph positions etc
-        with open(f"{outname}.fnt", "w") as fnt:
+        with open(f"{outname}.fnt", "w", encoding="utf-8") as fnt:
             fnt.write("101\n")
             fnt.write("%d %d\n" % (w, h))
             fnt.write("%s\n" % ("rgba" if "colour" in renderstyle else "a"))

source/tools/i18n/extractors/extractors.py

@@ -511,7 +511,7 @@ class ini(Extractor):
         import ConfigParser
 
         config = ConfigParser.RawConfigParser()
-        with open(filepath) as fd:
+        with open(filepath, encoding="utf-8") as fd:
             config.read_file(FakeSectionHeader(fd))
         for keyword in self.keywords:
             message = config.get("root", keyword).strip('"').strip("'")

source/tools/i18n/updateTemplates.py

@@ -101,7 +101,7 @@ def generatePOT(templateSettings, rootPath):
 
 
 def generateTemplatesForMessagesFile(messagesFilePath):
-    with open(messagesFilePath) as fileObject:
+    with open(messagesFilePath, encoding="utf-8") as fileObject:
         settings = json.load(fileObject)
 
     for templateSettings in settings:

source/tools/mapcompatibility/a18_to_a19.py

@@ -126,7 +126,10 @@ for xmlFile in args.files:
         os.rename(pmpFile + "~", pmpFile)
 
     if os.path.isfile(xmlFile):
-        with open(xmlFile) as f1, open(xmlFile + "~", "w") as f2:
+        with (
+            open(xmlFile, encoding="utf-8") as f1,
+            open(xmlFile + "~", "w", encoding="utf-8") as f2,
+        ):
             data = f1.read()
 
             # bump version number (rely on how Atlas formats the XML)

source/tools/rlclient/python/samples/simple-example.py

@@ -33,7 +33,7 @@ game = zero_ad.ZeroAD("http://localhost:6000")
 # Load the Arcadia map
 samples_dir = path.dirname(path.realpath(__file__))
 scenario_config_path = path.join(samples_dir, "arcadia.json")
-with open(scenario_config_path, encoding="utf8") as f:
+with open(scenario_config_path, encoding="utf-8") as f:
     arcadia_config = f.read()
 
 state = game.reset(arcadia_config)

source/tools/rlclient/python/tests/test_actions.py

@@ -6,7 +6,7 @@ import zero_ad
 
 game = zero_ad.ZeroAD("http://localhost:6000")
 scriptdir = path.dirname(path.realpath(__file__))
-with open(path.join(scriptdir, "..", "samples", "arcadia.json"), encoding="utf8") as f:
+with open(path.join(scriptdir, "..", "samples", "arcadia.json"), encoding="utf-8") as f:
     config = f.read()
 
 

source/tools/rlclient/python/tests/test_evaluate.py

@@ -5,10 +5,10 @@ import zero_ad
 
 game = zero_ad.ZeroAD("http://localhost:6000")
 scriptdir = path.dirname(path.realpath(__file__))
-with open(path.join(scriptdir, "..", "samples", "arcadia.json")) as f:
+with open(path.join(scriptdir, "..", "samples", "arcadia.json"), encoding="utf-8") as f:
     config = f.read()
 
-with open(path.join(scriptdir, "fastactions.js")) as f:
+with open(path.join(scriptdir, "fastactions.js"), encoding="utf-8") as f:
     fastactions = f.read()
 
 

source/tools/spirv/compile.py

@@ -294,7 +294,7 @@ def compile_and_reflect(input_mod_path, dependencies, stage, path, out_path, def
 
 def output_xml_tree(tree, path):
     """We use a simple custom printer to have the same output for all platforms."""
-    with open(path, "w") as handle:
+    with open(path, "w", encoding="utf-8") as handle:
         handle.write('<?xml version="1.0" encoding="utf-8"?>\n')
         handle.write(f"<!-- DO NOT EDIT: GENERATED BY SCRIPT {os.path.basename(__file__)} -->\n")
 
@@ -555,7 +555,7 @@ def run():
     if not os.path.isfile(args.rules_path):
         sys.stderr.write(f'Rules "{args.rules_path}" are not found\n')
         return
-    with open(args.rules_path) as handle:
+    with open(args.rules_path, encoding="utf-8") as handle:
         rules = json.load(handle)
 
     if not os.path.isdir(args.input_mod_path):

source/tools/templatesanalyzer/unitTables.py

@@ -594,6 +594,7 @@ def writeHTML():
     f = open(
         os.path.realpath(__file__).replace("unitTables.py", "") + "unit_summary_table.html",
         "w",
+        encoding="utf-8",
     )
 
     f.write(
@@ -680,7 +681,11 @@ differences between the two.
         TemplatesByParent[parent].sort(key=lambda x: Civs.index(x[1]["Civ"]))
         for tp in TemplatesByParent[parent]:
             isChanged = False
-            ff = open(os.path.realpath(__file__).replace("unitTables.py", "") + ".cache", "w")
+            ff = open(
+                os.path.realpath(__file__).replace("unitTables.py", "") + ".cache",
+                "w",
+                encoding="utf-8",
+            )
 
             ff.write("<tr>")
             ff.write(
@@ -771,7 +776,10 @@ differences between the two.
             ff.write("</tr>\n")
 
             ff.close()  # to actually write into the file
-            with open(os.path.realpath(__file__).replace("unitTables.py", "") + ".cache") as ff:
+            with open(
+                os.path.realpath(__file__).replace("unitTables.py", "") + ".cache",
+                encoding="utf-8",
+            ) as ff:
                 unitStr = ff.read()
 
             if showChangedOnly: