[Gameplay] Implement a rough comparison function to sort entities within a template-determined percent error:https://wildfiregames.com/forum/topic/118677-less-precise-sorting-in-range-manager/#comment-575838

As compile.py only creates shaders for program combinations which don't
have already existing shaders on disk, this removes all SPIR-V shaders
prior to rebuilding them to ensure they actually do get recreated.

.gitea/CODEOWNERS

@@ -3,6 +3,7 @@
 \\.gitea/.* @Stan @Itms
 ## Linting
 \\.pre-commit-config\\.yaml @Dunedan
+ruff\\.toml @Dunedan
 
 ## == Build & Libraries
 (build|libraries)/.* @Itms @Stan

.pre-commit-config.yaml

@@ -30,8 +30,6 @@ repos:
       - id: ruff-format
         args:
           - --check
-          - --target-version
-          - py311
         exclude: ^source/tools/webservices/
   - repo: local
     hooks:

binaries/data/mods/public/simulation/components/UnitAI.js

@@ -42,6 +42,11 @@ UnitAI.prototype.Schema =
 			"<data type='nonNegativeInteger'/>" +
 		"</element>" +
 	"</optional>" +
+	"<optional>" +
+		"<element name='RangeError'>" +
+			"<data type='nonNegativeInteger'/>" +
+		"</element>" +
+	"</optional>" +
 	"<optional>" +
 		"<interleave>" +
 			"<element name='RoamDistance'>" +
@@ -3462,6 +3467,7 @@ UnitAI.prototype.Init = function()
 
 	this.formationAnimationVariant = undefined;
 	this.cheeringTime = +(this.template.CheeringTime || 0);
+	this.rangeError = +(this.template.RangeError || 0);
 	this.SetStance(this.template.DefaultStance);
 };
 
@@ -3776,7 +3782,7 @@ UnitAI.prototype.SetupLOSRangeQuery = function(enable = true)
 	// Do not compensate for entity sizes: LOS doesn't, and UnitAI relies on that.
 	this.losRangeQuery = cmpRangeManager.CreateActiveQuery(this.entity,
 		range.min, range.max, players, IID_Identity,
-		cmpRangeManager.GetEntityFlagMask("normal"), false);
+		cmpRangeManager.GetEntityFlagMask("normal"), false, this.rangeError);
 
 	if (enable)
 		cmpRangeManager.EnableActiveQuery(this.losRangeQuery);
@@ -3841,7 +3847,7 @@ UnitAI.prototype.SetupAttackRangeQuery = function(enable = true)
 	// Do not compensate for entity sizes: LOS doesn't, and UnitAI relies on that.
 	this.losAttackRangeQuery = cmpRangeManager.CreateActiveQuery(this.entity,
 		range.min, range.max, players, IID_Resistance,
-		cmpRangeManager.GetEntityFlagMask("normal"), false);
+		cmpRangeManager.GetEntityFlagMask("normal"), false, this.rangeError);
 
 	if (enable)
 		cmpRangeManager.EnableActiveQuery(this.losAttackRangeQuery);

binaries/data/mods/public/simulation/templates/template_unit.xml

@@ -109,6 +109,7 @@
     <CanPatrol>true</CanPatrol>
     <PatrolWaitTime>1</PatrolWaitTime>
     <CheeringTime>2800</CheeringTime>
+	<RangeError>10</RangeError>
     <Formations datatype="tokens">
       special/formations/null
       special/formations/box

binaries/data/mods/public/simulation/templates/template_unit_champion_infantry_crossbowman.xml

@@ -42,6 +42,7 @@
   </Sound>
   <Turretable/>
   <UnitAI>
+	<RangeError>15</RangeError>
     <Formations datatype="tokens">
       special/formations/skirmish
     </Formations>

binaries/data/mods/public/simulation/templates/template_unit_infantry_ranged_crossbowman.xml

@@ -45,6 +45,9 @@
       <attack_ranged>attack/weapon/bow_attack.xml</attack_ranged>
     </SoundGroups>
   </Sound>
+  <UnitAI>
+	  <RangeError>20</RangeError>
+  </UnitAI>
   <UnitMotion>
     <WalkSpeed op="add">0.6</WalkSpeed>
     <Acceleration op="add">1.2</Acceleration>

binaries/data/mods/public/simulation/templates/template_unit_siege_boltshooter.xml

@@ -67,6 +67,7 @@
     </SoundGroups>
   </Sound>
   <UnitAI>
+	<RangeError>3</RangeError>
     <DefaultStance>standground</DefaultStance>
   </UnitAI>
   <UnitMotion>

build/jenkins/pipelines/nightly-build.Jenkinsfile

@@ -53,7 +53,10 @@ pipeline {
 
 		stage("Check for shader changes") {
 			when {
-				changeset 'binaries/data/mods/**/shaders/**/*.xml'
+				anyOf {
+					changeset 'binaries/data/mods/**/shaders/**/*.xml'
+					changeset 'source/tools/spirv/compile.py'
+				}
 			}
 			steps {
 				script { buildSPIRV = true }
@@ -124,6 +127,8 @@ pipeline {
 			}
 			steps {
 				ws("workspace/nightly-svn") {
+					bat "del /s /q binaries\\data\\mods\\mod\\shaders\\spirv"
+					bat "del /s /q binaries\\data\\mods\\public\\shaders\\spirv"
 					bat "python source/tools/spirv/compile.py -d binaries/data/mods/mod binaries/data/mods/mod source/tools/spirv/rules.json binaries/data/mods/mod"
 					bat "python source/tools/spirv/compile.py -d binaries/data/mods/mod binaries/data/mods/public source/tools/spirv/rules.json binaries/data/mods/public"
 				}

libraries/build-macos-libs.sh

@@ -811,6 +811,7 @@ GNUTLS_DIR="$(pwd)/gnutls"
 				HOGWEED_LIBS="-L${NETTLE_DIR}/lib -lhogweed" \
 				GMP_CFLAGS="-I${GMP_DIR}/include" \
 				GMP_LIBS="-L${GMP_DIR}/lib -lgmp" \
+				"$HOST_PLATFORM" \
 				--prefix="$INSTALL_DIR" \
 				--enable-shared=no \
 				--without-idn \
@@ -818,8 +819,10 @@ GNUTLS_DIR="$(pwd)/gnutls"
 				--with-included-libtasn1 \
 				--without-p11-kit \
 				--without-brotli \
+				--without-zstd \
+				--without-tpm2 \
+				--disable-libdane \
 				--disable-tests \
-				--disable-guile \
 				--disable-doc \
 				--disable-tools \
 				--disable-nls

ruff.toml

@@ -1,5 +1,7 @@
 line-length = 99
 
+target-version = "py311"
+
 [format]
 line-ending = "lf"
 
@@ -10,36 +12,29 @@ ignore = [
     "C90",
     "COM812",
     "D10",
-    "DTZ005",
     "EM",
     "FA",
     "FIX",
     "FBT",
     "ISC001",
     "N817",
-    "PERF203",
     "PERF401",
     "PLR0912",
-    "PLR0913",
     "PLR0915",
     "PLR2004",
     "PLW2901",
     "PT",
     "PTH",
-    "RUF012",
     "S101",
     "S310",
     "S314",
-    "S324",
     "S320",
     "S603",
     "S607",
     "T20",
     "TD002",
     "TD003",
-    "TRY002",
     "TRY003",
-    "TRY004",
     "UP038",
     "W505"
 ]
@@ -52,3 +47,6 @@ max-doc-length = 72
 
 [lint.pydocstyle]
 convention = "pep257"
+
+[lint.pylint]
+max-args = 8

source/maths/FixedVector2D.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2020 Wildfire Games.
+/* Copyright (C) 2024 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -169,6 +169,29 @@ public:
 		return 0;
 	}
 
+	/**
+	 * Returns -1, 0, +1 depending on whether length estimate is less/equal/greater
+	 * than the argument's length.
+	 * Uses a percent error parameter to compare lengths with that percent error.
+	 */
+	int CompareLengthRough(const CFixedVector2D& other, u8 rangeError) const
+	{
+		u64 d2 = SQUARE_U64_FIXED(X) + SQUARE_U64_FIXED(Y);
+		u64 od2 = SQUARE_U64_FIXED(other.X) + SQUARE_U64_FIXED(other.Y);
+
+		//overflow risk with long ranges (designed for unit ranges)
+		CheckMultiplicationOverflow(u64, d2, 100+rangeError, "Overflow in CFixedVector2D::CompareLengthRough()","Underflow in CFixedVector2D::CompareLengthRough()")
+		d2 = d2 * (100-rangeError + d2 % (1+(rangeError*2)));
+		CheckMultiplicationOverflow(u64, od2, 100, "Overflow in CFixedVector2D::CompareLengthRough()", "Underflow in CFixedVector2D::CompareLengthRough()")
+		od2 = od2 * 100;
+
+		if (d2 < od2)
+			return -1;
+		if (d2 > od2)
+			return +1;
+		return 0;
+	}
+
 	bool IsZero() const
 	{
 		return X.IsZero() && Y.IsZero();

source/simulation2/components/CCmpRangeManager.cpp

@@ -1,4 +1,4 @@
-/* Copyright (C) 2022 Wildfire Games.
+/* Copyright (C) 2024 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -162,6 +162,7 @@ struct Query
 	u32 ownersMask;
 	i32 interface;
 	u8 flagsMask;
+	u8 rangeError;
 	bool enabled;
 	bool parabolic;
 	bool accountForSize; // If true, the query accounts for unit sizes, otherwise it treats all entities as points.
@@ -255,8 +256,8 @@ static_assert(sizeof(EntityData) == 24);
 class EntityDistanceOrdering
 {
 public:
-	EntityDistanceOrdering(const EntityMap<EntityData>& entities, const CFixedVector2D& source) :
-		m_EntityData(entities), m_Source(source)
+	EntityDistanceOrdering(const EntityMap<EntityData>& entities, const CFixedVector2D& source, u8 rangeError = 0) :
+		m_EntityData(entities), m_Source(source), m_RangeError(rangeError)
 	{
 	}
 
@@ -268,11 +269,12 @@ public:
 		const EntityData& db = m_EntityData.find(b)->second;
 		CFixedVector2D vecA = CFixedVector2D(da.x, da.z) - m_Source;
 		CFixedVector2D vecB = CFixedVector2D(db.x, db.z) - m_Source;
-		return (vecA.CompareLength(vecB) < 0);
+		return m_RangeError > 0 ? vecA.CompareLengthRough(vecB, m_RangeError) < 0 : vecA.CompareLength(vecB) < 0;
 	}
 
 	const EntityMap<EntityData>& m_EntityData;
 	CFixedVector2D m_Source;
+	u8 m_RangeError;
 
 private:
 	EntityDistanceOrdering& operator=(const EntityDistanceOrdering&);
@@ -294,6 +296,7 @@ struct SerializeHelper<Query>
 		serialize.NumberU32_Unbounded("owners mask", value.ownersMask);
 		serialize.NumberI32_Unbounded("interface", value.interface);
 		Serializer(serialize, "last match", value.lastMatch);
+		serialize.NumberU8_Unbounded("range percent error", value.rangeError);
 		serialize.NumberU8_Unbounded("flagsMask", value.flagsMask);
 		serialize.Bool("enabled", value.enabled);
 		serialize.Bool("parabolic",value.parabolic);
@@ -923,20 +926,20 @@ public:
 
 	tag_t CreateActiveQuery(entity_id_t source,
 		entity_pos_t minRange, entity_pos_t maxRange,
-		const std::vector<int>& owners, int requiredInterface, u8 flags, bool accountForSize) override
+		const std::vector<int>& owners, int requiredInterface, u8 flags, bool accountForSize, u8 rangeError) override
 	{
 		tag_t id = m_QueryNext++;
-		m_Queries[id] = ConstructQuery(source, minRange, maxRange, owners, requiredInterface, flags, accountForSize);
+		m_Queries[id] = ConstructQuery(source, minRange, maxRange, owners, requiredInterface, flags, accountForSize, rangeError);
 
 		return id;
 	}
 
 	tag_t CreateActiveParabolicQuery(entity_id_t source,
 		entity_pos_t minRange, entity_pos_t maxRange, entity_pos_t yOrigin,
-		const std::vector<int>& owners, int requiredInterface, u8 flags) override
+		const std::vector<int>& owners, int requiredInterface, u8 flags, u8 rangeError) override
 	{
 		tag_t id = m_QueryNext++;
-		m_Queries[id] = ConstructParabolicQuery(source, minRange, maxRange, yOrigin, owners, requiredInterface, flags, true);
+		m_Queries[id] = ConstructParabolicQuery(source, minRange, maxRange, yOrigin, owners, requiredInterface, flags, true, rangeError);
 
 		return id;
 	}
@@ -993,25 +996,25 @@ public:
 
 	std::vector<entity_id_t> ExecuteQueryAroundPos(const CFixedVector2D& pos,
 		entity_pos_t minRange, entity_pos_t maxRange,
-		const std::vector<int>& owners, int requiredInterface, bool accountForSize) override
+		const std::vector<int>& owners, int requiredInterface, bool accountForSize, u8 rangeError) override
 	{
-		Query q = ConstructQuery(INVALID_ENTITY, minRange, maxRange, owners, requiredInterface, GetEntityFlagMask("normal"), accountForSize);
+		Query q = ConstructQuery(INVALID_ENTITY, minRange, maxRange, owners, requiredInterface, GetEntityFlagMask("normal"), accountForSize, rangeError);
 		std::vector<entity_id_t> r;
 		PerformQuery(q, r, pos);
 
 		// Return the list sorted by distance from the entity
-		std::stable_sort(r.begin(), r.end(), EntityDistanceOrdering(m_EntityData, pos));
+		std::stable_sort(r.begin(), r.end(), EntityDistanceOrdering(m_EntityData, pos, q.rangeError));
 
 		return r;
 	}
 
 	std::vector<entity_id_t> ExecuteQuery(entity_id_t source,
 		entity_pos_t minRange, entity_pos_t maxRange,
-		const std::vector<int>& owners, int requiredInterface, bool accountForSize) override
+		const std::vector<int>& owners, int requiredInterface, bool accountForSize, u8 rangeError) override
 	{
 		PROFILE("ExecuteQuery");
 
-		Query q = ConstructQuery(source, minRange, maxRange, owners, requiredInterface, GetEntityFlagMask("normal"), accountForSize);
+		Query q = ConstructQuery(source, minRange, maxRange, owners, requiredInterface, GetEntityFlagMask("normal"), accountForSize, rangeError);
 
 		std::vector<entity_id_t> r;
 
@@ -1026,7 +1029,7 @@ public:
 		PerformQuery(q, r, pos);
 
 		// Return the list sorted by distance from the entity
-		std::stable_sort(r.begin(), r.end(), EntityDistanceOrdering(m_EntityData, pos));
+		std::stable_sort(r.begin(), r.end(), EntityDistanceOrdering(m_EntityData, pos, q.rangeError));
 
 		return r;
 	}
@@ -1061,7 +1064,7 @@ public:
 		q.lastMatch = r;
 
 		// Return the list sorted by distance from the entity
-		std::stable_sort(r.begin(), r.end(), EntityDistanceOrdering(m_EntityData, pos));
+		std::stable_sort(r.begin(), r.end(), EntityDistanceOrdering(m_EntityData, pos, q.rangeError));
 
 		return r;
 	}
@@ -1145,7 +1148,7 @@ public:
 				continue;
 
 			if (cmpSourcePosition && cmpSourcePosition->IsInWorld())
-				std::stable_sort(added.begin(), added.end(), EntityDistanceOrdering(m_EntityData, cmpSourcePosition->GetPosition2D()));
+				std::stable_sort(added.begin(), added.end(), EntityDistanceOrdering(m_EntityData, cmpSourcePosition->GetPosition2D(),query.rangeError));
 
 			messages.resize(messages.size() + 1);
 			std::pair<entity_id_t, CMessageRangeUpdate>& back = messages.back();
@@ -1404,7 +1407,7 @@ public:
 
 	Query ConstructQuery(entity_id_t source,
 		entity_pos_t minRange, entity_pos_t maxRange,
-		const std::vector<int>& owners, int requiredInterface, u8 flagsMask, bool accountForSize) const
+		const std::vector<int>& owners, int requiredInterface, u8 flagsMask, bool accountForSize, u8 rangeError = 0) const
 	{
 		// Min range must be non-negative.
 		if (minRange < entity_pos_t::Zero())
@@ -1453,6 +1456,7 @@ public:
 			LOGWARNING("CCmpRangeManager: No owners in query for entity %u", source);
 
 		q.interface = requiredInterface;
+		q.rangeError = rangeError;
 		q.flagsMask = flagsMask;
 
 		return q;
@@ -1460,9 +1464,9 @@ public:
 
 	Query ConstructParabolicQuery(entity_id_t source,
 		entity_pos_t minRange, entity_pos_t maxRange, entity_pos_t yOrigin,
-		const std::vector<int>& owners, int requiredInterface, u8 flagsMask, bool accountForSize) const
+		const std::vector<int>& owners, int requiredInterface, u8 flagsMask, bool accountForSize, u8 rangeError = 0) const
 	{
-		Query q = ConstructQuery(source, minRange, maxRange, owners, requiredInterface, flagsMask, accountForSize);
+		Query q = ConstructQuery(source, minRange, maxRange, owners, requiredInterface, flagsMask, accountForSize, rangeError);
 		q.parabolic = true;
 		q.yOrigin = yOrigin;
 		return q;

source/simulation2/components/ICmpRangeManager.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2022 Wildfire Games.
+/* Copyright (C) 2024 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -132,7 +132,7 @@ public:
 	 * @return list of entities matching the query, ordered by increasing distance from the source entity.
 	 */
 	virtual std::vector<entity_id_t> ExecuteQuery(entity_id_t source, entity_pos_t minRange, entity_pos_t maxRange,
-		const std::vector<int>& owners, int requiredInterface, bool accountForSize) = 0;
+		const std::vector<int>& owners, int requiredInterface, bool accountForSize, u8 rangeError = 0) = 0;
 
 	/**
 	 * Execute a passive query.
@@ -145,7 +145,7 @@ public:
 	 * @return list of entities matching the query, ordered by increasing distance from the source entity.
 	 */
 	virtual std::vector<entity_id_t> ExecuteQueryAroundPos(const CFixedVector2D& pos, entity_pos_t minRange, entity_pos_t maxRange,
-		const std::vector<int>& owners, int requiredInterface, bool accountForSize) = 0;
+		const std::vector<int>& owners, int requiredInterface, bool accountForSize, u8 rangeError = 0) = 0;
 
 	/**
 	 * Construct an active query. The query will be disabled by default.
@@ -159,7 +159,7 @@ public:
 	 * @return unique non-zero identifier of query.
 	 */
 	virtual tag_t CreateActiveQuery(entity_id_t source, entity_pos_t minRange, entity_pos_t maxRange,
-		const std::vector<int>& owners, int requiredInterface, u8 flags, bool accountForSize) = 0;
+		const std::vector<int>& owners, int requiredInterface, u8 flags, bool accountForSize, u8 rangeError = 0) = 0;
 
     /**
 	 * Construct an active query of a paraboloic form around the unit.
@@ -177,7 +177,7 @@ public:
 	 * @return unique non-zero identifier of query.
 	 */
 	virtual tag_t CreateActiveParabolicQuery(entity_id_t source, entity_pos_t minRange, entity_pos_t maxRange, entity_pos_t yOrigin,
-		const std::vector<int>& owners, int requiredInterface, u8 flags) = 0;
+		const std::vector<int>& owners, int requiredInterface, u8 flags, u8 rangeError = 0) = 0;
 
 
 	/**

source/tools/entity/checkrefs.py

@@ -8,7 +8,6 @@ from json import load, loads
 from logging import INFO, WARNING, Filter, Formatter, StreamHandler, getLogger
 from pathlib import Path
 from struct import calcsize, unpack
-from typing import Dict, List, Set, Tuple
 from xml.etree import ElementTree as ET
 
 from scriptlib import SimulTemplateEntity, find_files
@@ -27,9 +26,9 @@ class SingleLevelFilter(Filter):
 
 class CheckRefs:
     def __init__(self):
-        self.files: List[Path] = []
-        self.roots: List[Path] = []
-        self.deps: List[Tuple[Path, Path]] = []
+        self.files: list[Path] = []
+        self.roots: list[Path] = []
+        self.deps: list[tuple[Path, Path]] = []
         self.vfs_root = Path(__file__).resolve().parents[3] / "binaries" / "data" / "mods"
         self.supportedTextureFormats = ("dds", "png")
         self.supportedMeshesFormats = ("pmd", "dae")
@@ -794,7 +793,7 @@ class CheckRefs:
         uniq_files = {r.as_posix() for r in self.files}
         lower_case_files = {f.lower(): f for f in uniq_files}
 
-        missing_files: Dict[str, Set[str]] = defaultdict(set)
+        missing_files: dict[str, set[str]] = defaultdict(set)
 
         for parent, dep in self.deps:
             dep_str = dep.as_posix()

source/tools/entity/entvalidate.py

@@ -7,7 +7,6 @@ import shutil
 import sys
 from pathlib import Path
 from subprocess import CalledProcessError, run
-from typing import Sequence
 from xml.etree import ElementTree as ET
 
 from scriptlib import SimulTemplateEntity, find_files
@@ -50,7 +49,7 @@ errorch.setFormatter(logging.Formatter("%(levelname)s - %(message)s"))
 logger.addHandler(errorch)
 
 
-def main(argv: Sequence[str] | None = None) -> int:
+def main() -> int:
     parser = argparse.ArgumentParser(description="Validate templates")
     parser.add_argument("-m", "--mod-name", required=True, help="The name of the mod to validate.")
     parser.add_argument(
@@ -73,7 +72,7 @@ def main(argv: Sequence[str] | None = None) -> int:
     )
     parser.add_argument("-v", "--verbose", help="Be verbose about the output.", default=False)
 
-    args = parser.parse_args(argv)
+    args = parser.parse_args()
 
     if not args.relaxng_schema.exists():
         logging.error(RELAXNG_SCHEMA_ERROR_MSG.format(args.relaxng_schema))

source/tools/fontbuilder2/font_loader.py

@@ -1,7 +1,10 @@
 # Adapted from http://cairographics.org/freetypepython/
 
+# ruff: noqa: TRY002
+
 import ctypes
 import sys
+from typing import ClassVar
 
 import cairo
 
@@ -40,7 +43,7 @@ _surface = cairo.ImageSurface(cairo.FORMAT_A8, 0, 0)
 
 
 class PycairoContext(ctypes.Structure):
-    _fields_ = [
+    _fields_: ClassVar = [
         ("PyObject_HEAD", ctypes.c_byte * object.__basicsize__),
         ("ctx", ctypes.c_void_p),
         ("base", ctypes.c_void_p),

source/tools/i18n/check_diff.py

@@ -22,7 +22,6 @@ import io
 import os
 import subprocess
 from itertools import islice
-from typing import List
 
 from i18n_helper import PROJECT_ROOT_DIRECTORY
 
@@ -38,7 +37,7 @@ def get_diff():
     return io.StringIO(diff_process.stdout.decode("utf-8"))
 
 
-def check_diff(diff: io.StringIO) -> List[str]:
+def check_diff(diff: io.StringIO) -> list[str]:
     """Check a diff of .po files for meaningful changes.
 
     Run through a diff of .po files and check that some of the changes
@@ -87,7 +86,7 @@ def check_diff(diff: io.StringIO) -> List[str]:
     return list(files.difference(keep))
 
 
-def revert_files(files: List[str], verbose=False):
+def revert_files(files: list[str], verbose=False):
     def batched(iterable, n):
         """Split an iterable in equally sized chunks.
 

source/tools/i18n/clean_translation_files.py

@@ -27,51 +27,64 @@ However that needs to be fixed on the transifex side, see rP25896. For now
 strip the e-mails using this script.
 """
 
-import fileinput
 import glob
 import os
 import re
-import sys
 
 from i18n_helper import L10N_FOLDER_NAME, PROJECT_ROOT_DIRECTORY, TRANSIFEX_CLIENT_FOLDER
 
 
+TRANSLATOR_REGEX = re.compile(r"^(#\s+[^,<]*)\s+<.*>(.*)$")
+LAST_TRANSLATION_REGEX = re.compile(r"^(\"Last-Translator:[^,<]*)\s+<.*>(.*)$")
+
+
 def main():
-    translator_match = re.compile(r"^(#\s+[^,<]*)\s+<.*>(.*)")
-    last_translator_match = re.compile(r"^(\"Last-Translator:[^,<]*)\s+<.*>(.*)")
+    for folder in glob.iglob(
+        f"**/{L10N_FOLDER_NAME}/{TRANSIFEX_CLIENT_FOLDER}/",
+        root_dir=PROJECT_ROOT_DIRECTORY,
+        recursive=True,
+    ):
+        for file in glob.iglob(
+            f"{os.path.join(folder, os.pardir)}/*.po", root_dir=PROJECT_ROOT_DIRECTORY
+        ):
+            absolute_file_path = os.path.abspath(f"{PROJECT_ROOT_DIRECTORY}/{file}")
 
-    for root, folders, _ in os.walk(PROJECT_ROOT_DIRECTORY):
-        for folder in folders:
-            if folder != L10N_FOLDER_NAME:
-                continue
-
-            if not os.path.exists(os.path.join(root, folder, TRANSIFEX_CLIENT_FOLDER)):
-                continue
-
-            path = os.path.join(root, folder, "*.po")
-            files = glob.glob(path)
-            for file in files:
-                usernames = []
-                reached = False
-                for line in fileinput.input(
-                    file.replace("\\", "/"), inplace=True, encoding="utf-8"
-                ):
-                    if reached:
+            file_content = []
+            usernames = []
+            changes = False
+            in_translators = False
+            found_last_translator = False
+            with open(absolute_file_path, "r+", encoding="utf-8") as fd:
+                for line in fd:
+                    if line.strip() == "# Translators:":
+                        in_translators = True
+                    elif not line.strip().startswith("#"):
+                        in_translators = False
+                    elif in_translators:
                         if line == "# \n":
-                            line = ""
-                        m = translator_match.match(line)
-                        if m:
-                            if m.group(1) in usernames:
-                                line = ""
-                            else:
-                                line = m.group(1) + m.group(2) + "\n"
-                                usernames.append(m.group(1))
-                        m2 = last_translator_match.match(line)
-                        if m2:
-                            line = re.sub(last_translator_match, r"\1\2", line)
-                    elif line.strip() == "# Translators:":
-                        reached = True
-                    sys.stdout.write(line)
+                            changes = True
+                            continue
+                        translator_match = TRANSLATOR_REGEX.match(line)
+                        if translator_match:
+                            changes = True
+                            if translator_match.group(1) in usernames:
+                                continue
+                            line = TRANSLATOR_REGEX.sub(r"\1\2", line)
+                            usernames.append(translator_match.group(1))
+
+                    if not in_translators and not found_last_translator:
+                        last_translator_match = LAST_TRANSLATION_REGEX.match(line)
+                        if last_translator_match:
+                            found_last_translator = True
+                            changes = True
+                            line = LAST_TRANSLATION_REGEX.sub(r"\1\2", line)
+
+                    file_content.append(line)
+
+                if changes:
+                    fd.seek(0)
+                    fd.truncate()
+                    fd.writelines(file_content)
 
 
 if __name__ == "__main__":

source/tools/i18n/i18n_helper/catalog.py

@@ -1,6 +1,6 @@
 """Wrapper around babel Catalog / .po handling."""
 
-from datetime import datetime
+from datetime import UTC, datetime
 
 from babel.messages.catalog import Catalog as BabelCatalog
 from babel.messages.pofile import read_po, write_po
@@ -10,7 +10,7 @@ class Catalog(BabelCatalog):
     """Wraps a BabelCatalog for convenience."""
 
     def __init__(self, *args, project=None, copyright_holder=None, **other_kwargs):
-        date = datetime.now()
+        date = datetime.now(tz=UTC)
         super().__init__(
             *args,
             header_comment=(

source/tools/i18n/i18n_helper/globber.py

@@ -1,12 +1,11 @@
 """Utils to list .po."""
 
 import os
-from typing import List, Optional
 
 from i18n_helper.catalog import Catalog
 
 
-def get_catalogs(input_file_path, filters: Optional[List[str]] = None) -> List[Catalog]:
+def get_catalogs(input_file_path, filters: list[str] | None = None) -> list[Catalog]:
     """Return a list of "real" catalogs (.po) in the given folder."""
     existing_translation_catalogs = []
     l10n_folder_path = os.path.dirname(input_file_path)

source/tools/i18n/tests/test_check_diff.py

@@ -97,7 +97,7 @@ PATCHES_EXPECT_REVERT = [
 ]
 
 
-@pytest.fixture(params=zip(PATCHES, PATCHES_EXPECT_REVERT))
+@pytest.fixture(params=zip(PATCHES, PATCHES_EXPECT_REVERT, strict=False))
 def patch(request):
     return [io.StringIO(request.param[0]), request.param[1]]
 

source/tools/rlclient/python/samples/simple_example.py

@@ -7,11 +7,11 @@ import zero_ad
 
 
 def dist(p1, p2):
-    return math.sqrt(sum(math.pow(x2 - x1, 2) for (x1, x2) in zip(p1, p2)))
+    return math.sqrt(sum(math.pow(x2 - x1, 2) for (x1, x2) in zip(p1, p2, strict=False)))
 
 
 def center(units):
-    sum_position = map(sum, zip(*(u.position() for u in units)))
+    sum_position = map(sum, zip(*(u.position() for u in units), strict=False))
     return [x / len(units) for x in sum_position]
 
 

source/tools/rlclient/python/tests/test_actions.py

@@ -11,11 +11,11 @@ with open(path.join(scriptdir, "..", "samples", "arcadia.json"), encoding="utf-8
 
 
 def dist(p1, p2):
-    return math.sqrt(sum(math.pow(x2 - x1, 2) for (x1, x2) in zip(p1, p2)))
+    return math.sqrt(sum(math.pow(x2 - x1, 2) for (x1, x2) in zip(p1, p2, strict=False)))
 
 
 def center(units):
-    sum_position = map(sum, zip(*(u.position() for u in units)))
+    sum_position = map(sum, zip(*(u.position() for u in units), strict=False))
     return [x / len(units) for x in sum_position]
 
 

source/tools/rlclient/python/zero_ad/api.py

@@ -26,7 +26,7 @@ class RLAPI:
     def get_templates(self, names):
         post_data = "\n".join(names)
         response = self.post("templates", post_data)
-        return zip(names, response.decode().split("\n"))
+        return zip(names, response.decode().split("\n"), strict=False)
 
     def evaluate(self, code):
         response = self.post("evaluate", code)

source/tools/rlclient/python/zero_ad/environment.py

@@ -17,7 +17,7 @@ class ZeroAD:
             actions = []
         player_ids = cycle([self.player_id]) if player is None else cycle(player)
 
-        cmds = zip(player_ids, actions)
+        cmds = zip(player_ids, actions, strict=False)
         cmds = ((player, action) for (player, action) in cmds if action is not None)
         state_json = self.api.step(cmds)
         self.current_state = GameState(json.loads(state_json), self)

source/tools/spirv/compile.py

@@ -46,15 +46,7 @@ def execute(command):
 def calculate_hash(path):
     assert os.path.isfile(path)
     with open(path, "rb") as handle:
-        return hashlib.sha1(handle.read()).hexdigest()
-
-
-def compare_spirv(path1, path2):
-    with open(path1, "rb") as handle:
-        spirv1 = handle.read()
-    with open(path2, "rb") as handle:
-        spirv2 = handle.read()
-    return spirv1 == spirv2
+        return hashlib.sha256(handle.read()).hexdigest()
 
 
 def resolve_if(defines, expression):
@@ -456,19 +448,10 @@ def build(rules, input_mod_path, output_mod_path, dependencies, program_name):
 
             spirv_hash = calculate_hash(output_spirv_path)
             if spirv_hash not in hashed_cache:
-                hashed_cache[spirv_hash] = [file_name]
+                hashed_cache[spirv_hash] = file_name
             else:
-                found_candidate = False
-                for candidate_name in hashed_cache[spirv_hash]:
-                    candidate_path = os.path.join(output_spirv_mod_path, candidate_name)
-                    if compare_spirv(output_spirv_path, candidate_path):
-                        found_candidate = True
-                        file_name = candidate_name
-                        break
-                if found_candidate:
-                    os.remove(output_spirv_path)
-                else:
-                    hashed_cache[spirv_hash].append(file_name)
+                file_name = hashed_cache[spirv_hash]
+                os.remove(output_spirv_path)
 
             shader_element = ET.SubElement(program_root, shader["type"])
             shader_element.set("file", "spirv/" + file_name)

source/tools/xmlvalidator/validate_grammar.py

@@ -201,7 +201,7 @@ class RelaxNGValidator:
             try:
                 doc = lxml.etree.parse(str(file[1]))
                 relaxng.assertValid(doc)
-            except Exception:
+            except (lxml.etree.DocumentInvalid, lxml.etree.XMLSyntaxError):
                 error_count = error_count + 1
                 self.logger.exception(file[1])