From 42672442b95fe82c2a0d02b99bb5fe9c7f1db926 Mon Sep 17 00:00:00 2001 From: Dunedan Date: Sun, 22 Sep 2024 10:56:05 +0200 Subject: [PATCH] Make SPIR-V file names reproducible This ensures the file names of SPIR-V program combinations and shaders are reproducible. Up to now they only were if the order of program combinations in the rules.json didn't change, as the file names contained the position of the program combination in the rules.json. With this change files names of program combinations will be named based on the details of the combination used to create them and the file names of shaders will be based on their content respectively. Changing the file names avoids wrong shaders when partially rebuilding them after a new combination for a program got added in between the other combinations in rules.json and removes the need for keeping track of identical shaders in the script. It's also a preparation for being able to build shaders in parallel, while also keeping the result reproducible. --- source/tools/spirv/compile.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/source/tools/spirv/compile.py b/source/tools/spirv/compile.py index a2b4e67743..9759c41bb4 100755 --- a/source/tools/spirv/compile.py +++ b/source/tools/spirv/compile.py @@ -49,6 +49,12 @@ def calculate_hash(path): return hashlib.sha256(handle.read()).hexdigest() +def get_combination_hash(combination: list[dict[str, str]]) -> str: + """Turn combination information into a unique hash.""" + hashable_combination = tuple([tuple(sorted(i.items())) for i in combination]) + return hashlib.sha256(json.dumps(hashable_combination, sort_keys=True).encode()).hexdigest() + + def resolve_if(defines, expression): for item in expression.strip().split("||"): item = item.strip() @@ -398,11 +404,10 @@ def build(rules, input_mod_path, output_mod_path, dependencies, program_name): else: combinations = list(itertools.product(*defines)) - hashed_cache = {} + for combination in combinations: + combination_hash = get_combination_hash(combination) - for index, combination in enumerate(combinations): - assert index < 10000 - program_path = "spirv/" + program_name + ("_%04d" % index) + ".xml" + program_path = f"spirv/{program_name}_{combination_hash}.xml" programs_element = ET.SubElement(root, "program") programs_element.set("type", "spirv") @@ -423,8 +428,8 @@ def build(rules, input_mod_path, output_mod_path, dependencies, program_name): program_root.set("type", "spirv") for shader in shaders: extension = stage_extension[shader["type"]] - file_name = program_name + ("_%04d" % index) + extension + ".spv" - output_spirv_path = os.path.join(output_spirv_mod_path, file_name) + tmp_file_name = f"{program_name}_{combination_hash}{extension}.spv" + tmp_output_spirv_path = os.path.join(output_spirv_mod_path, tmp_file_name) input_glsl_path = os.path.join(input_mod_path, "shaders", shader["file"]) # Some shader programs might use vs and fs shaders from different mods. @@ -442,16 +447,20 @@ def build(rules, input_mod_path, output_mod_path, dependencies, program_name): dependencies, shader["type"], input_glsl_path, - output_spirv_path, + tmp_output_spirv_path, combination + program_defines, ) - spirv_hash = calculate_hash(output_spirv_path) - if spirv_hash not in hashed_cache: - hashed_cache[spirv_hash] = file_name + spirv_hash = calculate_hash(tmp_output_spirv_path) + file_name = f"{program_name}_{spirv_hash}{extension}.spv" + output_spirv_path = os.path.join(output_spirv_mod_path, file_name) + if not os.path.exists(output_spirv_path): + try: + os.rename(tmp_output_spirv_path, output_spirv_path) + except FileExistsError: + os.remove(tmp_output_spirv_path) else: - file_name = hashed_cache[spirv_hash] - os.remove(output_spirv_path) + os.remove(tmp_output_spirv_path) shader_element = ET.SubElement(program_root, shader["type"]) shader_element.set("file", "spirv/" + file_name)