Dunedan/0ad
1
0
Fork 0
forked from 0ad/0ad
Code Pull Requests Wiki Activity

Make SPIR-V file names reproducible

Browse Source 
This ensures the file names of SPIR-V program combinations and shaders
are reproducible. Up to now they only were if the order of program
combinations in the rules.json didn't change, as the file names
contained the position of the program combination in the rules.json.
With this change files names of program combinations will be named based
on the details of the combination used to create them and the file names
of shaders will be based on their content respectively.

Changing the file names avoids wrong shaders when partially rebuilding
them after a new combination for a program got added in between the
other combinations in rules.json and removes the need for keeping track
of identical shaders in the script. It's also a preparation for being
able to build shaders in parallel, while also keeping the result
reproducible.
This commit is contained in:
Dunedan 2024-09-22 10:56:05 +02:00
parent 57308bb847
commit 42672442b9
Signed by: Dunedan
GPG Key ID: 885B16854284E0B2
1 changed files with 21 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
source/tools/spirv/compile.py
View File

@ -49,6 +49,12 @@ def calculate_hash(path):
return hashlib.sha256(handle.read()).hexdigest()
def get_combination_hash(combination: list[dict[str, str]]) -> str:
"""Turn combination information into a unique hash."""
hashable_combination = tuple([tuple(sorted(i.items())) for i in combination])
return hashlib.sha256(json.dumps(hashable_combination, sort_keys=True).encode()).hexdigest()
def resolve_if(defines, expression):
for item in expression.strip().split("||"):
item = item.strip()
@ -398,11 +404,10 @@ def build(rules, input_mod_path, output_mod_path, dependencies, program_name):
else:
combinations = list(itertools.product(*defines))
hashed_cache = {}
for combination in combinations:
combination_hash = get_combination_hash(combination)
for index, combination in enumerate(combinations):
assert index < 10000
program_path = "spirv/" + program_name + ("_%04d" % index) + ".xml"
program_path = f"spirv/{program_name}_{combination_hash}.xml"
programs_element = ET.SubElement(root, "program")
programs_element.set("type", "spirv")
@ -423,8 +428,8 @@ def build(rules, input_mod_path, output_mod_path, dependencies, program_name):
program_root.set("type", "spirv")
for shader in shaders:
extension = stage_extension[shader["type"]]
file_name = program_name + ("_%04d" % index) + extension + ".spv"
output_spirv_path = os.path.join(output_spirv_mod_path, file_name)
tmp_file_name = f"{program_name}_{combination_hash}{extension}.spv"
tmp_output_spirv_path = os.path.join(output_spirv_mod_path, tmp_file_name)
input_glsl_path = os.path.join(input_mod_path, "shaders", shader["file"])
# Some shader programs might use vs and fs shaders from different mods.
@ -442,16 +447,20 @@ def build(rules, input_mod_path, output_mod_path, dependencies, program_name):
dependencies,
shader["type"],
input_glsl_path,
output_spirv_path,
tmp_output_spirv_path,
combination + program_defines,
)
spirv_hash = calculate_hash(output_spirv_path)
if spirv_hash not in hashed_cache:
hashed_cache[spirv_hash] = file_name
spirv_hash = calculate_hash(tmp_output_spirv_path)
file_name = f"{program_name}_{spirv_hash}{extension}.spv"
output_spirv_path = os.path.join(output_spirv_mod_path, file_name)
if not os.path.exists(output_spirv_path):
try:
os.rename(tmp_output_spirv_path, output_spirv_path)
except FileExistsError:
os.remove(tmp_output_spirv_path)
else:
file_name = hashed_cache[spirv_hash]
os.remove(output_spirv_path)
os.remove(tmp_output_spirv_path)
shader_element = ET.SubElement(program_root, shader["type"])
shader_element.set("file", "spirv/" + file_name)