From b6ee7fd1a3e13c6cde80f066d6ef9abac352d3bd Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 13 Oct 2026 16:42:22 +0200 Subject: [PATCH 1/4] Make a functional architecture mapping for different EESSI versions --- scripts/process_eessi_software_metadata.py | 104 +++++++++++++-------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/scripts/process_eessi_software_metadata.py b/scripts/process_eessi_software_metadata.py index 5663b09..1e52491 100644 --- a/scripts/process_eessi_software_metadata.py +++ b/scripts/process_eessi_software_metadata.py @@ -9,6 +9,8 @@ # EB 5.4.0 brought in a change that required the import below import easybuild.easyblocks.python +# I'm hardcoding this but you can easily figure it out from +# ls /cvmfs/software.eessi.io/versions/*/software/linux/*/*/.lmod/cache/spiderT.lua /cvmfs/software.eessi.io/versions/*/software/linux/*/*/*/.lmod/cache/spiderT.lua ARCHITECTURES = [ "aarch64/generic", "aarch64/a64fx", @@ -33,6 +35,26 @@ "riscv64/generic", ] +# Missing architecture -> architecture key to use as a fallback. +# (to verify these check https://github.com/EESSI/software-layer-scripts/tree/main/tests/archdetect) +CPU_ARCHITECTURE_OVERRIDES = { + "2023.06": { + # aarch64 + "aarch64/aws/graviton4": "aarch64/neoverse_v1", + # x86_64 + "x86_64/amd/zen5": "x86_64/amd/zen4", + "x86_64/intel/graniterapids": "x86_64/intel/sapphirerapids", + # RISC-V + "riscv64/generic": "", + }, + "2025.06": { + "aarch64/aws/graviton4": "aarch64/neoverse_v1", + "riscv64/generic": "", + "x86_64/intel/graniterapids": "x86_64/intel/sapphirerapids", + }, + "2026.06": {}, +} + NVIDIA_ARCHITECTURES = [ "accel/nvidia/cc70", "accel/nvidia/cc75", @@ -113,48 +135,51 @@ def get_software_information_by_filename(file_metadata, original_path=None, tool # 2) Construct the modulefile path before_arch, _, _ = original_path.partition(detected_arch) + path_eessi_version = original_path.split("/versions/")[1].split("/")[0] # Remember, detected_accel_arch can be an empty string modulefile = os.path.join(before_arch, detected_arch, detected_accel_arch, "modules/all", file_metadata["module"]["full_module_name"] + ".lua") spider_cache = before_arch + detected_arch + "/.lmod/cache/spiderT.lua" # 3) Substitute each architecture and test module file existence in spider cache for arch in architecture_group: - substituted_modulefile = modulefile.replace(detected_arch, arch) - substituted_spider_cache = spider_cache.replace(detected_arch, arch) - # os.path.exists is very expensive for CVMFS so we just look for the file in the spider cache (if we can) - if os.path.exists(substituted_spider_cache): - found = subprocess.run(["grep", "-q", substituted_modulefile, substituted_spider_cache]).returncode == 0 - else: - found = os.path.exists(substituted_modulefile) - if found: - base_version_dict["cpu_arch"].append(arch) - # If we have an accelerator module let's check which architectures are supported - if detected_accel_arch: - base_version_dict["gpu_arch"][arch] = [] - if accel_arch in ROCM_ARCHITECTURES: - accel_architectures = ROCM_ARCHITECTURES - else: - accel_architectures = NVIDIA_ARCHITECTURES - for accel_arch in accel_architectures: - accel_substituted_modulefile = substituted_modulefile.replace(detected_accel_arch, accel_arch) - found = subprocess.run(["grep", "-q", accel_substituted_modulefile, substituted_spider_cache]).returncode == 0 - if found: - # Let's not include the "accel/" part of the accel_arch - base_version_dict["gpu_arch"][arch].append(accel_arch.replace("accel/", "", 1)) + # skip checking if the arch is not explicitly supported + if arch not in CPU_ARCHITECTURE_OVERRIDES[path_eessi_version]: + substituted_modulefile = modulefile.replace(detected_arch, arch) + substituted_spider_cache = spider_cache.replace(detected_arch, arch) + # os.path.exists is very expensive for CVMFS so we just look for the file in the spider cache (if we can) + if os.path.exists(substituted_spider_cache): + found = subprocess.run(["grep", "-q", substituted_modulefile, substituted_spider_cache]).returncode == 0 + else: + found = os.path.exists(substituted_modulefile) + if found: + base_version_dict["cpu_arch"].append(arch) + # If we have an accelerator module let's check which architectures are supported + if detected_accel_arch: + base_version_dict["gpu_arch"][arch] = [] + if accel_arch in ROCM_ARCHITECTURES: + accel_architectures = ROCM_ARCHITECTURES else: - # Let's not be too noisy here, we know we don't have some CUDA archs in 2023.06 - if not ( - accel_substituted_modulefile.startswith('/cvmfs/software.eessi.io/versions/2023.06') - and accel_arch in ["accel/nvidia/cc100", "accel/nvidia/cc120"] - ): - print( - f"No module {accel_substituted_modulefile}... " - f"not adding software for architecture {arch}/{accel_arch}" - ) - continue - else: - print(f"No module {substituted_modulefile}...not adding software for architecture {arch}") - continue + accel_architectures = NVIDIA_ARCHITECTURES + for accel_arch in accel_architectures: + accel_substituted_modulefile = substituted_modulefile.replace(detected_accel_arch, accel_arch) + found = subprocess.run(["grep", "-q", accel_substituted_modulefile, substituted_spider_cache]).returncode == 0 + if found: + # Let's not include the "accel/" part of the accel_arch + base_version_dict["gpu_arch"][arch].append(accel_arch.replace("accel/", "", 1)) + else: + # Let's not be too noisy here, we know we don't have some CUDA archs in 2023.06 + if not ( + accel_substituted_modulefile.startswith('/cvmfs/software.eessi.io/versions/2023.06') + and accel_arch in ["accel/nvidia/cc100", "accel/nvidia/cc120"] + ): + print( + f"No module {accel_substituted_modulefile}... " + f"not adding software for architecture {arch}/{accel_arch}" + ) + continue + else: + print(f"No module {substituted_modulefile}...not adding software for architecture {arch}") + continue # Now we can cycle throught the possibilities # - software application itself @@ -411,9 +436,14 @@ def main(): eessi_versions = software_metadata["eessi_version"].keys() base_json_metadata["architectures_map"] = {} for eessi_version in eessi_versions: - base_json_metadata["architectures_map"][eessi_version] = {} + if eessi_version in CPU_ARCHITECTURE_OVERRIDES: + base_json_metadata["architectures_map"][eessi_version] = CPU_ARCHITECTURE_OVERRIDES[eessi_version] + else: + raise ValueError(f"EESSI version {eessi_version} not present in known overrides f{CPU_ARCHITECTURE_OVERRIDES}") for architecture in ARCHITECTURES + RISCV_ARCHITECTURES: - base_json_metadata["architectures_map"][eessi_version][architecture] = architecture + # If an override exists, we prefer it + if architecture not in base_json_metadata["architectures_map"][eessi_version]: + base_json_metadata["architectures_map"][eessi_version][architecture] = architecture base_json_metadata["gpu_architectures_map"] = {} base_json_metadata["category_details"] = {} full_software_information = get_all_software(software_metadata["eessi_version"]) From 2f21fdd07d124d4692393904f10acb584b6ecbf2 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 13 Oct 2026 17:03:57 +0200 Subject: [PATCH 2/4] EESSI versions in dev.eessi.io are a special case --- scripts/process_eessi_software_metadata.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/process_eessi_software_metadata.py b/scripts/process_eessi_software_metadata.py index 1e52491..6b52a1e 100644 --- a/scripts/process_eessi_software_metadata.py +++ b/scripts/process_eessi_software_metadata.py @@ -438,6 +438,9 @@ def main(): for eessi_version in eessi_versions: if eessi_version in CPU_ARCHITECTURE_OVERRIDES: base_json_metadata["architectures_map"][eessi_version] = CPU_ARCHITECTURE_OVERRIDES[eessi_version] + # Some unusual EESSI versions (2025.06-001) for the dev.eessi.io RISC-V stuff, let's ignore that + elif '-00' in eessi_version: + print(f"Not sanity checking EESSI version {eessi_version}, as it looks a dev.eessi.io version") else: raise ValueError(f"EESSI version {eessi_version} not present in known overrides f{CPU_ARCHITECTURE_OVERRIDES}") for architecture in ARCHITECTURES + RISCV_ARCHITECTURES: From 32e0ce53916b69db2c8377d7b38a4a822171c04b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 13 Oct 2026 17:56:46 +0200 Subject: [PATCH 3/4] Be more careful --- scripts/process_eessi_software_metadata.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/process_eessi_software_metadata.py b/scripts/process_eessi_software_metadata.py index 6b52a1e..d20360b 100644 --- a/scripts/process_eessi_software_metadata.py +++ b/scripts/process_eessi_software_metadata.py @@ -436,13 +436,16 @@ def main(): eessi_versions = software_metadata["eessi_version"].keys() base_json_metadata["architectures_map"] = {} for eessi_version in eessi_versions: + # Some unusual EESSI versions (2025.06-001) for the dev.eessi.io RISC-V stuff, let's ignore those + is_dev_version = "-00" in eessi_version + if eessi_version in CPU_ARCHITECTURE_OVERRIDES: base_json_metadata["architectures_map"][eessi_version] = CPU_ARCHITECTURE_OVERRIDES[eessi_version] - # Some unusual EESSI versions (2025.06-001) for the dev.eessi.io RISC-V stuff, let's ignore that - elif '-00' in eessi_version: - print(f"Not sanity checking EESSI version {eessi_version}, as it looks a dev.eessi.io version") + elif is_dev_version: + print(f"Not sanity checking EESSI version {eessi_version}, as it looks like a dev.eessi.io version") + base_json_metadata["architectures_map"][eessi_version] = {} else: - raise ValueError(f"EESSI version {eessi_version} not present in known overrides f{CPU_ARCHITECTURE_OVERRIDES}") + raise ValueError(f"EESSI version {eessi_version} not present in known overrides {CPU_ARCHITECTURE_OVERRIDES}") for architecture in ARCHITECTURES + RISCV_ARCHITECTURES: # If an override exists, we prefer it if architecture not in base_json_metadata["architectures_map"][eessi_version]: From 24705ccd89f601b4b96ba7d9660d718b1f91a729 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 13 Oct 2026 19:52:59 +0200 Subject: [PATCH 4/4] More special case work for dev.eessi.io --- scripts/process_eessi_software_metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/process_eessi_software_metadata.py b/scripts/process_eessi_software_metadata.py index d20360b..f05a502 100644 --- a/scripts/process_eessi_software_metadata.py +++ b/scripts/process_eessi_software_metadata.py @@ -143,7 +143,9 @@ def get_software_information_by_filename(file_metadata, original_path=None, tool # 3) Substitute each architecture and test module file existence in spider cache for arch in architecture_group: # skip checking if the arch is not explicitly supported - if arch not in CPU_ARCHITECTURE_OVERRIDES[path_eessi_version]: + # (this will run the check if path_eessi_version is not in + # CPU_ARCHITECTURE_OVERRIDES, which is what we want for dev.eessi.io versions) + if arch not in CPU_ARCHITECTURE_OVERRIDES.get(path_eessi_version, {}): substituted_modulefile = modulefile.replace(detected_arch, arch) substituted_spider_cache = spider_cache.replace(detected_arch, arch) # os.path.exists is very expensive for CVMFS so we just look for the file in the spider cache (if we can)