diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4ebd7c0..6a0cf41 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -14,10 +14,10 @@ jobs: environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - - uses: actions/configure-pages@v5 - - uses: actions/checkout@v5 + - uses: actions/configure-pages@v6 + - uses: actions/checkout@v7 - uses: eessi/github-action-eessi@v3 with: use_eessi_module: true @@ -150,7 +150,7 @@ jobs: - run: | source /tmp/venv_docs/bin/activate zensical build --clean - - uses: actions/upload-pages-artifact@v4 + - uses: actions/upload-pages-artifact@v7 with: path: site - - uses: actions/deploy-pages@v4 + - uses: actions/deploy-pages@v5 diff --git a/.github/workflows/prs.yml b/.github/workflows/prs.yml index 5e1f204..24b1764 100644 --- a/.github/workflows/prs.yml +++ b/.github/workflows/prs.yml @@ -10,9 +10,9 @@ permissions: contents: read jobs: test_data_generation: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - uses: eessi/github-action-eessi@v3 with: use_eessi_module: true @@ -121,7 +121,7 @@ jobs: source /tmp/venv_docs/bin/activate zensical build --clean - name: Upload EESSI API metadata - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: eessi-api-metadata path: docs/data/eessi_api_metadata*software*.json diff --git a/scripts/process_eessi_software_metadata.py b/scripts/process_eessi_software_metadata.py index a1f6a8b..36ab1c2 100644 --- a/scripts/process_eessi_software_metadata.py +++ b/scripts/process_eessi_software_metadata.py @@ -142,42 +142,55 @@ def get_software_information_by_filename(file_metadata, original_path=None, tool # 3) Substitute each architecture and test module file existence in spider cache for arch in architecture_group: - substituted_modulefile = modulefile.replace(detected_arch, arch) - substituted_spider_cache = spider_cache.replace(detected_arch, arch) - # os.path.exists is very expensive for CVMFS so we just look for the file in the spider cache (if we can) - if os.path.exists(substituted_spider_cache): - found = subprocess.run(["grep", "-q", substituted_modulefile, substituted_spider_cache]).returncode == 0 - else: - found = os.path.exists(substituted_modulefile) - if found: - base_version_dict["cpu_arch"].append(arch) - # If we have an accelerator module let's check which architectures are supported - if detected_accel_arch: - base_version_dict["gpu_arch"][arch] = [] - if detected_accel_arch in ROCM_ARCHITECTURES: - accel_architectures = ROCM_ARCHITECTURES - else: - accel_architectures = NVIDIA_ARCHITECTURES - for accel_arch in accel_architectures: - accel_substituted_modulefile = substituted_modulefile.replace(detected_accel_arch, accel_arch) - found = subprocess.run(["grep", "-q", accel_substituted_modulefile, substituted_spider_cache]).returncode == 0 - if found: - # Let's not include the "accel/" part of the accel_arch - base_version_dict["gpu_arch"][arch].append(accel_arch.replace("accel/", "", 1)) + # We check the arch under two conditions + # - the version attached is included CPU_ARCHITECTURE_OVERRIDES + # (which won't be the case for weird dev.eessi.io versions) + # - it doesn't appear in the overrides for that EESSI version + if arch not in CPU_ARCHITECTURE_OVERRIDES.get(path_eessi_version, {}): + substituted_modulefile = modulefile.replace(detected_arch, arch) + substituted_spider_cache = spider_cache.replace(detected_arch, arch) + # os.path.exists is very expensive for CVMFS so we just look for the file in the spider cache (if we can) + if os.path.exists(substituted_spider_cache): + found = subprocess.run(["grep", "-q", substituted_modulefile, substituted_spider_cache]).returncode == 0 + else: + found = os.path.exists(substituted_modulefile) + if found: + base_version_dict["cpu_arch"].append(arch) + # If we have an accelerator module let's check which architectures are supported + # (no accelerators for a64fx) + if detected_accel_arch and arch != 'aarch64/a64fx': + base_version_dict["gpu_arch"][arch] = [] + if detected_accel_arch in ROCM_ARCHITECTURES: + accel_architectures = ROCM_ARCHITECTURES else: - # Let's not be too noisy here, we know we don't have some CUDA archs in 2023.06 - if not ( - accel_substituted_modulefile.startswith('/cvmfs/software.eessi.io/versions/2023.06') - and accel_arch in ["accel/nvidia/cc100", "accel/nvidia/cc120"] - ): - print( - f"No module {accel_substituted_modulefile}... " - f"not adding software for architecture {arch}/{accel_arch}" - ) - continue - else: - print(f"No module {substituted_modulefile}...not adding software for architecture {arch}") - continue + accel_architectures = NVIDIA_ARCHITECTURES + for accel_arch in accel_architectures: + accel_substituted_modulefile = substituted_modulefile.replace(detected_accel_arch, accel_arch) + found = subprocess.run(["grep", "-q", accel_substituted_modulefile, substituted_spider_cache]).returncode == 0 + if found: + # Let's not include the "accel/" part of the accel_arch + base_version_dict["gpu_arch"][arch].append(accel_arch.replace("accel/", "", 1)) + else: + # Let's not be too noisy here, we know we don't have some CUDA archs in 2023.06/2025.06 + if not ( + # Condition A: Original 2023.06 constraints + (path_eessi_version in ('2023.06') and accel_arch in ["accel/nvidia/cc100", "accel/nvidia/cc120"]) + # Condition B: No cc75 in 2023.06 or 2025.06 + or (path_eessi_version in ('2023.06', '2025.06') and accel_arch == "accel/nvidia/cc75") + ): + print( + f"No accelerator module {accel_substituted_modulefile}... " + f"not adding software for architecture {arch}/{accel_arch}" + ) + continue + else: + if not (detected_accel_arch and arch == 'aarch64/a64fx'): + print(f"No module {substituted_modulefile}...not adding software for architecture {arch}") + continue + + # There should be _at least_ one CPU architecture (the original one) or there is a major problem + if len(base_version_dict["cpu_arch"]) == 0: + raise RuntimeError(f"List of CPU archs for {modulefile} is 0, that can't be right!") # Now we can cycle throught the possibilities # - software application itself @@ -438,7 +451,8 @@ def main(): is_dev_version = "-00" in eessi_version if eessi_version in CPU_ARCHITECTURE_OVERRIDES: - base_json_metadata["architectures_map"][eessi_version] = CPU_ARCHITECTURE_OVERRIDES[eessi_version] + # CPU_ARCHITECTURE_OVERRIDES is a nested dict so we need a deepcopy + base_json_metadata["architectures_map"][eessi_version] = copy.deepcopy(CPU_ARCHITECTURE_OVERRIDES[eessi_version]) elif is_dev_version: print(f"Not sanity checking EESSI version {eessi_version}, as it looks like a dev.eessi.io version") base_json_metadata["architectures_map"][eessi_version] = {}