diff --git a/.github/workflows/test-eb-hooks-toolchains.yml b/.github/workflows/test-eb-hooks-toolchains.yml new file mode 100644 index 00000000..2041a3ae --- /dev/null +++ b/.github/workflows/test-eb-hooks-toolchains.yml @@ -0,0 +1,33 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Unit tests for supported toolchains in EasyBuild hooks +on: + push: + paths: + - eb_hooks.py + - eessi_supported_toolchains.toml + - tests/eb_hooks/** + - .github/workflows/test-eb-hooks-toolchains.yml + pull_request: + paths: + - eb_hooks.py + - eessi_supported_toolchains.toml + - tests/eb_hooks/** + - .github/workflows/test-eb-hooks-toolchains.yml + workflow_dispatch: +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + test_supported_toolchains: + runs-on: ubuntu-24.04 + steps: + - name: Check out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Install EasyBuild and pytest + run: | + python3 -m venv "$RUNNER_TEMP/venv" + "$RUNNER_TEMP/venv/bin/pip" install easybuild pytest + + - name: Run unit tests + run: | + "$RUNNER_TEMP/venv/bin/python" -m pytest -v tests/eb_hooks diff --git a/.github/workflows/test-eb-hooks.yml b/.github/workflows/test-eb-init-files.yml similarity index 84% rename from .github/workflows/test-eb-hooks.yml rename to .github/workflows/test-eb-init-files.yml index dfcc576c..c8bd3981 100644 --- a/.github/workflows/test-eb-hooks.yml +++ b/.github/workflows/test-eb-init-files.yml @@ -1,5 +1,5 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Run checks on EasyBuild hooks script +name: Run checks on EasyBuild hooks script and related init files on: push: pull_request: @@ -26,30 +26,36 @@ jobs: eessi_stack_version: ${{matrix.EESSI_VERSION}} use_eessi_module: true - - name: Check whether eb_hooks.py script is up-to-date + - name: Check whether eb_hooks.py and related files are up-to-date if: ${{ github.event_name == 'pull_request' }} run: | - FILE="eb_hooks.py" - TEMP_FILE="$(mktemp)" + # Files that are installed in the same directory as the hooks, and have to be up-to-date + FILES=(eb_hooks.py eessi_supported_toolchains.toml) # Fetch base branch git fetch origin ${{ github.base_ref }} - # Check if the hooks has changed in the PR - if git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -q "^$FILE$"; then - echo "Hooks changed in PR. Using PR version." - cp "$FILE" "$TEMP_FILE" - else - echo "File not changed in PR. Using default branch version." - git show origin/${{ github.base_ref }}:$FILE > "$TEMP_FILE" - fi - - # replace placeholder (as is also done in install_scripts.sh) - sed -i "s//${{matrix.EESSI_VERSION}}/g" "${TEMP_FILE}" - - # Compare the hooks to what is shipped in the repository module load EESSI-extend - diff "$TEMP_FILE" "$EASYBUILD_HOOKS" + + for FILE in "${FILES[@]}"; do + echo "Checking whether $FILE is up-to-date" + TEMP_FILE="$(mktemp)" + + # Check if the file has changed in the PR + if git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -q "^$FILE$"; then + echo "$FILE changed in PR. Using PR version." + cp "$FILE" "$TEMP_FILE" + else + echo "$FILE not changed in PR. Using default branch version." + git show origin/${{ github.base_ref }}:$FILE > "$TEMP_FILE" + fi + + # replace placeholder (as is also done in install_scripts.sh) + sed -i "s//${{matrix.EESSI_VERSION}}/g" "${TEMP_FILE}" + + # Compare to what is shipped in the repository (all files are installed in the same directory as the hooks) + diff "$TEMP_FILE" "$(dirname "$EASYBUILD_HOOKS")/$FILE" + done check_eb_hooks_functionality: runs-on: ubuntu-24.04 diff --git a/eb_hooks.py b/eb_hooks.py index fc26f88b..7125cf51 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -10,6 +10,14 @@ import tarfile from typing import NamedTuple +try: + import tomllib # in the standard library since Python 3.11 +except ImportError: + try: + import tomli as tomllib # backport for older Python versions + except ImportError: + tomllib = None + import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import obtain_config_guess from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS @@ -67,31 +75,60 @@ STACK_REPROD_SUBDIR = 'reprod' -EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS = { - '2023.06': [ - {'name': 'foss', 'version': '2022b'}, - {'name': 'foss', 'version': '2023a'}, - {'name': 'foss', 'version': '2023b'}, - ], - '2025.06': [ - {'name': 'foss', 'version': '2024a'}, - {'name': 'foss', 'version': '2025a'}, - {'name': 'foss', 'version': '2025b'}, - ], - '2026.06': [ - {'name': 'foss', 'version': '2026.1'}, - {'name': 'lfoss', 'version': '2026.1'}, - ], -} -if EASYBUILD_VERSION >= '5.2.0': - EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS['2025.06'].append( - {'name': 'lfoss', 'version': '2025b'} - ) +# Environment variable that can be used to point to a custom TOML file with the supported top-level toolchains +SUPPORTED_TOOLCHAINS_FILE_ENVVAR = 'EESSI_SUPPORTED_TOOLCHAINS_FILE' -if EASYBUILD_VERSION >= '5.3.1': - EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS['2025.06'].append( - {'name': 'rompi', 'version': '2025a'} - ) + +def load_supported_top_level_toolchains(): + """ + Load the supported top-level toolchains per EESSI version from a TOML file. + + The location of the TOML file can be set through the environment variable EESSI_SUPPORTED_TOOLCHAINS_FILE. + If that is not set, eessi_supported_toolchains.toml is expected next to this hooks file (both in the + software-layer-scripts repository, and when installed in /init/easybuild). + + Toolchains that require a more recent EasyBuild version than the one being used (as specified via + 'min_easybuild_version') are left out. + + Returns: + supported_toolchains (dict): maps each EESSI version to a list of dicts with the 'name' and 'version' + of a supported top-level toolchain + """ + default_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'eessi_supported_toolchains.toml') + toolchains_file = os.getenv(SUPPORTED_TOOLCHAINS_FILE_ENVVAR) + envvar_msg = f" (set via ${SUPPORTED_TOOLCHAINS_FILE_ENVVAR})" + if not toolchains_file: + toolchains_file = default_file + envvar_msg = "" + + if tomllib is None: + raise EasyBuildError("Parsing TOML files requires Python 3.11 or newer, or the 'tomli' Python package") + + try: + with open(toolchains_file, 'rb') as fh: + toolchains = tomllib.load(fh) + except OSError as err: + msg = (f"Failed to read the file with supported toolchains {toolchains_file}{envvar_msg}: {err}. " + f"By default, it is expected next to the EasyBuild hooks file; its location can be configured " + f"through the environment variable {SUPPORTED_TOOLCHAINS_FILE_ENVVAR}.") + if envvar_msg and os.path.isfile(default_file): + msg += (f" Note that a file with supported toolchains does exist in the default location {default_file}. " + f"If that is the file you intended to use, unset {SUPPORTED_TOOLCHAINS_FILE_ENVVAR}.") + raise EasyBuildError(msg) + except ValueError as err: + raise EasyBuildError(f"The file with supported toolchains {toolchains_file}{envvar_msg} " + f"does not contain valid TOML: {err}") + + return { + eessi_version: [ + {'name': tc['name'], 'version': tc['version']} for tc in tcs + if EASYBUILD_VERSION >= tc.get('min_easybuild_version', '0') + ] + for eessi_version, tcs in toolchains.items() + } + + +EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS = load_supported_top_level_toolchains() # Supported compute capabilities by CUDA toolkit version # Obtained by installing all CUDAs from 12.0.0 to 13.3.0, then using: diff --git a/eessi_supported_toolchains.toml b/eessi_supported_toolchains.toml new file mode 100644 index 00000000..f3734119 --- /dev/null +++ b/eessi_supported_toolchains.toml @@ -0,0 +1,28 @@ +# Top-level toolchains that are supported for installations in each EESSI version. +# This file is read by eb_hooks.py (see load_supported_top_level_toolchains()), and has to be installed next to it. +# +# Each key is an EESSI version, with a list of toolchains ('name' and 'version'). +# An optional 'min_easybuild_version' can be used for toolchains that are only supported when +# a sufficiently recent EasyBuild version is used (e.g. because older versions don't know the toolchain). + +# If you need to use this from a shell and don't have yq available, you can convert it to json on the fly using +# python -c "import tomllib, json; print(json.dumps(tomllib.load(open('input.toml','rb')), indent=2))" > output.json + +"2023.06" = [ + { name = "foss", version = "2022b" }, + { name = "foss", version = "2023a" }, + { name = "foss", version = "2023b" }, +] + +"2025.06" = [ + { name = "foss", version = "2024a" }, + { name = "foss", version = "2025a" }, + { name = "foss", version = "2025b" }, + { name = "lfoss", version = "2025b", min_easybuild_version = "5.2.0" }, + { name = "rompi", version = "2025a", min_easybuild_version = "5.3.1" }, +] + +"2026.06" = [ + { name = "foss", version = "2026.1" }, + { name = "lfoss", version = "2026.1" }, +] diff --git a/install_scripts.sh b/install_scripts.sh index 48745a3d..ae6a854b 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -222,11 +222,12 @@ if [[ -d ${host_injections_easystacks_dir} ]]; then copy_files_by_list ${host_injections_easystacks_dir} ${INSTALL_PREFIX}/scripts/gpu_support/nvidia/easystacks "${host_injections_easystacks[@]}" fi -# Copy over EasyBuild hooks file used for installations -hook_files=( +# Copy over EasyBuild hooks file used for installations (and the files it needs, which have to be located next to it) +easybuild_init_files=( eb_hooks.py + eessi_supported_toolchains.toml ) -copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/init/easybuild "${hook_files[@]}" +copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/init/easybuild "${easybuild_init_files[@]}" # replace version placeholders in scripts; # note: the commands below are always run, regardless of whether the scripts were changed, diff --git a/tests/eb_hooks/test_supported_toolchains.py b/tests/eb_hooks/test_supported_toolchains.py new file mode 100644 index 00000000..b0c18ce5 --- /dev/null +++ b/tests/eb_hooks/test_supported_toolchains.py @@ -0,0 +1,166 @@ +# Unit tests for the supported top-level toolchains (eessi_supported_toolchains.toml) +# and the function in eb_hooks.py that loads them. +# Requires EasyBuild to be importable, e.g.: pip install easybuild pytest +import os +import sys + +import pytest + +try: + import tomllib +except ImportError: # Python < 3.11 + import tomli as tomllib + +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) +sys.path.insert(0, REPO_ROOT) + +import eb_hooks # noqa: E402 +from easybuild.tools.build_log import EasyBuildError # noqa: E402 + +TOOLCHAINS_FILE = os.path.join(REPO_ROOT, 'eessi_supported_toolchains.toml') +ENVVAR = 'EESSI_SUPPORTED_TOOLCHAINS_FILE' + + +@pytest.fixture(autouse=True) +def clean_env(monkeypatch): + monkeypatch.delenv(ENVVAR, raising=False) + + +def write_file(path, content): + path.write_text(content) + return str(path) + + +def test_toml_format(): + """Check that the shipped TOML file has the expected structure.""" + with open(TOOLCHAINS_FILE, 'rb') as fh: + data = tomllib.load(fh) + + assert isinstance(data, dict) and data + for eessi_version, toolchains in data.items(): + assert isinstance(eessi_version, str) + assert isinstance(toolchains, list) and toolchains, f"No toolchains for {eessi_version}" + for tc in toolchains: + assert set(tc) <= {'name', 'version', 'min_easybuild_version'}, tc + assert isinstance(tc.get('name'), str) and tc['name'], tc + assert isinstance(tc.get('version'), str) and tc['version'], tc + if 'min_easybuild_version' in tc: + assert isinstance(tc['min_easybuild_version'], str), tc + # no duplicates + pairs = [(tc['name'], tc['version']) for tc in toolchains] + assert len(pairs) == len(set(pairs)), f"Duplicate toolchains for {eessi_version}" + + +DEFAULT_FILENAME = 'eessi_supported_toolchains.toml' + + +@pytest.fixture +def default_dir(monkeypatch, tmp_path): + """Make the 'directory of eb_hooks.py' (i.e. the default location of the TOML file) an empty temporary dir.""" + monkeypatch.setattr(eb_hooks, '__file__', str(tmp_path / 'eb_hooks.py')) + return tmp_path + + +def test_load_default_location(default_dir): + """Without the environment variable, the file next to eb_hooks.py is used.""" + write_file(default_dir / DEFAULT_FILENAME, ''' + "2099.01" = [ + # comments are the reason for using TOML + { name = "foo", version = "1" }, + { name = "bar", version = "2" }, + ] + "2099.02" = [{ name = "baz", version = "3" }] + ''') + assert eb_hooks.load_supported_top_level_toolchains() == { + '2099.01': [{'name': 'foo', 'version': '1'}, {'name': 'bar', 'version': '2'}], + '2099.02': [{'name': 'baz', 'version': '3'}], + } + + +@pytest.mark.parametrize('eb_version, expected', [ + ('4.9.0', ['always']), + ('5.2.0', ['always', 'since_5_2_0']), + ('5.2.1', ['always', 'since_5_2_0']), + ('5.3.0', ['always', 'since_5_2_0']), + ('5.3.1', ['always', 'since_5_2_0', 'since_5_3_1']), + ('6.0.0', ['always', 'since_5_2_0', 'since_5_3_1']), +]) +def test_min_easybuild_version(monkeypatch, default_dir, eb_version, expected): + """Toolchains with a 'min_easybuild_version' are only included for that EasyBuild version or newer.""" + write_file(default_dir / DEFAULT_FILENAME, ''' + "2099.01" = [ + { name = "always", version = "1" }, + { name = "since_5_2_0", version = "1", min_easybuild_version = "5.2.0" }, + { name = "since_5_3_1", version = "1", min_easybuild_version = "5.3.1" }, + ] + "2099.02" = [ + { name = "only_future", version = "1", min_easybuild_version = "99.0.0" }, + ] + ''') + monkeypatch.setattr(eb_hooks, 'EASYBUILD_VERSION', eb_version) + result = eb_hooks.load_supported_top_level_toolchains() + assert [tc['name'] for tc in result['2099.01']] == expected + # An EESSI version for which no toolchain is supported by this EasyBuild version is kept, with an empty list + assert result['2099.02'] == [] + # The minimum version is not part of the returned toolchain dicts + assert all(set(tc) == {'name', 'version'} for tcs in result.values() for tc in tcs) + + +def test_envvar_overrides_location(monkeypatch, default_dir, tmp_path): + write_file(default_dir / DEFAULT_FILENAME, '"2099.01" = [{ name = "default", version = "1" }]') + custom = write_file(tmp_path / 'custom.toml', '"2099.01" = [{ name = "custom", version = "1" }]') + monkeypatch.setenv(ENVVAR, custom) + assert eb_hooks.load_supported_top_level_toolchains() == {'2099.01': [{'name': 'custom', 'version': '1'}]} + + +def test_envvar_missing_file(monkeypatch, default_dir, tmp_path): + default_file = write_file(default_dir / DEFAULT_FILENAME, '"2099.01" = []') + missing = str(tmp_path / 'does_not_exist.toml') + monkeypatch.setenv(ENVVAR, missing) + with pytest.raises(EasyBuildError) as excinfo: + eb_hooks.load_supported_top_level_toolchains() + msg = str(excinfo.value) + assert missing in msg + assert ENVVAR in msg + # a file exists in the default location, so the user should be pointed to it and told how to use it + assert default_file in msg + assert f"unset {ENVVAR}" in msg + + +def test_envvar_missing_file_no_default(monkeypatch, default_dir, tmp_path): + missing = str(tmp_path / 'does_not_exist.toml') + monkeypatch.setenv(ENVVAR, missing) + with pytest.raises(EasyBuildError) as excinfo: + eb_hooks.load_supported_top_level_toolchains() + msg = str(excinfo.value) + assert missing in msg + assert 'unset' not in msg + + +def test_missing_default_file(default_dir): + with pytest.raises(EasyBuildError) as excinfo: + eb_hooks.load_supported_top_level_toolchains() + msg = str(excinfo.value) + assert str(default_dir / DEFAULT_FILENAME) in msg + assert ENVVAR in msg # mentions how to configure the location + assert 'unset' not in msg + + +@pytest.mark.parametrize('use_envvar', [False, True]) +def test_invalid_toml(monkeypatch, default_dir, tmp_path, use_envvar): + if use_envvar: + bad = write_file(tmp_path / 'bad.toml', '"2099.01" = [') + monkeypatch.setenv(ENVVAR, bad) + else: + bad = write_file(default_dir / DEFAULT_FILENAME, '"2099.01" = [') + with pytest.raises(EasyBuildError, match='does not contain valid TOML') as excinfo: + eb_hooks.load_supported_top_level_toolchains() + assert bad in str(excinfo.value) + + +def test_no_toml_support(monkeypatch, default_dir): + """A clear error is raised if there is no TOML parser available (Python < 3.11 without tomli).""" + write_file(default_dir / DEFAULT_FILENAME, '"2099.01" = []') + monkeypatch.setattr(eb_hooks, 'tomllib', None) + with pytest.raises(EasyBuildError, match='tomli'): + eb_hooks.load_supported_top_level_toolchains()