[release/2.2] Container site updates; move to new compilers and Ubuntu 26.04

configs/sites/tier1/container/Dockerfile.gcc

@@ -2,9 +2,9 @@
 # See the README.md for building instructions.
 
 ## ---------- build base container used by builder and runtime ----------
-FROM ubuntu:24.04 AS ubuntu_base
+FROM ubuntu:26.04 AS ubuntu_base
 
-ARG BUILD_JOBS=4
+ARG BUILD_JOBS=8
 ARG SPACK_STACK_TEMPLATE=unified-dev
 
 SHELL ["/bin/bash", "-c"]
@@ -22,14 +22,14 @@ RUN set -euo pipefail; \
     apt-get -yqq update && \
     apt-get -yqq upgrade && \
     apt-get -yqq install --no-install-recommends \
-        # Core build tools.
+        # Core build tools. build-essential pulls in the default gcc-15; we add
+        # gcc-14 alongside it, since spack-stack supports gcc up to 14.x.
         build-essential \
-        gcc-13 \
-        gcc++-13 \
-        gfortran-13 \
-        cpp-13 \
+        gcc-14 \
+        g++-14 \
+        gfortran-14 \
+        cpp-14 \
         make \
-        llvm-14 \
         autoconf \
         # External dependencies.
         libcurl4-openssl-dev \
@@ -55,10 +55,11 @@ RUN set -euo pipefail; \
         locales && \
     locale-gen en_US.UTF-8 && \
     # Update alternatives for compilers and tools
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \
-    update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \
-    update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \
+    # Make gcc-14 the default gcc/g++/gfortran for container users. Spack uses
+    # the -14 binaries by full path (see packages_gcc.yaml).
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100 && \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100 && \
+    update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-14 100 && \
     rm -rf /var/lib/apt/lists/*
 
 
@@ -80,7 +81,8 @@ COPY --exclude=configs/sites/tier1/container/Dockerfile.oneapi \
 
 # Create spack-stack environment
 WORKDIR /opt/spack-stack
-RUN mkdir -p /tmp/spack-stack && \
+RUN set -o pipefail && \
+    mkdir -p /tmp/spack-stack && \
     source setup.sh && \
     spack stack create env \
         --site container \
@@ -93,7 +95,7 @@ RUN mkdir -p /tmp/spack-stack && \
     spack concretize 2>&1 | tee log.concretize
 
 # Build the spack environment.
-RUN  source setup.sh &&  cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \
+RUN  set -o pipefail && source setup.sh &&  cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \
         spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install
 
 # Setup modules and meta-modules.
@@ -126,16 +128,19 @@ RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \
     && echo "export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe" >> /etc/spack_container_rc.sh \
     && echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \
     && echo "source /etc/profile.d/modules.sh" >> /etc/spack_container_rc.sh \
-    && echo "module use /opt/spack-software/modulefiles" >> /etc/spack_container_rc.sh \
     && echo "module use /opt/spack-software/modulefiles/Core" >> /etc/spack_container_rc.sh \
     && echo "# Bash function to load modules necessary to build the JEDI bundle." >> /etc/spack_container_rc.sh \
     && echo "load_jedi_bundle_env() {" >> /etc/spack_container_rc.sh \
     && echo "    module purge" >> /etc/spack_container_rc.sh \
-    && echo "    module load stack-gcc" >> /etc/spack_container_rc.sh \
-    && echo "    module load stack-openmpi" >> /etc/spack_container_rc.sh \
+    && echo "    module load stack-gcc/14.3.0" >> /etc/spack_container_rc.sh \
+    && echo "    module load stack-openmpi/5.0.10" >> /etc/spack_container_rc.sh \
     && echo "    module load base-env" >> /etc/spack_container_rc.sh \
     && echo "    module load jedi-mpas-env" >> /etc/spack_container_rc.sh \
+    && echo "    module load jedi-python-env" >> /etc/spack_container_rc.sh \
     && echo "    module load py-fortranformat" >> /etc/spack_container_rc.sh \
+    && echo "    module load py-matplotlib" >> /etc/spack_container_rc.sh \
+    && echo "    module load py-cartopy" >> /etc/spack_container_rc.sh \
+    && echo "    module load py-pyproj" >> /etc/spack_container_rc.sh \
     && echo "    module load jedi-fv3-env" >> /etc/spack_container_rc.sh \
     && echo "    module load ewok-env" >> /etc/spack_container_rc.sh \
     && echo "    module load ip" >> /etc/spack_container_rc.sh \
@@ -153,6 +158,14 @@ RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891
  && mkdir /home/nonroot/.pmix \
  && chown -R nonroot:nonroot /home/nonroot/.gitconfig /home/nonroot/.pmix
 
+# Fail the build if the stack meta-modules don't resolve to real compiler and
+# MPI modules (and nothing else) in the final image.
+RUN source /etc/spack_container_rc.sh && \
+    module load stack-gcc/14.3.0 stack-openmpi/5.0.10 && \
+    module is-loaded gcc/14.3.0 && module is-loaded openmpi/5.0.10 && \
+    ! module is-loaded ufs-weather-model-env && \
+    which mpicc mpif90 && mpif90 --version | grep -q " 14\.3\."
+
 # Ensure the container rc is run by non-login shells too.
 ENV BASH_ENV=/etc/spack_container_rc.sh
 

configs/sites/tier1/container/Dockerfile.oneapi

@@ -2,9 +2,9 @@
 # See the README.md for building instructions.
 
 ## ---------- build base container used by builder and runtime ----------
-FROM ubuntu:24.04 AS ubuntu_base
+FROM ubuntu:26.04 AS ubuntu_base
 
-ARG BUILD_JOBS=4
+ARG BUILD_JOBS=8
 ARG SPACK_STACK_TEMPLATE=unified-dev
 
 SHELL ["/bin/bash", "-c"]
@@ -22,14 +22,14 @@ RUN set -euo pipefail; \
     apt-get -yqq update && \
     apt-get -yqq upgrade && \
     apt-get -yqq install --no-install-recommends \
-        # Core build tools.
+        # Core build tools. build-essential pulls in the default gcc-15; we add
+        # gcc-14 alongside it, since spack-stack supports gcc up to 14.x.
         build-essential \
-        gcc-13 \
-        gcc++-13 \
-        gfortran-13 \
-        cpp-13 \
+        gcc-14 \
+        g++-14 \
+        gfortran-14 \
+        cpp-14 \
         make \
-        llvm-14 \
         autoconf \
         # External dependencies.
         libcurl4-openssl-dev \
@@ -55,10 +55,11 @@ RUN set -euo pipefail; \
         locales && \
     locale-gen en_US.UTF-8 && \
     # Update alternatives for compilers and tools
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \
-    update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \
-    update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \
+    # Make gcc-14 the default gcc/g++/gfortran for container users. Spack uses
+    # the -14 binaries by full path (see packages_gcc.yaml).
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100 && \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100 && \
+    update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-14 100 && \
     # Install the intel
     apt install -y apt-utils gpg && \
     wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
@@ -68,11 +69,11 @@ RUN set -euo pipefail; \
         | tee /etc/apt/sources.list.d/oneAPI.list && \
     apt update && \
     apt install -y \
-        intel-oneapi-compiler-dpcpp-cpp-2025.3 \
-        intel-oneapi-compiler-fortran-2025.3 \
-        intel-oneapi-mpi-devel-2021.17 \
-        intel-oneapi-tbb-devel-2022.3 \
-        intel-oneapi-mkl-devel-2025.3 && \
+        intel-oneapi-compiler-dpcpp-cpp-2026.1 \
+        intel-oneapi-compiler-fortran-2026.1 \
+        intel-oneapi-mpi-devel-2021.18 \
+        intel-oneapi-tbb-devel-2023.1 \
+        intel-oneapi-mkl-devel-2026.1 && \
     /opt/intel/oneapi/modulefiles-setup.sh --output-dir=/opt/intel/oneapi/modulefiles && \
     echo "source /etc/profile.d/modules.sh && module use /opt/intel/oneapi/modulefiles" > /etc/profile.d/z01_oneapi_modules.sh && \
     rm -rf /var/lib/apt/lists/*
@@ -82,7 +83,12 @@ RUN set -euo pipefail; \
 FROM ubuntu_base AS builder
 
 ENV SPACK_STACK_DIR=/opt/spack-stack \
-    SPACK_ROOT=/opt/spack-stack/spack
+    SPACK_ROOT=/opt/spack-stack/spack \
+    # Spack loads the oneAPI compiler/MPI modules itself (also during
+    # concretization). RUN shells don't source /etc/profile.d, so point spack at
+    # environment-modules and the oneAPI modulefiles explicitly.
+    MODULESHOME=/usr/share/modules \
+    MODULEPATH=/opt/intel/oneapi/modulefiles
 
 
 # Copy spack-stack from build context. Note the build context must be the
@@ -97,7 +103,8 @@ COPY --exclude=configs/sites/tier1/container/Dockerfile.oneapi \
 
 # Create spack-stack environment
 WORKDIR /opt/spack-stack
-RUN mkdir -p /tmp/spack-stack && \
+RUN set -o pipefail && \
+    mkdir -p /tmp/spack-stack && \
     source setup.sh && \
     spack stack create env \
         --site container \
@@ -110,8 +117,11 @@ RUN mkdir -p /tmp/spack-stack && \
     spack concretize 2>&1 | tee log.concretize
 
 # Build the spack environment.
-RUN  source setup.sh &&  cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \
-        spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install
+# Spack loads the external oneAPI modules under LC_ALL=<build_language> (default
+# C). In the C locale environment-modules prints the "®" in Intel's umf
+# modulefile as a bare Latin-1 byte, which spack fails to decode as UTF-8.
+RUN  set -o pipefail && source setup.sh &&  cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \
+        spack -c config:build_language:C.UTF-8 install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install
 
 # Setup modules and meta-modules.
 RUN source setup.sh &&  cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \
@@ -140,16 +150,19 @@ RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \
     && echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \
     && echo "source /etc/profile.d/modules.sh" >> /etc/spack_container_rc.sh \
     && echo "module use /opt/intel/oneapi/modulefiles" >> /etc/spack_container_rc.sh \
-    && echo "module use /opt/spack-software/modulefiles" >> /etc/spack_container_rc.sh \
     && echo "module use /opt/spack-software/modulefiles/Core" >> /etc/spack_container_rc.sh \
     && echo "# Bash function to load modules necessary to build the JEDI bundle." >> /etc/spack_container_rc.sh \
     && echo "load_jedi_bundle_env() {" >> /etc/spack_container_rc.sh \
     && echo "    module purge" >> /etc/spack_container_rc.sh \
-    && echo "    module load stack-intel-oneapi-compilers" >> /etc/spack_container_rc.sh \
-    && echo "    module load stack-intel-oneapi-mpi" >> /etc/spack_container_rc.sh \
+    && echo "    module load stack-intel-oneapi-compilers/2026.1.1" >> /etc/spack_container_rc.sh \
+    && echo "    module load stack-intel-oneapi-mpi/2021.18" >> /etc/spack_container_rc.sh \
     && echo "    module load base-env" >> /etc/spack_container_rc.sh \
     && echo "    module load jedi-mpas-env" >> /etc/spack_container_rc.sh \
+    && echo "    module load jedi-python-env" >> /etc/spack_container_rc.sh \
     && echo "    module load py-fortranformat" >> /etc/spack_container_rc.sh \
+    && echo "    module load py-matplotlib" >> /etc/spack_container_rc.sh \
+    && echo "    module load py-cartopy" >> /etc/spack_container_rc.sh \
+    && echo "    module load py-pyproj" >> /etc/spack_container_rc.sh \
     && echo "    module load jedi-fv3-env" >> /etc/spack_container_rc.sh \
     && echo "    module load ewok-env" >> /etc/spack_container_rc.sh \
     && echo "    module load ip" >> /etc/spack_container_rc.sh \
@@ -166,6 +179,13 @@ RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891
  && mkdir /home/nonroot/.pmix \
  && chown -R nonroot:nonroot /home/nonroot/.gitconfig /home/nonroot/.pmix
 
+# Fail the build if the stack meta-modules don't load the oneAPI compilers and
+# MPI in the final image.
+RUN source /etc/spack_container_rc.sh && \
+    module load stack-intel-oneapi-compilers/2026.1.1 stack-intel-oneapi-mpi/2021.18 && \
+    module is-loaded compiler/2026.1.1 && module is-loaded mpi/2021.18 && \
+    which icx icpx ifx mpiifx
+
 # Ensure the container rc is run by non-login shells too.
 ENV BASH_ENV=/etc/spack_container_rc.sh
 

configs/sites/tier1/container/README.md

@@ -24,14 +24,73 @@ docker build \
     -t spack-stack-gcc:local \
     -f "$(git rev-parse --show-toplevel)/configs/sites/tier1/container/Dockerfile.gcc" \
     --build-arg SPACK_STACK_TEMPLATE=unified-dev \
-    --build-arg BUILD_JOBS=10 \
+    --build-arg BUILD_JOBS=8 \
     "$(git rev-parse --show-toplevel)"  2>&1 | tee ${HOME}/log.docker_gcc
 
 
 docker build \
     -t spack-stack-oneapi:local \
     -f "$(git rev-parse --show-toplevel)/configs/sites/tier1/container/Dockerfile.oneapi" \
     --build-arg SPACK_STACK_TEMPLATE=unified-dev \
-    --build-arg BUILD_JOBS=10 \
+    --build-arg BUILD_JOBS=8 \
     "$(git rev-parse --show-toplevel)" 2>&1 | tee ${HOME}/log.docker_oneapi
 ```
+
+`BUILD_JOBS` is the number of make jobs per package. `config.yaml` sets
+`concurrent_packages: 4`, so a build runs up to 4 x `BUILD_JOBS` jobs at once
+(32 on a 16-core machine). A full `unified-dev` build takes about 4 hours on 16
+cores and needs well over 100 GB of free disk for the build cache.
+
+The `COPY` of the repository includes `.git` (`spack stack create env` records
+the spack-stack and spack commit hashes), so staging or committing a change
+invalidates the Docker cache from that layer on and starts the spack install
+over.
+
+## Image Contents
+
+Both images are based on Ubuntu 26.04.
+
+* **gcc**: gcc 14.3.0 and openmpi 5.0.10. Ubuntu 26.04's default gcc is 15.2,
+  which is newer than spack-stack supports. The image installs gcc-14 next to
+  it and makes it the default `gcc`/`g++`/`gfortran` with `update-alternatives`.
+  Spack uses the `-14` binaries by full path.
+* **oneapi**: Intel oneAPI 2026.1.1 compilers (icx/icpx/ifx), Intel MPI 2021.18
+  and MKL 2026.1, installed from Intel's apt repository into `/opt/intel/oneapi`.
+  The compilers are pinned to the gcc-14 toolchain with compiler flags (see
+  `packages_oneapi.yaml` and the tier1/aws-ubuntu README). gcc 14.3.0 builds a
+  few packages (cmake, rust, py-numpy, ...).
+
+The site configs follow tier1/aws-ubuntu, which tracks the same Ubuntu LTS.
+Differences: tcl modules (environment-modules) instead of Lmod, the
+container-specific Python pins in `packages.yaml`, and build stages under
+`/tmp` (not a tmpfs inside a container).
+
+## Using the Container
+
+The images are development images: they include the compilers, MPI, and the
+spack-stack modules under `/opt/spack-software/modulefiles`. Every bash shell
+sources `/etc/spack_container_rc.sh`, which puts the `Core` module directory on
+`MODULEPATH` and defines `load_jedi_bundle_env`:
+
+```bash
+load_jedi_bundle_env    # stack-gcc + stack-openmpi (or the oneAPI meta-modules) and the JEDI modules
+module list
+```
+
+To load modules by hand, start from the compiler and MPI meta-modules:
+
+```bash
+# gcc
+module load stack-gcc/14.3.0 stack-openmpi/5.0.10
+# oneapi
+module load stack-intel-oneapi-compilers/2026.1.1 stack-intel-oneapi-mpi/2021.18
+```
+
+Load modules such as `py-cartopy` and `py-pyproj` **before** running `ecbuild`
+on jedi-bundle: some tests record `PYTHONPATH` at configure time.
+
+OpenMPI is set up to run as root and to oversubscribe, so MPI tests work with
+the default `docker run`. Intel MPI uses `/dev/shm` for on-node communication,
+and Docker's default of 64 MB is too small: 12-task jedi-bundle tests die with a
+bus error. Run the oneAPI image with a larger `/dev/shm`, for example
+`docker run --shm-size=16g ...`.

configs/sites/tier1/container/config.yaml

@@ -1,5 +1,8 @@
 config:
-  build_jobs: 10
+  # Old installer (see common/config.yaml): build_jobs is per package, so this
+  # runs up to concurrent_packages x build_jobs compile jobs (32 on 16 cores).
+  build_jobs: 8
+  concurrent_packages: 4
 
   # Container-specific paths for build and staging areas.
   # Use /tmp for transient build data (not preserved in final image).

configs/sites/tier1/container/modules.yaml

@@ -5,3 +5,9 @@ modules:
     tcl:
       include:
       - python
+      # common/modules_tcl.yaml excludes compiler and MPI modules. The stack-gcc
+      # and stack-openmpi meta-modules load gcc/<ver> and openmpi/<ver>, so the
+      # gcc env needs them. The oneAPI meta-modules load Intel's own modulefiles
+      # (the externals' modules:), so those packages stay excluded.
+      - gcc
+      - openmpi