From dc76872852ea8cb859a38ca91d7738e34d122846 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 1 Apr 2024 20:28:28 -0600 Subject: [PATCH 01/16] Bug fix in configs/sites/discover-scu17/compilers.yaml: remove duplicate env variable --- configs/sites/discover-scu17/compilers.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/configs/sites/discover-scu17/compilers.yaml b/configs/sites/discover-scu17/compilers.yaml index d24bde1ba..873d9a693 100644 --- a/configs/sites/discover-scu17/compilers.yaml +++ b/configs/sites/discover-scu17/compilers.yaml @@ -20,11 +20,10 @@ compilers: CPATH: '/usr/local/other/gcc/11.4.0/include' LD_LIBRARY_PATH: '/usr/local/intel/oneapi/2021/compiler/2023.2.1/linux/compiler/lib/intel64_lin:/usr/local/other/gcc/11.4.0/lib64' set: - # https://github.com/JCSDA/spack-stack/issues/1012 - I_MPI_EXTRA_FILESYSTEM: 'ON' # https://github.com/JCSDA/spack-stack/issues/1011 I_MPI_SHM_HEAP_VSIZE: '512' PSM2_MEMORY: 'large' + # https://github.com/JCSDA/spack-stack/issues/1012 I_MPI_EXTRA_FILESYSTEM: '1' I_MPI_EXTRA_FILESYSTEM_FORCE: 'gpfs' I_MPI_FABRICS: 'ofi' From 4ee5664397f4ad15980dd5b0adb8fa368aa83822 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Apr 2024 08:37:44 -0600 Subject: [PATCH 02/16] Update doc/source/PreConfiguredSites.rst for Discover part 1 --- doc/source/PreConfiguredSites.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 66cb1aa34..17cfa92c9 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -26,9 +26,10 @@ On selected systems, developmental versions / release candidates are installed t | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env`` | EPIC / JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Discover SCU16 | GCC, Intel | ``/gpfsm/dnb55/projects/p01/s2127/dheinzel/sp170rc1/envs/unified-env-*`` | JCSDA | +| | Discover SCU16 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs/ue-*`` | JCSDA | | NASA +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Discover SCU17 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-20240228/envs/unified-env-*`` | JCSDA | +| | Discover SCU17 | Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-intel-2021.10.0`` | JCSDA | +| | Discover SCU17 | GCC | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-20240228/envs/unified-env-gcc-12.3.0`` | JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.6.0/envs/unified-env`` | JCSDA / EPIC | | NCAR-Wyoming +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -169,11 +170,11 @@ The following is required for building new spack environments and for using spac module load miniconda/3.9.7 module load ecflow/5.8.4 -For ``spack-stack-1.7.0-rc1`` with Intel, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: .. code-block:: console - module use /gpfsm/dnb55/projects/p01/s2127/dheinzel/sp170rc1/envs/unified-env-intel-2021.5.0/install/modulefiles/Core + module use /gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs/ue-intel-2021.5.0/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.0 module load stack-python/3.10.13 @@ -183,7 +184,7 @@ For ``spack-stack-1.7.0-rc1`` with GNU, proceed with loading the following modul .. code-block:: console - module use /gpfsm/dnb55/projects/p01/s2127/dheinzel/sp170rc1/envs/unified-env-gcc-12.1.0/install/modulefiles/Core + module use /gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs/ue-gcc-12.1.0/install/modulefiles/Core module load stack-gcc/12.1.0 module load stack-openmpi/4.1.3 module load stack-python/3.10.13 @@ -202,11 +203,11 @@ The following is required for building new spack environments and for using spac module use /discover/swdev/jcsda/spack-stack/scu17/modulefiles module load ecflow/5.11.4 -For ``spack-stack-20240228`` with Intel, load the following modules after loading ecflow: +For ``spack-stack-1.7.0`` with Intel, load the following modules after loading ecflow: .. code-block:: console - module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-20240228/envs/unified-env-intel-2021.10.0/install/modulefiles/Core + module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-intel-2021.10.0/install/modulefiles/Core module load stack-intel/2021.10.0 module load stack-intel-oneapi-mpi/2021.10.0 module load stack-python/3.10.13 From 9fceb491c69ee4cf569254cdc5eda58d4d0def82 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Apr 2024 10:55:45 -0600 Subject: [PATCH 03/16] Documentation updates for release/1.7.0 part 2: S4, ParallelWorks, AWS ParallelCluster --- doc/source/PreConfiguredSites.rst | 69 ++++--------------------------- 1 file changed, 8 insertions(+), 61 deletions(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 17cfa92c9..e56311537 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -49,7 +49,7 @@ On selected systems, developmental versions / release candidates are installed t | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Nautilus | AOCC | *currently not supported* | JCSDA / NRL | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/unified-env`` | JCSDA | +| | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0`` | JCSDA | | Univ. of Wisconsin +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | S4 | GCC | *currently not supported* | JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -57,11 +57,9 @@ On selected systems, developmental versions / release candidates are installed t +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.6.0/envs/unified-env`` | JCSDA | + Amazon Web Services +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Parallelcluster JCSDA R&D | Intel | ``/mnt/experiments-efs/skylab-v8/spack-stack-20240207/envs/unified-env-*`` | JCSDA | +| | Parallelcluster JCSDA R&D | GCC, Intel | *currently unavailable* | JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env`` | EPIC / JCSDA | -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | RDHPCS Gcloud only for testing | Intel | ``/contrib/spack-stack/spack-stack-1.7.0-rc1/envs/ue-intel-2021.3.0`` | JCSDA | +| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.3.0`` | EPIC / JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ For more information about a specific platform, please see the individual sections below. @@ -112,9 +110,6 @@ For ``spack-stack-1.6.0`` with GNU, load the following modules after loading min .. note:: The unified environment on Orion uses ``cdo@2.0.5`` instead of the default ``cdo@2.2.0`` because of a bug in the ``cdo`` package recipe that affects systems that don't have a ``python3`` interpreter in the default search paths (see https://github.com/spack/spack/issues/41947) for more information. This is a temporary change on Orion for the spack-stack-1.6.0 release and will be reverted once the ``cdo`` package is updated in the upstream spack develop code. -.. note:: - spack-stack-1.6.0 on Orion provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. - ------------------------------ MSU Hercules ------------------------------ @@ -148,9 +143,6 @@ For ``spack-stack-1.7.0-rc1`` with GNU, proceed with loading the following modul module load stack-python/3.10.13 module available -.. note:: - spack-stack-1.6.0 on Hercules provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. - .. note:: spack-stack-1.6.0 on Hercules has additional packages `fms@2023.02.01`, `sp@2.3.0`, and `crtm@2.4.0` installed in the unified environment, in addition to the two default versions `fms@2023.04` and `fms@release-jcsda`. @@ -456,21 +448,11 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load git-lfs/2.4.1 -For testing, JCSDA installed ``spack-stack-1.7.0-rc1`` with Intel on Gcloud, which can be loaded as follows: - -.. code-block:: console - - module use /contrib/spack-stack/spack-stack-1.7.0-rc1/envs/ue-intel-2021.3.0/install/modulefiles/Core - module load stack-intel/2021.3.0 - module load stack-intel-oneapi-mpi/2021.3.0 - module load stack-python/3.10.13 - module available - -For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: .. code-block:: console - module use /contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /contrib/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.3.0/install/modulefiles/Core module load stack-intel/2021.3.0 module load stack-intel-oneapi-mpi/2021.3.0 module load stack-python/3.10.13 @@ -551,10 +533,6 @@ For ``spack-stack-1.6.0`` with GNU, proceed with loading the following modules: Note that on Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow01``). Users starting ``ecflow_server`` on the regular login nodes will see their servers being killed every few minutes, and may be barred from accessing the system. -.. note:: - - spack-stack-1.6.0 on Hera provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. - .. _Preconfigured_Sites_Jet: ------------------------------ @@ -591,10 +569,6 @@ For ``spack-stack-1.6.0`` with GNU, proceed with loading the following modules: module load stack-python/3.10.8 module available -.. note:: - - spack-stack-1.6.0 on Jet provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. - ------------------------------ UW (Univ. of Wisconsin) S4 ------------------------------ @@ -608,11 +582,11 @@ The following is required for building new spack environments and for using spac module load miniconda/3.9.12 module load ecflow/5.8.4 -For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: .. code-block:: console - module use /data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /data/prod/jedi/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.0 module load stack-python/3.10.13 @@ -622,10 +596,6 @@ For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules Note the two `module unuse` commands, that need to be run after the stack metamodules are loaded. Loading the Intel compiler meta module loads the Intel compiler module provided by the sysadmins, which adds those two directories to the module path. These contain duplicate libraries that are not compatible with our stack, such as ``hdf4``. -.. note:: - - spack-stack-1.6.0 on S4 provides a chained environment `gsi-addon-env` for GSI with Intel. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. - .. note:: There is currently no support for GNU on S4, because recent updates to ``hdf5`` require a newer version of ``mpich`` (or other MPI library) than available on the system. @@ -634,30 +604,7 @@ Note the two `module unuse` commands, that need to be run after the stack metamo Amazon Web Services Parallelcluster Ubuntu 20.04 ------------------------------------------------ -Access to the JCSDA-managed AWS Parallel Clusters is not available to the public. The following instructions are for JCSDA core staff and in-kind contributors. - -For ``spack-stack-20240207`` with Intel on the JCSDA R&D cluster (``hpc6a.48xlarge`` instances), run the following commands/load the following modules: - -.. code-block:: console - - module purge - ulimit -s unlimited - source /opt/intel/oneapi/compiler/2022.1.0/env/vars.sh - module use /mnt/experiments-efs/skylab-v8/spack-stack-20240207/envs/unified-env-intel-2021.6.0/install/modulefiles/Core - module load stack-intel/2021.6.0 - module load stack-intel-oneapi-mpi/2021.6.0 - module load stack-python/3.10.13 - module available - -For ``spack-stack-20240207`` with GNU on the JCSDA R&D cluster (``hpc6a.48xlarge`` instances), run the following commands/load the following modules: - - module purge - ulimit -s unlimited - module use /mnt/experiments-efs/skylab-v8/spack-stack-20240207/envs/unified-env-gcc-9.4.0/install/modulefiles/Core - module load stack-gcc/9.4.0 - module load stack-openmpi/4.1.4 - module load stack-python/3.10.13 - module available +The JCSDA-managed AWS Parallel Cluster is currently unavailable. ----------------------------- Amazon Web Services Red Hat 8 From 8c9e40f3658294b8d4aaaf810f522fe609077c40 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 3 Apr 2024 10:14:00 -0600 Subject: [PATCH 04/16] Documentation updates for release-1.7.0 for more sites --- configs/sites/noaa-gcloud/README.md | 3 +++ doc/source/PreConfiguredSites.rst | 41 +++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/configs/sites/noaa-gcloud/README.md b/configs/sites/noaa-gcloud/README.md index a7399c401..d60a111d2 100644 --- a/configs/sites/noaa-gcloud/README.md +++ b/configs/sites/noaa-gcloud/README.md @@ -26,6 +26,9 @@ mkdir -p /contrib/admin cat < /contrib/admin/basic_setup.sh #!/bin/bash +yum-config-manager --disable intel-clck-2019-repo +yum-config-manager --disable intel-hpc-platform +yum-config-manager --disable intelpython chmod 777 /contrib yum install -y qt5-qtbase-devel yum install -y qt5-qtsvg-devel diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index e56311537..3e1204948 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -28,10 +28,9 @@ On selected systems, developmental versions / release candidates are installed t +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Discover SCU16 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs/ue-*`` | JCSDA | | NASA +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Discover SCU17 | Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-intel-2021.10.0`` | JCSDA | -| | Discover SCU17 | GCC | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-20240228/envs/unified-env-gcc-12.3.0`` | JCSDA | +| | Discover SCU17 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-*`` | JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.6.0/envs/unified-env`` | JCSDA / EPIC | +| | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.7.0/envs/ue-gcc-12.2.0`` | JCSDA / EPIC | | NCAR-Wyoming +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env`` | EPIC / JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -43,9 +42,9 @@ On selected systems, developmental versions / release candidates are installed t | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | EPIC / NOAA-EMC | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | GCC, Intel | ``/p/work1/heinzell/spack-stack-1.7.0-rc1/envs/unified-env-*-craywrappers`` | JCSDA / NRL | +| | Narwhal | GCC, Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-`` | JCSDA / NRL | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| U.S. Navy (HPCMP) | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env`` | JCSDA / NRL | +| U.S. Navy (HPCMP) | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0`` | JCSDA / NRL | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Nautilus | AOCC | *currently not supported* | JCSDA / NRL | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -172,7 +171,7 @@ For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules module load stack-python/3.10.13 module available -For ``spack-stack-1.7.0-rc1`` with GNU, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: .. code-block:: console @@ -205,11 +204,11 @@ For ``spack-stack-1.7.0`` with Intel, load the following modules after loading e module load stack-python/3.10.13 module available -For ``spack-stack-20240228`` with GNU, load the following modules after loading ecflow: +For ``spack-stack-1.7.0`` with GNU, load the following modules after loading ecflow: .. code-block:: console - module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-20240228/envs/unified-env-gcc-12.3.0/install/modulefiles/Core + module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-gcc-12.3.0/install/modulefiles/Core module load stack-gcc/12.3.0 module load stack-openmpi/4.1.6 module load stack-python/3.10.13 @@ -240,7 +239,7 @@ With Intel, the following is required for building new spack environments and fo module use /p/app/projects/NEPTUNE/spack-stack/modulefiles module load ecflow/5.8.4 -For ``spack-stack-1.7.0-rc1`` with Intel, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: .. code-block:: console @@ -248,7 +247,7 @@ For ``spack-stack-1.7.0-rc1`` with Intel, proceed with loading the following mod # Note we can't load craype-network-ucx for building spack-stack environments, must do here module unload craype-network-ofi module load craype-network-ucx - module use /p/work1/heinzell/spack-stack-1.7.0-rc1/envs/unified-env-intel-2021.4.0-craywrappers/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.4.0/install/modulefiles/Core module load stack-intel/2021.4.0 module load stack-cray-mpich/8.1.14 module load stack-python/3.10.13 @@ -272,7 +271,7 @@ With GNU, the following is required for building new spack environments and for module use /p/app/projects/NEPTUNE/spack-stack/modulefiles module load ecflow/5.8.4 -For ``spack-stack-1.7.0-rc1`` with GNU, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: .. code-block:: console @@ -280,7 +279,7 @@ For ``spack-stack-1.7.0-rc1`` with GNU, proceed with loading the following modul # Note we can't load craype-network-ucx for building spack-stack environments, must do here module unload craype-network-ofi module load craype-network-ucx - module use /p/work1/heinzell/spack-stack-1.7.0-rc1/envs/unified-env-gcc-10.3.0-craywrappers/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-gcc-10.3.0/install/modulefiles/Core module load stack-gcc/10.3.0 module load stack-cray-mpich/8.1.14 module load stack-python/3.10.13 @@ -305,11 +304,11 @@ With Intel, the following is required for building new spack environments and fo module use /p/app/projects/NEPTUNE/spack-stack/modulefiles module load ecflow/5.8.4 -For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/ue-openmpi416/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-openmpi/4.1.6 module load stack-python/3.10.13 @@ -331,16 +330,12 @@ With AMD clang/flang (aocc), the following is required for building new spack en .. note:: - ``spack-stack-1.6.0`` is not yet supported with the Arm clang/flang compilers. Use Intel instead. + ``spack-stack-1.7.0`` is not yet supported with the Arm clang/flang compilers. Use Intel instead. .. note:: `wgrib2@2.0.8` does not build on Nautilus, therefore we are using `wgrib2@3.1.1` on this system. -.. note:: - - There are still problems launching the ecflow GUI, although the package is installed. - .. _Preconfigured_Sites_Casper: ------------------------------ @@ -358,12 +353,11 @@ The following is required for building new spack environments and for using spac module use /glade/work/epicufsrt/contrib/spack-stack/casper/modulefiles module load ecflow/5.8.4 -For ``spack-stack-1.6.0`` with GNU, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: .. code-block:: console - module use /glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core - + module use /glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.7.0/envs/ue-gcc-12.2.0/install/modulefiles/Core module load stack-gcc/12.2.0 module load stack-openmpi/4.1.6 module load stack-python/3.10.13 @@ -606,6 +600,9 @@ Amazon Web Services Parallelcluster Ubuntu 20.04 The JCSDA-managed AWS Parallel Cluster is currently unavailable. + +CI CLUSTER: /mnt/experiments-efs/skylab-v8/spack-stack-1.7.0-ci-c6i/envs/ue-intel-2021.6.0/install/modulefiles/Core + ----------------------------- Amazon Web Services Red Hat 8 ----------------------------- From 90f160a86762f82ee8d12f3d4cd2d476a694abc7 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 4 Apr 2024 07:32:39 -0600 Subject: [PATCH 05/16] Update discover-scu16 intel config and documentation, and add note in doc/source/NewSiteConfigs.rst about problems of intel@2021.5.0 with zlib-ng --- configs/sites/discover-scu16/compilers.yaml | 16 ++++++++-------- configs/sites/discover-scu16/packages.yaml | 8 ++++---- doc/source/NewSiteConfigs.rst | 11 +++++++---- doc/source/PreConfiguredSites.rst | 6 +++--- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/configs/sites/discover-scu16/compilers.yaml b/configs/sites/discover-scu16/compilers.yaml index f763d7ccd..84cfc9b9a 100644 --- a/configs/sites/discover-scu16/compilers.yaml +++ b/configs/sites/discover-scu16/compilers.yaml @@ -1,24 +1,24 @@ compilers: - compiler: - spec: intel@=2021.5.0 + spec: intel@=2021.6.0 paths: - cc: /usr/local/intel/oneapi/2021/compiler/2022.0.1/linux/bin/intel64/icc - cxx: /usr/local/intel/oneapi/2021/compiler/2022.0.1/linux/bin/intel64/icpc - f77: /usr/local/intel/oneapi/2021/compiler/2022.0.1/linux/bin/intel64/ifort - fc: /usr/local/intel/oneapi/2021/compiler/2022.0.1/linux/bin/intel64/ifort + cc: /usr/local/intel/oneapi/2021/compiler/2022.1.0/linux/bin/intel64/icc + cxx: /usr/local/intel/oneapi/2021/compiler/2022.1.0/linux/bin/intel64/icpc + f77: /usr/local/intel/oneapi/2021/compiler/2022.1.0/linux/bin/intel64/ifort + fc: /usr/local/intel/oneapi/2021/compiler/2022.1.0/linux/bin/intel64/ifort flags: ldflags: '-L/usr/local/other/gcc/11.2.0/lib64' operating_system: sles12 target: x86_64 modules: - - comp/intel/2021.5.0 + - comp/intel/2021.6.0 environment: prepend_path: PATH: '/usr/local/other/gcc/11.2.0/bin' CPATH: '/usr/local/other/gcc/11.2.0/include' - LD_LIBRARY_PATH: '/usr/local/intel/oneapi/2021/compiler/2022.0.1/linux/compiler/lib/intel64_lin:/usr/local/other/gcc/11.2.0/lib64' + LD_LIBRARY_PATH: '/usr/local/intel/oneapi/2021/compiler/2022.1.0/linux/compiler/lib/intel64_lin:/usr/local/other/gcc/11.2.0/lib64' set: - I_MPI_ROOT: '/usr/local/intel/oneapi/2021/mpi/2021.5.0' + I_MPI_ROOT: '/usr/local/intel/oneapi/2021/mpi/2021.6.0' extra_rpaths: [] - compiler: spec: gcc@=12.1.0 diff --git a/configs/sites/discover-scu16/packages.yaml b/configs/sites/discover-scu16/packages.yaml index 72b05b604..83532853f 100644 --- a/configs/sites/discover-scu16/packages.yaml +++ b/configs/sites/discover-scu16/packages.yaml @@ -1,18 +1,18 @@ packages: all: - compiler:: [intel@2021.5.0, gcc@12.1.0] + compiler:: [intel@2021.6.0, gcc@12.1.0] providers: - mpi:: [intel-oneapi-mpi@2021.5.0, openmpi@4.1.3] + mpi:: [intel-oneapi-mpi@2021.6.0, openmpi@4.1.3] ### MPI, Python, MKL mpi: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.5.0%intel@2021.5.0 + - spec: intel-oneapi-mpi@2021.6.0%intel@2021.6.0 prefix: /usr/local/intel/oneapi/2021 modules: - - mpi/impi/2021.5.0 + - mpi/impi/2021.6.0 openmpi:: externals: - spec: openmpi@4.1.3%gcc@12.1.0 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index 6eac881af..9aa0af700 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -13,13 +13,13 @@ It is also instructive to peruse the GitHub actions scripts in ``.github/workflo +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ | Compiler | Versions tested/in use in one or more site configs | Spack compiler identifier | +===========================================+======================================================================+===========================+ -| Intel classic (icc, icpc, ifort) | 2021.3.0 to the latest available version in oneAPI 2023.1.0 | ``intel@`` | +| Intel classic (icc, icpc, ifort) | 2021.3.0 to the latest available version in oneAPI 2023.1.0 [#fn1]_ | ``intel@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ | Intel mixed (icx, icpx, ifort) | all versions up to latest available version in oneAPI 2023.1.0 | ``intel@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ | GNU (gcc, g++, gfortran) | 9.2.0 to 12.2.0 (note: 13.x.y is **not** yet supported) | ``gcc@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ -| Apple clang (clang, clang++, w/ gfortran) | 13.1.6 to 15.0.0 [#fn1]_ | ``apple-clang@`` | +| Apple clang (clang, clang++, w/ gfortran) | 13.1.6 to 15.0.0 [#fn2]_ | ``apple-clang@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ | LLVM clang (clang, clang++, w/ gfortran) | 10.0.0 to 14.0.3 | ``clang@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ @@ -27,8 +27,11 @@ It is also instructive to peruse the GitHub actions scripts in ``.github/workflo .. rubric:: Footnotes .. [#fn1] - Note that apple-clang@14.x compiler versions are fully supported, and apple-clang@15.0.0 will work but requires the :ref:`workaround noted below`. - Also, when using apple-clang@15.0.0 you must use Command Line Tools version 15.1, and the Command Line Tools versions 15.3 and newer are not yet supported. + We have noted problems on some - not all - platforms with ``intel@2021.5.0`` when we switched from ``zlib`` to ``zlib-ng`` in spack-stack-1.7.0. These issues went away when using a different version of the compiler (anything between 2021.3.0 and 2021.11.0). It is therefore recommeneded to avoid using ``intel@2021.5.0`` unless it is the only option. + +.. [#fn2] + Note that ``apple-clang@14.x`` compiler versions are fully supported, and ``apple-clang@15.0.0`` will work but requires the :ref:`workaround noted below`. + Also, when using ``apple-clang@15.0.0`` you must use Command Line Tools version 15.1, and the Command Line Tools versions 15.3 and newer are not yet supported. .. _NewSiteConfigs_macOS: diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 3e1204948..50c1ac2bd 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -165,9 +165,9 @@ For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules .. code-block:: console - module use /gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs/ue-intel-2021.5.0/install/modulefiles/Core - module load stack-intel/2021.5.0 - module load stack-intel-oneapi-mpi/2021.5.0 + module use /gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs/ue-intel-2021.6.0/install/modulefiles/Core + module load stack-intel/2021.6.0 + module load stack-intel-oneapi-mpi/2021.6.0 module load stack-python/3.10.13 module available From d311e3db3da64bafe512646a6c66331a70ccb50b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 5 Apr 2024 08:23:43 -0600 Subject: [PATCH 06/16] Update documentation for S4 and for problems of zlib-ng with intel@2021.5.0 on some systems --- configs/sites/s4/packages.yaml | 2 ++ doc/source/NewSiteConfigs.rst | 2 +- doc/source/PreConfiguredSites.rst | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/configs/sites/s4/packages.yaml b/configs/sites/s4/packages.yaml index 20ad01b68..e13d63cb2 100644 --- a/configs/sites/s4/packages.yaml +++ b/configs/sites/s4/packages.yaml @@ -7,6 +7,8 @@ packages: compiler:: [intel@2021.5.0] providers: mpi:: [intel-oneapi-mpi@2021.5.0] + # https://github.com/JCSDA/spack-stack/issues/1055 + zlib-api:: [zlib] ### MPI, Python, MKL mpi: diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index 9aa0af700..37afdb830 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -27,7 +27,7 @@ It is also instructive to peruse the GitHub actions scripts in ``.github/workflo .. rubric:: Footnotes .. [#fn1] - We have noted problems on some - not all - platforms with ``intel@2021.5.0`` when we switched from ``zlib`` to ``zlib-ng`` in spack-stack-1.7.0. These issues went away when using a different version of the compiler (anything between 2021.3.0 and 2021.11.0). It is therefore recommeneded to avoid using ``intel@2021.5.0`` unless it is the only option. + We have noted problems on some - not all - platforms with ``intel@2021.5.0`` when we switched from ``zlib`` to ``zlib-ng`` in spack-stack-1.7.0. These issues went away when using a different version of the compiler (anything between 2021.3.0 and 2021.11.0). It is therefore recommended to avoid using ``intel@2021.5.0`` unless it is the only option. .. [#fn2] Note that ``apple-clang@14.x`` compiler versions are fully supported, and ``apple-clang@15.0.0`` will work but requires the :ref:`workaround noted below`. diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 50c1ac2bd..8cf94199a 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -592,7 +592,7 @@ Note the two `module unuse` commands, that need to be run after the stack metamo .. note:: - There is currently no support for GNU on S4, because recent updates to ``hdf5`` require a newer version of ``mpich`` (or other MPI library) than available on the system. + There is currently no support for GNU on S4, because recent updates to ``hdf5`` require a newer version of ``mpich`` (or other MPI library) than available on the system. Also, for spack-stack-1.7.0, S4 is the only system that uses ``zlib`` instead of ``zlib-ng`` due to the issues described in https://github.com/JCSDA/spack-stack/issues/1055. ------------------------------------------------ Amazon Web Services Parallelcluster Ubuntu 20.04 From c7f85e81619098aff80dddf9f51a35b75354a9ea Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 5 Apr 2024 09:15:51 -0600 Subject: [PATCH 07/16] Update doc/source/PreConfiguredSites.rst for other platforms --- doc/source/PreConfiguredSites.rst | 97 ++++++++++++++----------------- 1 file changed, 44 insertions(+), 53 deletions(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 8cf94199a..09376ac5c 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -20,9 +20,7 @@ On selected systems, developmental versions / release candidates are installed t +=====================+==================================+=================+=========================================================================================================+===============================+ | **HPC platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| MSU | Hercules GCC+OpenMPI recommended | GCC | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/ue-gcc12-openmpi416`` | EPIC / JCSDA | -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Hercules | Intel | ``/work2/noaa/jcsda/dheinzel/spack-stack-1.7.0-rc1/envs/unified-env-*`` | EPIC / JCSDA | +| | Hercules | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs/ue-*`` | EPIC / JCSDA | | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env`` | EPIC / JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -32,29 +30,25 @@ On selected systems, developmental versions / release candidates are installed t +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.7.0/envs/ue-gcc-12.2.0`` | JCSDA / EPIC | | NCAR-Wyoming +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env`` | EPIC / JCSDA | +| | Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.7.0/envs/ue-*`` | EPIC / JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.6.0/envs/unified-env-intel{19,2022}`` | NOAA-EMC | +| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.7.0/envs/ue-intel*`` | NOAA-EMC | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Gaea C5 | Intel | ``/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | EPIC / NOAA-EMC | +| | Gaea C5 | Intel | ``/ncrc/proj/epic/spack-stack/spack-stack-1.7.0/envs/ue-intel`` | EPIC / NOAA-EMC | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | EPIC / NOAA-EMC | +| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-*`` | EPIC / NOAA-EMC | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | EPIC / NOAA-EMC | +| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-*`` | EPIC / NOAA-EMC | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Narwhal | GCC, Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-`` | JCSDA / NRL | -| +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| U.S. Navy (HPCMP) | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0`` | JCSDA / NRL | -| +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Nautilus | AOCC | *currently not supported* | JCSDA / NRL | +| U.S. Navy (HPCMP) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ +| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0`` | JCSDA / NRL | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0`` | JCSDA | -| Univ. of Wisconsin +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | S4 | GCC | *currently not supported* | JCSDA | +| Univ. of Wisconsin | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0`` | JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | **Cloud platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.6.0/envs/unified-env`` | JCSDA | +| | AMI Red Hat 8 | GCC | *currently unavailable* | JCSDA | + Amazon Web Services +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Parallelcluster JCSDA R&D | GCC, Intel | *currently unavailable* | JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -107,7 +101,10 @@ For ``spack-stack-1.6.0`` with GNU, load the following modules after loading min module available .. note:: - The unified environment on Orion uses ``cdo@2.0.5`` instead of the default ``cdo@2.2.0`` because of a bug in the ``cdo`` package recipe that affects systems that don't have a ``python3`` interpreter in the default search paths (see https://github.com/spack/spack/issues/41947) for more information. This is a temporary change on Orion for the spack-stack-1.6.0 release and will be reverted once the ``cdo`` package is updated in the upstream spack develop code. + The unified environment on Orion uses ``cdo@2.0.5`` instead of the default ``cdo@2.2.0`` because of a bug in the ``cdo`` package recipe that affects systems that don't have a ``python3`` interpreter in the default search paths (see https://github.com/spack/spack/issues/41947) for more information. This is a temporary change on Orion for the spack-stack-1.7.0 release and will be reverted once the ``cdo`` package is updated in the upstream spack develop code. + +.. note:: + spack-stack-1.7.0 on Orion provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `ue` in the above `module use` statements with `gsi-addon`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. ------------------------------ MSU Hercules @@ -122,28 +119,28 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load git-lfs/3.1.2 -For ``spack-stack-1.7.0-rc1`` with Intel, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: .. code-block:: console - module use /work2/noaa/jcsda/dheinzel/spack-stack-1.7.0-rc1/envs/unified-env-intel-2021.9.0/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core module load stack-intel/2021.9.0 module load stack-intel-oneapi-mpi/2021.9.0 module load stack-python/3.10.13 module available -For ``spack-stack-1.7.0-rc1`` with GNU, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: .. code-block:: console - module use /work2/noaa/jcsda/dheinzel/spack-stack-1.7.0-rc1/envs/unified-env-gcc-12.2.0/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core module load stack-gcc/12.2.0 module load stack-openmpi/4.1.6 module load stack-python/3.10.13 module available .. note:: - spack-stack-1.6.0 on Hercules has additional packages `fms@2023.02.01`, `sp@2.3.0`, and `crtm@2.4.0` installed in the unified environment, in addition to the two default versions `fms@2023.04` and `fms@release-jcsda`. + spack-stack-1.7.0 on Hercules provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `ue` in the above `module use` statements with `gsi-addon`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. .. _Preconfigured_Sites_Discover_SCU16: @@ -380,21 +377,21 @@ The following is required for building new spack environments and for using spac module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles module load ecflow/5.8.4 -For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: .. code-block:: console - module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core module load stack-intel/2021.10.0 module load stack-cray-mpich/8.1.25 module load stack-python/3.10.13 module available -For ``spack-stack-1.6.0`` with GNU, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: .. code-block:: console - module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core module load stack-gcc/12.2.0 module load stack-cray-mpich/8.1.25 module load stack-python/3.10.13 @@ -409,7 +406,7 @@ For ``spack-stack-1.6.0`` with GNU, proceed with loading the following modules: NOAA Acorn (WCOSS2 test system) ------------------------------- -For spack-stack-1.6.0, the meta modules are in ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core``. +For spack-stack-1.7.0, the meta modules are in ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.7.0/envs/ue-intel{19,2022}/modulefiles/Core``. On WCOSS2 OpenSUSE sets ``CONFIG_SITE`` which causes libraries to be installed in ``lib64``, breaking the ``lib`` assumption made by some packages. Therefore, ``CONFIG_SITE`` should be set to empty in ``compilers.yaml``. Also, don't use ``module purge`` on Acorn! @@ -470,16 +467,19 @@ The following is required for building new spack environments and for using spac module use /ncrc/proj/epic/spack-stack/modulefiles module load ecflow/5.8.4 -For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: .. code-block:: console - module use /ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /ncrc/proj/epic/spack-stack/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core module load stack-intel/2023.1.0 module load stack-cray-mpich/8.1.25 module load stack-python/3.10.13 module -t available +.. note:: + spack-stack-1.7.0 on Gaea provides a chained environment `gsi-addon-env` for GSI with Intel. To use this environment, replace `ue` in the above `module use` statements with `gsi-addon`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. + .. note:: On Gaea C5, running ``module available`` without the option ``-t`` leads to an error: ``/usr/bin/lua5.3: /opt/cray/pe/lmod/lmod/libexec/Spider.lua:568: stack overflow`` @@ -505,21 +505,21 @@ The following is required for building new spack environments and for using spac module load miniconda/3.9.12 module load ecflow/5.5.3 -For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: .. code-block:: console - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 module load stack-python/3.10.13 module available -For ``spack-stack-1.6.0`` with GNU, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: .. code-block:: console - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core module load stack-gcc/9.2.0 module load stack-openmpi/4.1.5 module load stack-python/3.10.13 @@ -527,6 +527,9 @@ For ``spack-stack-1.6.0`` with GNU, proceed with loading the following modules: Note that on Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow01``). Users starting ``ecflow_server`` on the regular login nodes will see their servers being killed every few minutes, and may be barred from accessing the system. +.. note:: + spack-stack-1.7.0 on Hera provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `ue` in the above `module use` statements with `gsi-addon`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. + .. _Preconfigured_Sites_Jet: ------------------------------ @@ -543,26 +546,29 @@ The following is required for building new spack environments and for using spac module load ecflow/5.5.3 module use /lfs4/HFIP/hfv3gfs/role.epic/modulefiles -For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: .. code-block:: console - module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 module load stack-python/3.10.8 module available -For ``spack-stack-1.6.0`` with GNU, proceed with loading the following modules: +For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: .. code-block:: console - module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core module load stack-gcc/9.2.0 module load stack-openmpi/3.1.4 module load stack-python/3.10.8 module available +.. note:: + spack-stack-1.7.0 on Hercules provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `ue` in the above `module use` statements with `gsi-addon`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. + ------------------------------ UW (Univ. of Wisconsin) S4 ------------------------------ @@ -600,26 +606,11 @@ Amazon Web Services Parallelcluster Ubuntu 20.04 The JCSDA-managed AWS Parallel Cluster is currently unavailable. - -CI CLUSTER: /mnt/experiments-efs/skylab-v8/spack-stack-1.7.0-ci-c6i/envs/ue-intel-2021.6.0/install/modulefiles/Core - ----------------------------- Amazon Web Services Red Hat 8 ----------------------------- -Use a c6i.4xlarge instance or larger if running out of memory with AMI "skylab-7.1.0-redhat8" (see JEDI documentation at https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest for more information). - -For ``spack-stack-1.6.0``, run: - -.. code-block:: console - - ulimit -s unlimited - scl_source enable gcc-toolset-11 - module use /home/ec2-user/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core - module load stack-gcc/11.2.1 - module load stack-openmpi/4.1.5 - module load stack-python/3.10.13 - module available +The JCSDA-managed AWS Single Node AMI is currently unavailable. .. _Configurable_Sites_CreateEnv: From f705e8af13103b9e59285441d3546fc0135956c6 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sat, 6 Apr 2024 19:29:10 -0600 Subject: [PATCH 08/16] Update documentation for new Redhat 8 site configs --- doc/source/NewSiteConfigs.rst | 8 +++++--- doc/source/PreConfiguredSites.rst | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index 37afdb830..a0ecb388b 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -387,6 +387,8 @@ The following instructions were used to prepare a basic Red Hat 8 system as it i yum -y install automake yum -y install xorg-x11-xauth yum -y install xterm + yum -y install perl-IPC-Cmd + yum -y install gettext-devel yum -y install texlive # Do not install qt@5 for now @@ -514,8 +516,8 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi # JEDI-Skylab system (using R2D2 localhost) spack external find --scope system mysql + # Note - only needed for generating documentation spack external find --scope system texlive - spack external find --scope system sed 5. Find compilers, add to site config's ``compilers.yaml`` @@ -538,10 +540,10 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi spack config add "packages:all:compiler:[gcc@YOUR-VERSION]" # Example for Red Hat 8 following the above instructions - spack config add "packages:all:providers:mpi:[openmpi@4.1.6]" + spack config add "packages:all:providers:mpi:[openmpi@5.0.1]" # Example for Ubuntu 20.04 or 22.04 following the above instructions - spack config add "packages:all:providers:mpi:[mpich@4.1.1]" + spack config add "packages:all:providers:mpi:[mpich@4.1.2]" .. warning:: On some systems, the default compiler (e.g., ``gcc`` on Ubuntu 20) may not get used by spack if a newer version is found. Compare your entry to the output of the concretization step later and adjust the entry, if necessary. diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 09376ac5c..923525b0d 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -48,7 +48,7 @@ On selected systems, developmental versions / release candidates are installed t +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | **Cloud platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | AMI Red Hat 8 | GCC | *currently unavailable* | JCSDA | +| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.7.0/envs/unified-env-gcc-11.2.1`` | JCSDA | + Amazon Web Services +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Parallelcluster JCSDA R&D | GCC, Intel | *currently unavailable* | JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -610,8 +610,19 @@ The JCSDA-managed AWS Parallel Cluster is currently unavailable. Amazon Web Services Red Hat 8 ----------------------------- -The JCSDA-managed AWS Single Node AMI is currently unavailable. +Use a c6i.4xlarge instance or larger if running out of memory with AMI "skylab-8.0.0-redhat8" (see JEDI documentation at https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest for more information). +For ``spack-stack-1.7.0``, run: + +.. code-block:: console + ulimit -s unlimited + scl_source enable gcc-toolset-11 + module use /home/ec2-user/spack-stack/spack-stack-1.7.0/envs/unified-env-gcc-11.2.1/install/modulefiles/Core + module load stack-gcc/11.2.1 + module load stack-openmpi/4.1.5 + module load stack-python/3.10.13 + module available + .. _Configurable_Sites_CreateEnv: ======================== From 013375457f354043ff605a172894c9826c388091 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sat, 6 Apr 2024 19:36:27 -0600 Subject: [PATCH 09/16] Fix formatting in doc/source/PreConfiguredSites.rst --- doc/source/PreConfiguredSites.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 923525b0d..54c519994 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -615,6 +615,7 @@ Use a c6i.4xlarge instance or larger if running out of memory with AMI "skylab-8 For ``spack-stack-1.7.0``, run: .. code-block:: console + ulimit -s unlimited scl_source enable gcc-toolset-11 module use /home/ec2-user/spack-stack/spack-stack-1.7.0/envs/unified-env-gcc-11.2.1/install/modulefiles/Core From b2378cce47de3b0b47a91c408dceacae8d340408 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 8 Apr 2024 07:29:54 -0600 Subject: [PATCH 10/16] Update documentation for Orion for spack-stack-1.7.0 --- doc/source/PreConfiguredSites.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 54c519994..be54a23d9 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -22,7 +22,7 @@ On selected systems, developmental versions / release candidates are installed t +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Hercules | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs/ue-*`` | EPIC / JCSDA | | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env`` | EPIC / JCSDA | +| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.7.0/envs/ue-*`` | EPIC / JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Discover SCU16 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs/ue-*`` | JCSDA | | NASA +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -80,21 +80,21 @@ The following is required for building new spack environments and for using spac module load python/3.9.2 module load ecflow/5.8.4 -For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.7.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.7.0/envs/ue-intel-centos/install/modulefiles/Core module load stack-intel/2022.0.2 module load stack-intel-oneapi-mpi/2021.5.1 module load stack-python/3.10.13 module available -For ``spack-stack-1.6.0`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.7.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/ue-gcc-centos/install/modulefiles/Core module load stack-gcc/10.2.0 module load stack-openmpi/4.0.4 module load stack-python/3.10.13 @@ -620,7 +620,7 @@ For ``spack-stack-1.7.0``, run: scl_source enable gcc-toolset-11 module use /home/ec2-user/spack-stack/spack-stack-1.7.0/envs/unified-env-gcc-11.2.1/install/modulefiles/Core module load stack-gcc/11.2.1 - module load stack-openmpi/4.1.5 + module load stack-openmpi/5.0.1 module load stack-python/3.10.13 module available From 8c5ed686dd97eabdc66192cbebad3d78e129b022 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 8 Apr 2024 07:38:57 -0600 Subject: [PATCH 11/16] Final updates to doc/source/PreConfiguredSites.rst --- doc/source/PreConfiguredSites.rst | 98 +++++++++++++------------------ 1 file changed, 40 insertions(+), 58 deletions(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index be54a23d9..e55205642 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -15,45 +15,45 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful On selected systems, developmental versions / release candidates are installed that are newer than spack-stack 1.6.0 (see following table). For information on the spack-stack 1.6.0 releases on this platforms, please revert to version 1.6.0 of the documentation (https://spack-stack.readthedocs.io/en/1.6.0/PreConfiguredSites.html#pre-configured-sites-tier-1). -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| Organization | System | Compilers | Location | Maintainers (principal/backup)| -+=====================+==================================+=================+=========================================================================================================+===============================+ -| **HPC platforms** | -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Hercules | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs/ue-*`` | EPIC / JCSDA | -| MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.7.0/envs/ue-*`` | EPIC / JCSDA | -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Discover SCU16 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs/ue-*`` | JCSDA | -| NASA +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Discover SCU17 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-*`` | JCSDA | -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.7.0/envs/ue-gcc-12.2.0`` | JCSDA / EPIC | -| NCAR-Wyoming +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.7.0/envs/ue-*`` | EPIC / JCSDA | -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.7.0/envs/ue-intel*`` | NOAA-EMC | -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Gaea C5 | Intel | ``/ncrc/proj/epic/spack-stack/spack-stack-1.7.0/envs/ue-intel`` | EPIC / NOAA-EMC | -| +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-*`` | EPIC / NOAA-EMC | -| +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-*`` | EPIC / NOAA-EMC | -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | GCC, Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-`` | JCSDA / NRL | -| U.S. Navy (HPCMP) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0`` | JCSDA / NRL | -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| Univ. of Wisconsin | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0`` | JCSDA | -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| **Cloud platforms** | -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.7.0/envs/unified-env-gcc-11.2.1`` | JCSDA | -+ Amazon Web Services +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Parallelcluster JCSDA R&D | GCC, Intel | *currently unavailable* | JCSDA | -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.3.0`` | EPIC / JCSDA | -+---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ ++---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| Organization | System | Compilers | Location | Maintainers (principal/backup)| ++=====================+==================================+=================+==============================================================================+===============================+ +| **HPC platforms** | ++---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| | Hercules | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs`` | EPIC / JCSDA | +| MSU +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.7.0/envs`` | EPIC / JCSDA | ++---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| | Discover SCU16 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs`` | JCSDA | +| NASA +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| | Discover SCU17 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs`` | JCSDA | ++---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.7.0/envs`` | JCSDA / EPIC | +| NCAR-Wyoming +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| | Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.7.0/envs`` | EPIC / JCSDA | ++---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.7.0/envs`` | NOAA-EMC | ++---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| | Gaea C5 | Intel | ``/ncrc/proj/epic/spack-stack/spack-stack-1.7.0/envs`` | EPIC / NOAA-EMC | +| +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.7.0/envs`` | EPIC / NOAA-EMC | +| +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.7.0/envs`` | EPIC / NOAA-EMC | ++---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| | Narwhal | GCC, Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs`` | JCSDA / NRL | +| U.S. Navy (HPCMP) +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs`` | JCSDA / NRL | ++---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| Univ. of Wisconsin | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.7.0/envs`` | JCSDA | ++---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| **Cloud platforms** | ++---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.7.0/envs`` | JCSDA | ++ Amazon Web Services +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| | Parallelcluster JCSDA R&D | GCC, Intel | *currently unavailable* | JCSDA | ++---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ +| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.7.0/envs`` | EPIC / JCSDA | ++---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ For more information about a specific platform, please see the individual sections below. @@ -88,7 +88,6 @@ For ``spack-stack-1.7.0`` with Intel, load the following modules after loading m module load stack-intel/2022.0.2 module load stack-intel-oneapi-mpi/2021.5.1 module load stack-python/3.10.13 - module available For ``spack-stack-1.7.0`` with GNU, load the following modules after loading miniconda and ecflow: @@ -98,7 +97,6 @@ For ``spack-stack-1.7.0`` with GNU, load the following modules after loading min module load stack-gcc/10.2.0 module load stack-openmpi/4.0.4 module load stack-python/3.10.13 - module available .. note:: The unified environment on Orion uses ``cdo@2.0.5`` instead of the default ``cdo@2.2.0`` because of a bug in the ``cdo`` package recipe that affects systems that don't have a ``python3`` interpreter in the default search paths (see https://github.com/spack/spack/issues/41947) for more information. This is a temporary change on Orion for the spack-stack-1.7.0 release and will be reverted once the ``cdo`` package is updated in the upstream spack develop code. @@ -127,7 +125,6 @@ For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules module load stack-intel/2021.9.0 module load stack-intel-oneapi-mpi/2021.9.0 module load stack-python/3.10.13 - module available For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: @@ -137,7 +134,6 @@ For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: module load stack-gcc/12.2.0 module load stack-openmpi/4.1.6 module load stack-python/3.10.13 - module available .. note:: spack-stack-1.7.0 on Hercules provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `ue` in the above `module use` statements with `gsi-addon`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. @@ -166,7 +162,6 @@ For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules module load stack-intel/2021.6.0 module load stack-intel-oneapi-mpi/2021.6.0 module load stack-python/3.10.13 - module available For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: @@ -176,7 +171,6 @@ For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: module load stack-gcc/12.1.0 module load stack-openmpi/4.1.3 module load stack-python/3.10.13 - module available ------------------------------ NASA Discover SCU17 @@ -199,7 +193,6 @@ For ``spack-stack-1.7.0`` with Intel, load the following modules after loading e module load stack-intel/2021.10.0 module load stack-intel-oneapi-mpi/2021.10.0 module load stack-python/3.10.13 - module available For ``spack-stack-1.7.0`` with GNU, load the following modules after loading ecflow: @@ -209,7 +202,6 @@ For ``spack-stack-1.7.0`` with GNU, load the following modules after loading ecf module load stack-gcc/12.3.0 module load stack-openmpi/4.1.6 module load stack-python/3.10.13 - module available .. _Preconfigured_Sites_Narwhal: @@ -358,7 +350,6 @@ For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: module load stack-gcc/12.2.0 module load stack-openmpi/4.1.6 module load stack-python/3.10.13 - module available .. _Preconfigured_Sites_Derecho: @@ -385,7 +376,6 @@ For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules module load stack-intel/2021.10.0 module load stack-cray-mpich/8.1.25 module load stack-python/3.10.13 - module available For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: @@ -395,7 +385,6 @@ For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: module load stack-gcc/12.2.0 module load stack-cray-mpich/8.1.25 module load stack-python/3.10.13 - module available .. note:: CISL restricts the amount of memory available for processes on the login nodes. For example, it is impossible to compile JEDI with even one task (``make -j1``) with the Intel compiles in release mode (``-O2``). We therefore recommend compiling on compute nodes using interactive jobs, if possible. @@ -447,7 +436,6 @@ For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules module load stack-intel/2021.3.0 module load stack-intel-oneapi-mpi/2021.3.0 module load stack-python/3.10.13 - module available .. _Preconfigured_Sites_Gaea_C5: @@ -513,7 +501,6 @@ For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 module load stack-python/3.10.13 - module available For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: @@ -523,7 +510,6 @@ For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: module load stack-gcc/9.2.0 module load stack-openmpi/4.1.5 module load stack-python/3.10.13 - module available Note that on Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow01``). Users starting ``ecflow_server`` on the regular login nodes will see their servers being killed every few minutes, and may be barred from accessing the system. @@ -554,7 +540,6 @@ For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 module load stack-python/3.10.8 - module available For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: @@ -564,7 +549,6 @@ For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: module load stack-gcc/9.2.0 module load stack-openmpi/3.1.4 module load stack-python/3.10.8 - module available .. note:: spack-stack-1.7.0 on Hercules provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `ue` in the above `module use` statements with `gsi-addon`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. @@ -592,7 +576,6 @@ For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules module load stack-python/3.10.13 module unuse /opt/apps/modulefiles/Compiler/intel/non-default/22 module unuse /opt/apps/modulefiles/Compiler/intel/22 - module available Note the two `module unuse` commands, that need to be run after the stack metamodules are loaded. Loading the Intel compiler meta module loads the Intel compiler module provided by the sysadmins, which adds those two directories to the module path. These contain duplicate libraries that are not compatible with our stack, such as ``hdf4``. @@ -622,8 +605,7 @@ For ``spack-stack-1.7.0``, run: module load stack-gcc/11.2.1 module load stack-openmpi/5.0.1 module load stack-python/3.10.13 - module available - + .. _Configurable_Sites_CreateEnv: ======================== From 0fc2e204f5d2e87634cf66910c835202a7ca1966 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 8 Apr 2024 12:57:48 -0600 Subject: [PATCH 12/16] Switch to tag spack-stack-1.7.0 for spack submodule --- .gitmodules | 2 +- configs/containers/docker-ubuntu-clang-mpich.yaml | 2 +- configs/containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- configs/containers/docker-ubuntu-intel-impi.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index 63b4600ef..bfb8c7a2c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "spack"] path = spack url = https://github.com/jcsda/spack - branch = release/1.7.0 + branch = spack-stack-1.7.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index 1d96283c8..433c0f1c7 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -115,7 +115,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: release/1.7.0 + ref: spack-stack-1.7.0 resolve_sha: true # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 1a50ac134..97682077f 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -96,7 +96,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: release/1.7.0 + ref: spack-stack-1.7.0 resolve_sha: true # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 20948f971..9bb90e8d5 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -115,7 +115,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: release/1.7.0 + ref: spack-stack-1.7.0 resolve_sha: true # Whether or not to strip binaries From 2e619b531017191ef3b8a4db36d65f03dabad599 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 8 Apr 2024 13:19:59 -0600 Subject: [PATCH 13/16] Update doc/source/PreConfiguredSites.rst with information on mapl version/variant differences --- doc/source/PreConfiguredSites.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index e55205642..752b44c95 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -59,6 +59,24 @@ For more information about a specific platform, please see the individual sectio For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. +.. note:: + This release of spack-stack uses different versions of ``mapl`` with different variants, depending on the version of the compiler and whether the system is used for UFS or GEOS. Please see the following table. + ++----------------------------+--------------------------------------+-----------------------------------------------------------------------+ +| Compiler | mapl configuration | Affected systems | ++============================+======================================+=======================================================================+ +| gcc (any) | ``mapl@2.40.3 +pflogger +extdata2g`` | All systems with GCC stacks | +-----------------------------+--------------------------------------+-----------------------------------------------------------------------+ +| intel@2021.6.0 and earlier | ``mapl@2.40.3 +pflogger +extdata2g`` | Discover SCU16, Acorn, Hera, Jet, Narwhal, Nautilus, S4, RDHPCS Cloud | +-----------------------------+--------------------------------------+-----------------------------------------------------------------------+ +| intel@2021.7.0 and later | ``mapl@2.40.3 ~pflogger ~extdata2g`` | Hercules, Orion, Acorn(?), Gaea and Derecho | +-----------------------------+--------------------------------------+-----------------------------------------------------------------------+ +| intel@2021.7.0 and later | ``mapl@2.43.0 +pflogger +extdata2g`` | Discover SCU17 | +-----------------------------+--------------------------------------+-----------------------------------------------------------------------+ + +.. note:: + We have noted problems on some - not all - platforms with ``intel@2021.5.0`` when we switched from ``zlib`` to ``zlib-ng`` in spack-stack-1.7.0. These issues went away when using a different version of the compiler (anything between 2021.3.0 and 2021.11.0). It is therefore recommended to avoid using ``intel@2021.5.0`` unless it is the only option. + .. _Preconfigured_Sites_Tier1: ============================================================= From 8221b7b40d6fa1ac02ef1745ccbb21118bce6865 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 8 Apr 2024 13:24:30 -0600 Subject: [PATCH 14/16] Fix formatting of mapl table in doc/source/PreConfiguredSites.rst --- doc/source/PreConfiguredSites.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 752b44c95..932d9dc4a 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -66,13 +66,13 @@ For questions or problems, please consult the known issues in :numref:`Section % | Compiler | mapl configuration | Affected systems | +============================+======================================+=======================================================================+ | gcc (any) | ``mapl@2.40.3 +pflogger +extdata2g`` | All systems with GCC stacks | ------------------------------+--------------------------------------+-----------------------------------------------------------------------+ ++----------------------------+--------------------------------------+-----------------------------------------------------------------------+ | intel@2021.6.0 and earlier | ``mapl@2.40.3 +pflogger +extdata2g`` | Discover SCU16, Acorn, Hera, Jet, Narwhal, Nautilus, S4, RDHPCS Cloud | ------------------------------+--------------------------------------+-----------------------------------------------------------------------+ ++----------------------------+--------------------------------------+-----------------------------------------------------------------------+ | intel@2021.7.0 and later | ``mapl@2.40.3 ~pflogger ~extdata2g`` | Hercules, Orion, Acorn(?), Gaea and Derecho | ------------------------------+--------------------------------------+-----------------------------------------------------------------------+ ++----------------------------+--------------------------------------+-----------------------------------------------------------------------+ | intel@2021.7.0 and later | ``mapl@2.43.0 +pflogger +extdata2g`` | Discover SCU17 | ------------------------------+--------------------------------------+-----------------------------------------------------------------------+ ++----------------------------+--------------------------------------+-----------------------------------------------------------------------+ .. note:: We have noted problems on some - not all - platforms with ``intel@2021.5.0`` when we switched from ``zlib`` to ``zlib-ng`` in spack-stack-1.7.0. These issues went away when using a different version of the compiler (anything between 2021.3.0 and 2021.11.0). It is therefore recommended to avoid using ``intel@2021.5.0`` unless it is the only option. From d3531e865505887ce2373f53a2d3a74d26fa0258 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 9 Apr 2024 09:19:50 -0600 Subject: [PATCH 15/16] Fix remaining 1.6.0 notes in doc/source/PreConfiguredSites.rst --- doc/source/PreConfiguredSites.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 932d9dc4a..57b1dc67a 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -8,12 +8,10 @@ Directory ``configs/sites`` contains site configurations for several HPC systems Pre-configured sites are split into two categories: Tier 1 with officially supported spack-stack installations (see :numref:`Section %s `), and Tier 2 (sites with configuration files that were tested or contributed by others in the past, but that are not officially supported by the spack-stack team; see :numref:`Section %s `). ============================================================= -Officially supported spack-stack 1.6.0 installations (tier 1) +Officially supported spack-stack 1.7.0 installations (tier 1) ============================================================= -Ready-to-use spack-stack 1.6.0 installations are available on the following, fully supported platforms. This version supports JEDI-Skylab and various UFS Applications (UFS Weather Model, EMC Global Workflow, GSI, UFS Short Range Weather Application). Amazon Web Services AMI are available in the US East 1 or 2 regions. - -On selected systems, developmental versions / release candidates are installed that are newer than spack-stack 1.6.0 (see following table). For information on the spack-stack 1.6.0 releases on this platforms, please revert to version 1.6.0 of the documentation (https://spack-stack.readthedocs.io/en/1.6.0/PreConfiguredSites.html#pre-configured-sites-tier-1). +Ready-to-use spack-stack 1.7.0 installations are available on the following, fully supported platforms. This version supports JEDI-Skylab and various UFS Applications (UFS Weather Model, EMC Global Workflow, GSI, UFS Short Range Weather Application). Amazon Web Services AMI are available in the US East 1 or 2 regions. +---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ | Organization | System | Compilers | Location | Maintainers (principal/backup)| @@ -111,7 +109,7 @@ For ``spack-stack-1.7.0`` with GNU, load the following modules after loading min .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/ue-gcc-centos/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.7.0/envs/ue-gcc-centos/install/modulefiles/Core module load stack-gcc/10.2.0 module load stack-openmpi/4.0.4 module load stack-python/3.10.13 From 0a34d3543902979eb6e6e155c9faa63b146c1816 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 9 Apr 2024 13:38:08 -0600 Subject: [PATCH 16/16] Remove question mark in doc/source/PreConfiguredSites.rst on acorn-mapl --- doc/source/PreConfiguredSites.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 57b1dc67a..b7aec5771 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -67,7 +67,7 @@ For questions or problems, please consult the known issues in :numref:`Section % +----------------------------+--------------------------------------+-----------------------------------------------------------------------+ | intel@2021.6.0 and earlier | ``mapl@2.40.3 +pflogger +extdata2g`` | Discover SCU16, Acorn, Hera, Jet, Narwhal, Nautilus, S4, RDHPCS Cloud | +----------------------------+--------------------------------------+-----------------------------------------------------------------------+ -| intel@2021.7.0 and later | ``mapl@2.40.3 ~pflogger ~extdata2g`` | Hercules, Orion, Acorn(?), Gaea and Derecho | +| intel@2021.7.0 and later | ``mapl@2.40.3 ~pflogger ~extdata2g`` | Hercules, Orion, Acorn, Gaea and Derecho | +----------------------------+--------------------------------------+-----------------------------------------------------------------------+ | intel@2021.7.0 and later | ``mapl@2.43.0 +pflogger +extdata2g`` | Discover SCU17 | +----------------------------+--------------------------------------+-----------------------------------------------------------------------+