diff --git a/.gitignore b/.gitignore index f32b5b0ea..2a2a5fdb6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ doc/build # Ignore macOS cache files .DS_Store + # Ignore testsuite file/directories testsuite* caselist* diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 0450d2647..00280de06 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -213,7 +213,20 @@ cat >> ${jobfile} << EOFB #SBATCH -N ${nnodes} #SBATCH -e slurm%j.err #SBATCH -o slurm%j.out -#SBATCH --mail-type END,FAIL +#SBATCH --mail-type FAIL +#SBATCH --mail-user=robert.grumbine@noaa.gov +EOFB + +else if (${ICE_MACHINE} =~ hera*) then +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH -t `echo ${batchtime} | cut -f1-2 -d:` +#SBATCH -q batch +#SBATCH -A marine-cpu +#SBATCH -N ${nnodes} +#SBATCH -e slurm%j.err +#SBATCH -o slurm%j.out +#SBATCH --mail-type FAIL #SBATCH --mail-user=robert.grumbine@noaa.gov EOFB @@ -223,8 +236,25 @@ cat >> ${jobfile} << EOFB EOFB else if (${ICE_MACHINE} =~ phase3*) then +if ( ${nnodes} > 15) then + setenv p3tile 16 + setenv mem `expr 100 \* 1024 / $nnodes` +else + setenv p3tile ${nnodes} + setenv mem 8192 +endif +echo mem = ${mem} nnodes and p3tiles ${nnodes} ${p3tile} p3tile must be le nnodes cat >> ${jobfile} << EOFB -# nothing to do +#BSUB -J ${ICE_CASENAME} +#BSUB -q "dev_shared" +#BSUB -P RTO-T2O +#BSUB -W `echo ${batchtime} | cut -f1-2 -d:` +#BSUB -n ${nnodes} +#BSUB -R "affinity[core]" +#BSUB -R "span[ptile=${p3tile}]" +#BSUB -R "rusage[mem=${mem}]" +#BSUB -o /u/Robert.Grumbine/${ICE_CASENAME}.out.%J +#BSUB -e /u/Robert.Grumbine/${ICE_CASENAME}.err.%J EOFB else if (${ICE_MACHINE} =~ high_Sierra*) then diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index 17a260e06..3e5c3500f 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -197,6 +197,20 @@ cat >> ${jobfile} << EOFR ./cice >&! \$ICE_RUNLOG_FILE EOFR +#======= +else if (${ICE_MACHINE} =~ phase3*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR + +#======= +else if (${ICE_MACHINE} =~ phase2*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR + +======= + #======= else if (${ICE_MACHINE} =~ travisCI*) then if (${ICE_COMMDIR} =~ serial*) then diff --git a/configuration/scripts/machines/Macros.hera_intel b/configuration/scripts/machines/Macros.hera_intel new file mode 100644 index 000000000..38cc62ed8 --- /dev/null +++ b/configuration/scripts/machines/Macros.hera_intel @@ -0,0 +1,60 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost + +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) +#RG: this looks more like gfortran options: +## FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + FFLAGS += -O0 -g -check-bounds -implicit-none -fpe3 +else + FFLAGS += -O2 +endif + +#Note that on theia mpif90 refers to gfortran even though it is in the intel bin +FC := mpiifort +MPIFC:= mpiifort +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := + + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/env.hera_intel b/configuration/scripts/machines/env.hera_intel new file mode 100755 index 000000000..ef3afdb7a --- /dev/null +++ b/configuration/scripts/machines/env.hera_intel @@ -0,0 +1,26 @@ +#!/bin/csh -f + +source /etc/profile.d/modules.csh +#module list +module purge +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load esmf/7.1.0r +module load hdf5/1.10.4 netcdf/4.6.1 +module load wgrib wgrib2 +#echo renewed modules: +#module list + +setenv ICE_MACHINE_ENVNAME hera +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/rgdev/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /home/Robert.Grumbine/rgdev/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/rgdev/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "batch" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +#setenv ICE_MACHINE_QUIETMODE true diff --git a/configuration/scripts/machines/env.high_Sierra_gnu b/configuration/scripts/machines/env.high_Sierra_gnu index cabd66944..db8bc01a3 100755 --- a/configuration/scripts/machines/env.high_Sierra_gnu +++ b/configuration/scripts/machines/env.high_Sierra_gnu @@ -4,7 +4,7 @@ setenv ICE_MACHINE_ENVNAME high_Sierra setenv ICE_MACHINE_COMPILER gnu setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR /Volumes/ncep/CICE_RUNS -setenv ICE_MACHINE_INPUTDATA /Volumes/Data/CICE_data +setenv ICE_MACHINE_INPUTDATA /Volumes/Data/ setenv ICE_MACHINE_BASELINE /Volumes/ncep/CICE_BASELINE setenv ICE_MACHINE_SUBMIT " " setenv ICE_MACHINE_TPNODE 4 diff --git a/configuration/scripts/machines/env.phase3_intel b/configuration/scripts/machines/env.phase3_intel index 03aa7083f..4c8f65eee 100755 --- a/configuration/scripts/machines/env.phase3_intel +++ b/configuration/scripts/machines/env.phase3_intel @@ -1,8 +1,12 @@ -#!/bin/csh -f --login +#!/bin/csh -f -source /etc/profile.d/lmod.csh +#. $MODULESHOME/init/csh +source $MODULESHOME/init/csh +echo done with init.csh +module use -a $MODULESHOME/modulefiles module purge +module load lsf/10.1 module load ips/18.0.1.163 module load impi/18.0.1 module load NetCDF/4.5.0 @@ -15,10 +19,10 @@ setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /u/Robert.Grumbine/noscrub/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /u/Robert.Grumbine/noscrub/ setenv ICE_MACHINE_BASELINE /u/Robert.Grumbine/noscrub/CICE_BASELINE -setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_SUBMIT "bsub < " setenv ICE_MACHINE_TPNODE 4 setenv ICE_MACHINE_QUEUE "default" setenv ICE_MACHINE_ACCT P0000000 setenv ICE_MACHINE_BLDTHRDS 1 -setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QSTAT "bjobs "