From 6ecd128bfa99daccba2a5b263b11c658e806adc6 Mon Sep 17 00:00:00 2001 From: Dave Gill Date: Tue, 14 Jan 2020 16:06:40 -0700 Subject: [PATCH 1/2] Allow > 9999 MPI processes TYPE: enhancement KEYWORDS: MPI, rsl SOURCE: internal DESCRIPTION OF CHANGES: Problem: Previously, when the model requested more than 9999 MPI processes, the attempt to generate the fielnames for the out and error files from processes 10000 and up, caused the model to ungracefully stop. Previously, when the model requested more that 10000 MPI processes, a maximum number of MPI slots was exceeded. Solution: 1. The file name is now a function of the total number of MPI processes that are requested. When requeted 1 through 9999 MPI processes (inclusive), the files are named as before: rsl.out.0000 through rsl.out.9999. When the total number of MPI processes > 9999 (for example, mpirun -np 10000 wrf.exe), then the file names become rsl.out.00000000 through rsl.out.00010000. 2. A hard-coded maximum value of 10000 MPI processes in a header file was removed, and replaced with a more realistic value of 100000. 3. For users with very large MPI process counts, the WRF model has an existing ability to remove all standard out and error files except for the master process. That compile-time option is now easily accessible in the configure.defaults file. This mod is not a change in function, but an easy way to get access to this single-file option. LIST OF MODIFIED FILES: modified: ../../arch/configure.defaults modified: ../../external/RSL_LITE/c_code.c modified: ../../external/RSL_LITE/module_dm.F modified: ../../external/RSL_LITE/rsl_lite.h TESTS CONDUCTED: 1. To test the logic of a large number of MPI processes modifying the rsl file names, a temporary change was introduced in the code. Instead of the cutoff being 10000 MPI processes, the value chosen was 3. Any MPI job with 1 or 2 processes should then produce file names such as rsl.out.0000, and any MPI job with 3 or more MPI processes should produce a file name such as rsl.out.00000000. Here is a small piece of the logic. ``` if ( *size < ORIG_RSL_CUTOFF ) { sprintf(filename,"rsl.out.%04d",*me) ; } else { sprintf(filename,"rsl.out.%08d",*me) ; } ``` Here are tests with the maximum MPI process count at 1, 2, and 3. ``` > rm rsl* ; mpirun -np 1 real.exe ; ls -ls rsl* starting wrf task 0 of 1 16 -rw-r--r-- 1 gill 1500 6903 Jan 14 15:58 rsl.error.0000 40 -rw-r--r-- 1 gill 1500 19854 Jan 14 15:58 rsl.out.0000 ``` ``` > rm rsl* ; mpirun -np 2 real.exe ; ls -ls rsl* starting wrf task 0 of 2 starting wrf task 1 of 2 16 -rw-r--r-- 1 gill 1500 6903 Jan 14 15:58 rsl.error.0000 16 -rw-r--r-- 1 gill 1500 6699 Jan 14 15:58 rsl.error.0001 40 -rw-r--r-- 1 gill 1500 19854 Jan 14 15:58 rsl.out.0000 40 -rw-r--r-- 1 gill 1500 19650 Jan 14 15:58 rsl.out.0001 ``` ``` > rm rsl* ; mpirun -np 3 real.exe ; ls -ls rsl* starting wrf task 1 of 3 starting wrf task 2 of 3 starting wrf task 0 of 3 16 -rw-r--r-- 1 gill 1500 6903 Jan 14 15:58 rsl.error.00000000 16 -rw-r--r-- 1 gill 1500 6343 Jan 14 15:58 rsl.error.00000001 16 -rw-r--r-- 1 gill 1500 6699 Jan 14 15:58 rsl.error.00000002 40 -rw-r--r-- 1 gill 1500 19854 Jan 14 15:58 rsl.out.00000000 40 -rw-r--r-- 1 gill 1500 19294 Jan 14 15:58 rsl.out.00000001 40 -rw-r--r-- 1 gill 1500 19650 Jan 14 15:58 rsl.out.00000002 ``` RELEASE NOTE: The WRF model is now able to use > 9999 MPI processes. Users will notice that the file names are modified jobs request > 9999 MPI processes. For example, the command "mpirun -np 10000 wrf.exe" will produce file names rsl.out.00000000 through rsl.out.00010000. --- arch/configure.defaults | 80 ++++++++++++++++----------------- external/RSL_LITE/c_code.c | 85 +++++++++++++++++++++++++++-------- external/RSL_LITE/module_dm.F | 2 +- external/RSL_LITE/rsl_lite.h | 2 +- 4 files changed, 109 insertions(+), 60 deletions(-) diff --git a/arch/configure.defaults b/arch/configure.defaults index 45e1079dd4..f89d13a789 100644 --- a/arch/configure.defaults +++ b/arch/configure.defaults @@ -17,7 +17,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = #-fdefault-real-8 ARCH_LOCAL = -DNEC -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -c +CFLAGS_LOCAL = -c # -DRSL0_ONLY #-DNCARIBM_NOC99 -Xa -Kc99 LDFLAGS_LOCAL = -Wl,-h nodefs CPLUSPLUSLIB = @@ -62,7 +62,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = #-fdefault-real-8 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -c +CFLAGS_LOCAL = -w -O3 -c # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -105,7 +105,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -r$(RWORDSIZE) -i4 ARCH_LOCAL = -DF2CSTYLE -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -DF2CSTYLE +CFLAGS_LOCAL = -DF2CSTYLE # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -148,7 +148,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -r$(RWORDSIZE) -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 +CFLAGS_LOCAL = -w -O3 # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -191,7 +191,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -r$(RWORDSIZE) -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 +CFLAGS_LOCAL = -w -O3 # -DRSL0_ONLY LDFLAGS_LOCAL = -L$(MPI_ROOT)/lib -lmpi CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -234,7 +234,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -r$(RWORDSIZE) -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -D_ACCEL -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 +CFLAGS_LOCAL = -w -O3 # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -309,7 +309,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -real-size `expr 8 \* $(RWORDSIZE)` -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_FUNC -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars +CFLAGS_LOCAL = -w -O3 -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars # -DRSL0_ONLY LDFLAGS_LOCAL = -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -align all -fno-alias -fno-common CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -356,7 +356,7 @@ PROMOTION = -real-size `expr 8 \* $(RWORDSIZE)` -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_FUNC -DCHUNK=16 -DXEON_OPTIMIZED_WSM5 -DXEON_SIMD -DOPTIMIZE_CFL_TEST -DFSEEKO64_OK -DINTEL_YSU_KLUDGE -DWRF_USE_CLM OPTNOSIMD = OPTKNC = -fimf-precision=low -fimf-domain-exclusion=15 -opt-assume-safe-padding -opt-streaming-stores always -opt-streaming-cache-evict=0 -mP2OPT_hlo_pref_use_outer_strategy=F -CFLAGS_LOCAL = -w -O3 $(OPTKNC) +CFLAGS_LOCAL = -w -O3 $(OPTKNC) # -DRSL0_ONLY LDFLAGS_LOCAL = $(OPTKNC) CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -403,7 +403,7 @@ PROMOTION = -real-size `expr 8 \* $(RWORDSIZE)` -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_FUNC -DCHUNK=64 -DXEON_OPTIMIZED_WSM5 -DOPTIMIZE_CFL_TEST -DWRF_USE_CLM OPTNOSIMD = OPTAVX = -xAVX -CFLAGS_LOCAL = -w -O3 $(OPTAVX) +CFLAGS_LOCAL = -w -O3 $(OPTAVX) # -DRSL0_ONLY LDFLAGS_LOCAL = $(OPTAVX) CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -474,7 +474,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -real-size `expr 8 \* $(RWORDSIZE)` -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_FUNC -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars +CFLAGS_LOCAL = -w -O3 -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars # -DRSL0_ONLY LDFLAGS_LOCAL = -ip -lmpi #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -align all -fno-alias -fno-common CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -523,7 +523,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -real-size `expr 8 \* $(RWORDSIZE)` -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_FUNC -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars +CFLAGS_LOCAL = -w -O3 -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars # -DRSL0_ONLY LDFLAGS_LOCAL = -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -align all -fno-alias -fno-common CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -602,7 +602,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -real-size `expr 8 \* $(RWORDSIZE)` -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_FUNC -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars +CFLAGS_LOCAL = -w -O3 -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars # -DRSL0_ONLY LDFLAGS_LOCAL = -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -align all -fno-alias -fno-common CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -685,7 +685,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -real-size `expr 8 \* $(RWORDSIZE)` -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_FUNC -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars +CFLAGS_LOCAL = -w -O3 -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars # -DRSL0_ONLY LDFLAGS_LOCAL = -ip #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -align all -fno-alias -fno-common CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -731,7 +731,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -r$(RWORDSIZE) -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -D__PATHSCALE__ -CFLAGS_LOCAL = +CFLAGS_LOCAL = # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -774,7 +774,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = #-fdefault-real-8 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -c +CFLAGS_LOCAL = -w -O3 -c # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -817,7 +817,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -r$(RWORDSIZE) -i4 ARCH_LOCAL = -DMACOS -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -DMACOS +CFLAGS_LOCAL = -DMACOS # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -860,7 +860,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -real-size `expr 8 \* $(RWORDSIZE)` -i4 ARCH_LOCAL = -DMACOS -DNONSTANDARD_SYSTEM_FUNC -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -ip -DMACOS #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars -DMACOS +CFLAGS_LOCAL = -w -O3 -ip -DMACOS #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars -DMACOS # -DRSL0_ONLY # increase stack size; also note that for OpenMP, set environment OMP_STACKSIZE 4G or greater LDFLAGS_LOCAL = -ip -Wl,-stack_addr,0xF10000000 -Wl,-stack_size,0x64000000 #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -align all -fno-alias -fno-common CPLUSPLUSLIB = @@ -906,7 +906,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -real-size `expr 8 \* $(RWORDSIZE)` -i4 ARCH_LOCAL = -DMACOS -DNONSTANDARD_SYSTEM_FUNC -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -DMACOS #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars -DMACOS +CFLAGS_LOCAL = -w -O3 -DMACOS #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars -DMACOS # -DRSL0_ONLY # increase stack size; also note that for OpenMP, set environment OMP_STACKSIZE 4G or greater LDFLAGS_LOCAL = -Wl,-stack_addr,0xF10000000 -Wl,-stack_size,0x64000000 #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -align all -fno-alias -fno-common CPLUSPLUSLIB = @@ -951,7 +951,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -r$(RWORDSIZE) -i4 ARCH_LOCAL = -DG95 -DMACOS -DF2CSTYLE -DNONSTANDARD_SYSTEM_SUBR -DRCONFIG_CHARLEN=64 -DWRF_USE_CLM -CFLAGS_LOCAL = -DMACOS -DF2CSTYLE +CFLAGS_LOCAL = -DMACOS -DF2CSTYLE # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -995,7 +995,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = #-fdefault-real-8 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DMACOS -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -c -DMACOS +CFLAGS_LOCAL = -w -O3 -c -DMACOS # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1038,7 +1038,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = #-fdefault-real-8 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DMACOS -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -c -DMACOS +CFLAGS_LOCAL = -w -O3 -c -DMACOS # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1081,7 +1081,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -qrealsize=$(RWORDSIZE) -qintsize=4 ARCH_LOCAL = -DMAC_KLUDGE -DF2CSTYLE -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -DMACOS -DF2CSTYLE +CFLAGS_LOCAL = -DMACOS -DF2CSTYLE # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1127,7 +1127,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -qrealsize=$(RWORDSIZE) -qintsize=4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DNATIVE_MASSV -DWRF_USE_CLM -CFLAGS_LOCAL = -DNOUNDERSCORE +CFLAGS_LOCAL = -DNOUNDERSCORE # -DRSL0_ONLY LDFLAGS_LOCAL = -lmass -lmassv -bnoquiet # print diagnostic messages CPLUSPLUSLIB = -lC ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1175,7 +1175,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -qrealsize=$(RWORDSIZE) -qintsize=4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DNATIVE_MASSV -DWRF_USE_CLM -CFLAGS_LOCAL = -DNOUNDERSCORE +CFLAGS_LOCAL = -DNOUNDERSCORE # -DRSL0_ONLY LDFLAGS_LOCAL = -lmass_64 -lmassvp7_64 -q64 -bnoquiet # linking diagnostics CPLUSPLUSLIB = -lC ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1229,7 +1229,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -r$(RWORDSIZE) -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -CFLAGS_LOCAL = -w -O3 +CFLAGS_LOCAL = -w -O3 # -DRSL0_ONLY LDFLAGS_LOCAL = # module load libfast to use Cray XT fast math library #LIB_LOCAL = -lfast_mv @@ -1291,7 +1291,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -s integer32 -s real`expr 8 \* $(RWORDSIZE)` ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -O3 +CFLAGS_LOCAL = -O3 # -DRSL0_ONLY LDFLAGS_LOCAL = # uncomment this for wrfda build #LIB_LOCAL = -L$(WRF_SRC_ROOT_DIR)/external/fftpack/fftpack5 -lfftpack \ @@ -1341,7 +1341,7 @@ ARCH_LOCAL = -DNONSTANDARD_SYSTEM_FUNC -DWRF_USE_CLM OPTNOSIMD = # set this to override Cray 'craype' module setting #OPTAVX = -xAVX -CFLAGS_LOCAL = -w -O3 -ip $(OPTAVX) +CFLAGS_LOCAL = -w -O3 -ip $(OPTAVX) # -DRSL0_ONLY LDFLAGS_LOCAL = $(OPTAVX) CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1391,7 +1391,7 @@ RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -qrealsize=$(RWORDSIZE) -qintsize=4 # If system has even more processors, set VERY_LARGE_MAXPROC to that number ARCH_LOCAL = -DMOVE_NL_OUTSIDE_MODULE_CONFIGURE -DNONSTANDARD_SYSTEM_SUBR -DVERY_LARGE_MAXPROC=36768 -DBLUEGENE -DWRF_USE_CLM -CFLAGS_LOCAL = -DNOUNDERSCORE -DNCARIBM_NOC99 $(MPI_INC) +CFLAGS_LOCAL = -DNOUNDERSCORE -DNCARIBM_NOC99 $(MPI_INC) # -DRSL0_ONLY LIB_LOCAL = $(MPI_LIB) LDFLAGS_LOCAL = -Wl,--allow-multiple-definition -qstatic CPLUSPLUSLIB = @@ -1440,7 +1440,7 @@ RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -qrealsize=$(RWORDSIZE) -qintsize=4 # If system has even more processors, set VERY_LARGE_MAXPROC to that number ARCH_LOCAL = -DMOVE_NL_OUTSIDE_MODULE_CONFIGURE -DNONSTANDARD_SYSTEM_SUBR -DVERY_LARGE_MAXPROC=36768 -DBLUEGENE -DWRF_USE_CLM -CFLAGS_LOCAL = -DNOUNDERSCORE +CFLAGS_LOCAL = -DNOUNDERSCORE # -DRSL0_ONLY LIB_LOCAL = LDFLAGS_LOCAL = -Wl,--allow-multiple-definition,--relax -qstatic CPLUSPLUSLIB = @@ -1486,7 +1486,7 @@ RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -qrealsize=$(RWORDSIZE) -qintsize=4 # If system has even more processors, set VERY_LARGE_MAXPROC to that number ARCH_LOCAL = -DMOVE_NL_OUTSIDE_MODULE_CONFIGURE -DNONSTANDARD_SYSTEM_SUBR -DVERY_LARGE_MAXPROC=36768 -DWRF_USE_CLM -CFLAGS_LOCAL = -DNOUNDERSCORE +CFLAGS_LOCAL = -DNOUNDERSCORE # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = -lC ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1529,7 +1529,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -r$(RWORDSIZE) -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 +CFLAGS_LOCAL = -w -O3 # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1572,7 +1572,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -r$(RWORDSIZE) -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -D_WIN32 -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -DMEMCPY_FOR_BCOPY +CFLAGS_LOCAL = -w -O3 -DMEMCPY_FOR_BCOPY # -DRSL0_ONLY LDFLAGS_LOCAL = Ws2_32.lib # -lnetcdff CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1629,7 +1629,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -r$(RWORDSIZE) -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 +CFLAGS_LOCAL = -w -O3 # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1672,7 +1672,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -r$(RWORDSIZE) -i4 ARCH_LOCAL = -DMACOS -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -DMACOS +CFLAGS_LOCAL = -DMACOS # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1715,7 +1715,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -real-size `expr 8 \* $(RWORDSIZE)` -i4 ARCH_LOCAL = -DMACOS -DNONSTANDARD_SYSTEM_FUNC -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -ip -DMACOS #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars -DMACOS +CFLAGS_LOCAL = -w -O3 -ip -DMACOS #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars -DMACOS # -DRSL0_ONLY # increase stack size; also note that for OpenMP, set environment OMP_STACKSIZE 4G or greater LDFLAGS_LOCAL = -ip -Wl,-stack_addr,0xF10000000 -Wl,-stack_size,0x64000000 #-xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -align all -fno-alias -fno-common CPLUSPLUSLIB = @@ -1761,7 +1761,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = #-fdefault-real-8 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DMACOS -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -c -DMACOS +CFLAGS_LOCAL = -w -O3 -c -DMACOS # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1804,7 +1804,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -r$(RWORDSIZE) -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 +CFLAGS_LOCAL = -w -O3 # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1847,7 +1847,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -real-size `expr 8 \* $(RWORDSIZE)` -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_FUNC -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -ip -xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars -xCORE-AVX2 +CFLAGS_LOCAL = -w -O3 -ip -xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars -xCORE-AVX2 # -DRSL0_ONLY LDFLAGS_LOCAL = -ip -xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -align all -fno-alias -fno-common -xCORE-AVX2 CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1891,7 +1891,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -real-size `expr 8 \* $(RWORDSIZE)` -i4 ARCH_LOCAL = -DNONSTANDARD_SYSTEM_FUNC -DWRF_USE_CLM -CFLAGS_LOCAL = -w -O3 -ip -xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars -xMIC-AVX512 +CFLAGS_LOCAL = -w -O3 -ip -xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -no-multibyte-chars -xMIC-AVX512 # -DRSL0_ONLY LDFLAGS_LOCAL = -ip -xHost -fp-model fast=2 -no-prec-div -no-prec-sqrt -ftz -align all -fno-alias -fno-common -xMIC-AVX512 CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) @@ -1937,7 +1937,7 @@ LD = $(FC) RWORDSIZE = CONFIGURE_RWORDSIZE PROMOTION = -CcdRR$(RWORDSIZE) ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM -CFLAGS_LOCAL = -Kfast -Xg -DSUN +CFLAGS_LOCAL = -Kfast -Xg -DSUN # -DRSL0_ONLY LDFLAGS_LOCAL = CPLUSPLUSLIB = ESMF_LDFLAG = $(CPLUSPLUSLIB) diff --git a/external/RSL_LITE/c_code.c b/external/RSL_LITE/c_code.c index d5df731d25..27549bdf9c 100755 --- a/external/RSL_LITE/c_code.c +++ b/external/RSL_LITE/c_code.c @@ -30,7 +30,9 @@ #define F_PACK -void RSL_LITE_ERROR_DUP1 ( int *me ) +#define ORIG_RSL_CUTOFF 10000 + +void RSL_LITE_ERROR_DUP1 ( int *me , int *size ) { int newfd,rc ; char filename[256] ; @@ -44,16 +46,29 @@ void RSL_LITE_ERROR_DUP1 ( int *me ) /* redirect standard out*/ # ifndef RSL0_ONLY - sprintf(filename,"rsl.out.%04d",*me) ; + if ( *size < ORIG_RSL_CUTOFF ) + { + sprintf(filename,"rsl.out.%04d",*me) ; + } + else + { + sprintf(filename,"rsl.out.%08d",*me) ; + } # else if (*me == 0) - { - sprintf(filename,"rsl.out.%04d",*me) ; - } + { + if ( *size < ORIG_RSL_CUTOFF ) + { + sprintf(filename,"rsl.out.%04d",*me) ; + } + else { + sprintf(filename,"rsl.out.%08d",*me) ; + } + } else - { - sprintf(filename,"/dev/null") ; - } + { + sprintf(filename,"/dev/null") ; + } # endif if ((newfd = open( filename, O_CREAT | O_WRONLY | O_TRUNC, 0666 )) < 0 ) { @@ -74,16 +89,29 @@ void RSL_LITE_ERROR_DUP1 ( int *me ) if ( *me != 0 ) { /* stderr from task 0 should come to screen on windows because it is buffered if redirected */ #endif # ifndef RSL0_ONLY - sprintf(filename,"rsl.error.%04d",*me) ; + if ( *size < ORIG_RSL_CUTOFF ) + { + sprintf(filename,"rsl.error.%04d",*me) ; + } + else + { + sprintf(filename,"rsl.error.%08d",*me) ; + } # else if (*me == 0) - { - sprintf(filename,"rsl.error.%04d",*me) ; - } + { + if ( *size < ORIG_RSL_CUTOFF ) + { + sprintf(filename,"rsl.error.%04d",*me) ; + } + else { + sprintf(filename,"rsl.error.%08d",*me) ; + } + } else - { - sprintf(filename,"/dev/null") ; - } + { + sprintf(filename,"/dev/null") ; + } # endif if ((newfd = open( filename, O_CREAT | O_WRONLY | O_TRUNC, 0666 )) < 0 ) { @@ -134,7 +162,14 @@ void RSL_LITE_ERROR_DUP1 ( int *me ) /* TASKOUTPUT directory exists, continue with task specific directory */ - sprintf(dirname, "TASKOUTPUT/%04d", *me); + if ( *size < ORIG_RSL_CUTOFF ) + { + sprintf(dirname, "TASKOUTPUT/%04d", *me); + } + else + { + sprintf(dirname, "TASKOUTPUT/%08d", *me); + } rc=mkdir(dirname, 0777); if ( rc !=0 && errno!=EEXIST ) { perror("mkdir error"); @@ -144,7 +179,14 @@ void RSL_LITE_ERROR_DUP1 ( int *me ) /* Each tasks creates/opens its own output and error files */ - sprintf(filename, "%s/%04d/rsl.out.%04d","TASKOUTPUT",*me,*me) ; + if ( *size < ORIG_RSL_CUTOFF ) + { + sprintf(filename, "%s/%04d/rsl.out.%04d","TASKOUTPUT",*me,*me) ; + } + else + { + sprintf(filename, "%s/%08d/rsl.out.%08d","TASKOUTPUT",*me,*me) ; + } if ((newfd = open( filename, O_CREAT | O_WRONLY | O_TRUNC, 0666 )) < 0 ) { @@ -161,7 +203,14 @@ void RSL_LITE_ERROR_DUP1 ( int *me ) return ; } - sprintf(filename, "%s/%04d/rsl.error.%04d","TASKOUTPUT",*me,*me) ; + if ( *size < ORIG_RSL_CUTOFF ) + { + sprintf(filename, "%s/%04d/rsl.error.%04d","TASKOUTPUT",*me,*me) ; + } + else + { + sprintf(filename, "%s/%08d/rsl.error.%08d","TASKOUTPUT",*me,*me) ; + } if ((newfd = open( filename, O_CREAT | O_WRONLY | O_TRUNC, 0666 )) < 0 ) { perror("error_dup: cannot open ./TASKOUTPUT/nnnn/rsl.error.nnnn") ; diff --git a/external/RSL_LITE/module_dm.F b/external/RSL_LITE/module_dm.F index 2bb0367817..f317e93358 100644 --- a/external/RSL_LITE/module_dm.F +++ b/external/RSL_LITE/module_dm.F @@ -2647,7 +2647,7 @@ SUBROUTINE wrf_termio_dup( comm ) CALL mpi_comm_size(comm, ntasks, ierr ) CALL mpi_comm_rank(comm, mytask, ierr ) write(0,*)'starting wrf task ',mytask,' of ',ntasks - CALL rsl_error_dup1( mytask ) + CALL rsl_error_dup1( mytask, ntasks ) #else mytask = 0 ntasks = 1 diff --git a/external/RSL_LITE/rsl_lite.h b/external/RSL_LITE/rsl_lite.h index 042eb5320d..b99383a318 100644 --- a/external/RSL_LITE/rsl_lite.h +++ b/external/RSL_LITE/rsl_lite.h @@ -148,7 +148,7 @@ #define RSL_SENDBUF 0 #define RSL_RECVBUF 1 #define RSL_FREEBUF 3 -#define RSL_MAXPROC 10000 +#define RSL_MAXPROC 100000 #define RSL_INVALID -1 /* this must be the same as defined in frame/module_driver_constants.F */ From 68f08e6ead329fdb1e41dc63d327156c483c04c3 Mon Sep 17 00:00:00 2001 From: Dave Gill Date: Wed, 22 Jan 2020 09:14:14 -0700 Subject: [PATCH 2/2] 100001 is easier to find than 100000 --- external/RSL_LITE/rsl_lite.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/RSL_LITE/rsl_lite.h b/external/RSL_LITE/rsl_lite.h index b99383a318..5fcad78f5f 100644 --- a/external/RSL_LITE/rsl_lite.h +++ b/external/RSL_LITE/rsl_lite.h @@ -148,7 +148,7 @@ #define RSL_SENDBUF 0 #define RSL_RECVBUF 1 #define RSL_FREEBUF 3 -#define RSL_MAXPROC 100000 +#define RSL_MAXPROC 100001 #define RSL_INVALID -1 /* this must be the same as defined in frame/module_driver_constants.F */