From ed6934bc57ffec5a2d499de5a951231c3563ca35 Mon Sep 17 00:00:00 2001 From: Maff Glover Date: Mon, 18 Oct 2021 15:56:26 +0100 Subject: [PATCH 01/27] Feature 1926. Introduce OpenMP in GridStat. --- met/src/basic/vx_config/temp_file.cc | 4 +-- met/src/basic/vx_util/data_plane_util.cc | 33 +++++++++++++++-------- met/src/tools/core/grid_stat/grid_stat.cc | 11 ++++++++ 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/met/src/basic/vx_config/temp_file.cc b/met/src/basic/vx_config/temp_file.cc index 5746993957..463742d828 100644 --- a/met/src/basic/vx_config/temp_file.cc +++ b/met/src/basic/vx_config/temp_file.cc @@ -65,12 +65,10 @@ ConcatString make_temp_file_name(const char *prefix, const char *suffix) { //////////////////////////////////////////////////////////////////////// void remove_temp_file(const ConcatString file_name) { - int errno; - // // Attempt to remove the file and print out any error message // - if((errno = remove(file_name.c_str())) != 0) { + if(!remove(file_name.c_str())) { mlog << Error << "\nremove_temp_file() -> " << "can't delete temporary file: \"" << file_name << "\" (" diff --git a/met/src/basic/vx_util/data_plane_util.cc b/met/src/basic/vx_util/data_plane_util.cc index 3f5b5f7278..0637e2fb1a 100644 --- a/met/src/basic/vx_util/data_plane_util.cc +++ b/met/src/basic/vx_util/data_plane_util.cc @@ -16,6 +16,8 @@ using namespace std; #include #include +#include "omp.h" + #include "data_plane_util.h" #include "interp_util.h" #include "two_to_one.h" @@ -247,21 +249,28 @@ void fractional_coverage(const DataPlane &dp, DataPlane &frac_dp, } } - // Build the grid template - GridTemplateFactory gtf; - GridTemplate* gt = gtf.buildGT(shape, width, wrap_lon); +#pragma omp parallel default(shared) private(x,y,n_vld,n_thr,gp,v) + { - mlog << Debug(3) - << "Computing fractional coverage field using the " - << t.get_str() << " threshold and the " - << interpmthd_to_string(InterpMthd_Nbrhd) << "(" << gt->size() - << ") " << gt->getClassName() << " interpolation method.\n"; + // Build the grid template + GridTemplateFactory gtf; + GridTemplate* gt = gtf.buildGT(shape, width, wrap_lon); - // Initialize the fractional coverage field - frac_dp = dp; - frac_dp.set_constant(bad_data_double); +#pragma omp single + { + mlog << Debug(3) + << "Computing fractional coverage field using the " + << t.get_str() << " threshold and the " + << interpmthd_to_string(InterpMthd_Nbrhd) << "(" << gt->size() + << ") " << gt->getClassName() << " interpolation method.\n"; + + // Initialize the fractional coverage field + frac_dp = dp; + frac_dp.set_constant(bad_data_double); + } // Compute the fractional coverage meeting the threshold criteria +#pragma omp for schedule (static) for(x=0; x #include +#include "omp.h" + #include "grid_stat.h" #include "vx_statistics.h" @@ -185,6 +187,15 @@ static bool read_data_plane(VarInfo* info, DataPlane& dp, Met2dDataFile* mtddf, int main(int argc, char *argv[]) { +#ifdef _OPENMP +#pragma omp parallel +#pragma omp single + { + // Report number of threads if compiled with OpenMP + std::cout << "Running on " << omp_get_num_threads() << " threads.\n"; + } +#endif + // Set handler to be called for memory allocation error set_new_handler(oom); From 86c1a5054b170197336351443e9abc0412d261d8 Mon Sep 17 00:00:00 2001 From: Maff Glover Date: Thu, 4 Nov 2021 10:25:35 +0000 Subject: [PATCH 02/27] Feature 1926. Add MET_OPENMP flag to automake setup. --- met/configure.ac | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/met/configure.ac b/met/configure.ac index 2fc599e47a..c8748f5702 100644 --- a/met/configure.ac +++ b/met/configure.ac @@ -1053,6 +1053,7 @@ AC_SUBST([PYTHON_LIBS]) # Check for the MET_DEVELOPMENT environment variable AM_CONDITIONAL([ENABLE_DEVELOPMENT], [test -n "$MET_DEVELOPMENT"]) +AM_CONDITIONAL([ENABLE_OPENMP], [test -n "$MET_OPENMP"]) # Add MET_BASE to the CPPFLAGS @@ -1067,9 +1068,17 @@ AC_SUBST(FC_LIBS, [-lgfortran]) # For Intel compilers, "-g" slows down runtimes considerably (MET #1778). # For development, retain the "-g" option. Otherwise, discard it. -AM_COND_IF([ENABLE_DEVELOPMENT], [: ${CFLAGS="-g -O2"}], [: ${CFLAGS="-O2"}]) -AM_COND_IF([ENABLE_DEVELOPMENT], [: ${CXXFLAGS="-g"} ], [: ${CXXFLAGS=""}] ) -AM_COND_IF([ENABLE_DEVELOPMENT], [: ${FFLAGS="-g -O2"}], [: ${FFLAGS="-O2"}]) +AM_COND_IF([ENABLE_DEVELOPMENT], [: ${CFLAGS_OPT="-g -O2"}], [: ${CFLAGS_OPT="-O2"}]) +AM_COND_IF([ENABLE_DEVELOPMENT], [: ${CXXFLAGS_OPT="-g"} ], [: ${CXXFLAGS_OPT="-O3"}] ) +AM_COND_IF([ENABLE_DEVELOPMENT], [: ${FFLAGS_OPT="-g -O2"}], [: ${FFLAGS_OPT="-O2"}]) + +AM_COND_IF([ENABLE_OPENMP], [: ${CFLAGS_OMP="-fopenmp"}], [: ${CFLAGS_OMP=""}]) +AM_COND_IF([ENABLE_OPENMP], [: ${CXXFLAGS_OMP="-fopenmp"}], [: ${CXXFLAGS_OMP=""}]) +AM_COND_IF([ENABLE_OPENMP], [: ${FFLAGS_OMP="-fopenmp"}], [: ${FFLAGS_OMP=""}]) + +CFLAGS="$CFLAGS_OPT $CFLAGS_OMP" +CXXFLAGS="$CXXFLAGS_OPT $CXXFLAGS_OMP" +FFLAGS="$FFLAGS_OPT $FFLAGS_OMP" # Checks for programs. From b3a063aa2603775370d0a3f4afdd083458d9fce0 Mon Sep 17 00:00:00 2001 From: Maff Glover Date: Thu, 4 Nov 2021 10:40:43 +0000 Subject: [PATCH 03/27] Feature 1926. Indentation tweaks. --- met/src/basic/vx_util/data_plane_util.cc | 150 +++++++++++------------ 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/met/src/basic/vx_util/data_plane_util.cc b/met/src/basic/vx_util/data_plane_util.cc index 0637e2fb1a..0647219019 100644 --- a/met/src/basic/vx_util/data_plane_util.cc +++ b/met/src/basic/vx_util/data_plane_util.cc @@ -257,82 +257,82 @@ void fractional_coverage(const DataPlane &dp, DataPlane &frac_dp, GridTemplate* gt = gtf.buildGT(shape, width, wrap_lon); #pragma omp single - { - mlog << Debug(3) - << "Computing fractional coverage field using the " - << t.get_str() << " threshold and the " - << interpmthd_to_string(InterpMthd_Nbrhd) << "(" << gt->size() - << ") " << gt->getClassName() << " interpolation method.\n"; - - // Initialize the fractional coverage field - frac_dp = dp; - frac_dp.set_constant(bad_data_double); - } - - // Compute the fractional coverage meeting the threshold criteria + { + mlog << Debug(3) + << "Computing fractional coverage field using the " + << t.get_str() << " threshold and the " + << interpmthd_to_string(InterpMthd_Nbrhd) << "(" << gt->size() + << ") " << gt->getClassName() << " interpolation method.\n"; + + // Initialize the fractional coverage field + frac_dp = dp; + frac_dp.set_constant(bad_data_double); + } + + // Compute the fractional coverage meeting the threshold criteria #pragma omp for schedule (static) - for(x=0; xgetFirstInGrid(x, y, dp.nx(), dp.ny()); - gp != NULL; - gp = gt->getNextInGrid()) { - if(is_bad_data(v = dp.get(gp->x, gp->y))) continue; - n_vld++; - if(t.check(v, - (use_climo ? cmn->get(gp->x, gp->y) : bad), - (use_climo ? csd->get(gp->x, gp->y) : bad))) n_thr++; - } - } - // Subtract off the bottom edge, shift up, and add the top. - else { - - // Subtract points from the the bottom edge - for(gp = gt->getFirstInBotEdge(); - gp != NULL; - gp = gt->getNextInBotEdge()) { - if(is_bad_data(v = dp.get(gp->x, gp->y))) continue; - n_vld--; - if(t.check(v, - (use_climo ? cmn->get(gp->x, gp->y) : bad), - (use_climo ? csd->get(gp->x, gp->y) : bad))) n_thr--; - } - - // Increment Y - gt->incBaseY(1); - - // Add points from the the top edge - for(gp = gt->getFirstInTopEdge(); - gp != NULL; - gp = gt->getNextInTopEdge()) { - if(is_bad_data(v = dp.get(gp->x, gp->y))) continue; - n_vld++; - if(t.check(v, - (use_climo ? cmn->get(gp->x, gp->y) : bad), - (use_climo ? csd->get(gp->x, gp->y) : bad))) n_thr++; - } - } - - // Check for enough valid data and compute fractional coverage - if((double)(n_vld)/gt->size() >= vld_t && n_vld != 0) { - frac_dp.set((double) n_thr/n_vld, x, y); - } - - } // end for y - - // Increment X - if(x < (dp.nx() - 1)) gt->incBaseX(1); - - } // end for x - - delete gt; + for(x=0; xgetFirstInGrid(x, y, dp.nx(), dp.ny()); + gp != NULL; + gp = gt->getNextInGrid()) { + if(is_bad_data(v = dp.get(gp->x, gp->y))) continue; + n_vld++; + if(t.check(v, + (use_climo ? cmn->get(gp->x, gp->y) : bad), + (use_climo ? csd->get(gp->x, gp->y) : bad))) n_thr++; + } + } + // Subtract off the bottom edge, shift up, and add the top. + else { + + // Subtract points from the the bottom edge + for(gp = gt->getFirstInBotEdge(); + gp != NULL; + gp = gt->getNextInBotEdge()) { + if(is_bad_data(v = dp.get(gp->x, gp->y))) continue; + n_vld--; + if(t.check(v, + (use_climo ? cmn->get(gp->x, gp->y) : bad), + (use_climo ? csd->get(gp->x, gp->y) : bad))) n_thr--; + } + + // Increment Y + gt->incBaseY(1); + + // Add points from the the top edge + for(gp = gt->getFirstInTopEdge(); + gp != NULL; + gp = gt->getNextInTopEdge()) { + if(is_bad_data(v = dp.get(gp->x, gp->y))) continue; + n_vld++; + if(t.check(v, + (use_climo ? cmn->get(gp->x, gp->y) : bad), + (use_climo ? csd->get(gp->x, gp->y) : bad))) n_thr++; + } + } + + // Check for enough valid data and compute fractional coverage + if((double)(n_vld)/gt->size() >= vld_t && n_vld != 0) { + frac_dp.set((double) n_thr/n_vld, x, y); + } + + } // end for y + + // Increment X + if(x < (dp.nx() - 1)) gt->incBaseX(1); + + } // end for x + + delete gt; } // End of omp parallel From e0257c6277bb28a924a2b235c1d20fb6088c388c Mon Sep 17 00:00:00 2001 From: Maff Glover Date: Fri, 5 Nov 2021 14:22:43 +0000 Subject: [PATCH 04/27] Feature 1926. Revert temporary C++17 temporary file code tweak. --- met/src/basic/vx_config/temp_file.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/met/src/basic/vx_config/temp_file.cc b/met/src/basic/vx_config/temp_file.cc index 463742d828..5746993957 100644 --- a/met/src/basic/vx_config/temp_file.cc +++ b/met/src/basic/vx_config/temp_file.cc @@ -65,10 +65,12 @@ ConcatString make_temp_file_name(const char *prefix, const char *suffix) { //////////////////////////////////////////////////////////////////////// void remove_temp_file(const ConcatString file_name) { + int errno; + // // Attempt to remove the file and print out any error message // - if(!remove(file_name.c_str())) { + if((errno = remove(file_name.c_str())) != 0) { mlog << Error << "\nremove_temp_file() -> " << "can't delete temporary file: \"" << file_name << "\" (" From 7d81d417f7e8358c979b67c0db3f07c8155de89d Mon Sep 17 00:00:00 2001 From: Maff Glover Date: Thu, 18 Nov 2021 16:37:32 +0000 Subject: [PATCH 05/27] Feature 1926. Add _OPENMP ifdef for omp.h. --- met/src/basic/vx_util/data_plane_util.cc | 4 +++- met/src/tools/core/grid_stat/grid_stat.cc | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/met/src/basic/vx_util/data_plane_util.cc b/met/src/basic/vx_util/data_plane_util.cc index 0647219019..9dca30427c 100644 --- a/met/src/basic/vx_util/data_plane_util.cc +++ b/met/src/basic/vx_util/data_plane_util.cc @@ -16,7 +16,9 @@ using namespace std; #include #include -#include "omp.h" +#ifdef _OPENMP + #include "omp.h" +#endif #include "data_plane_util.h" #include "interp_util.h" diff --git a/met/src/tools/core/grid_stat/grid_stat.cc b/met/src/tools/core/grid_stat/grid_stat.cc index d663e696f0..398a74759e 100644 --- a/met/src/tools/core/grid_stat/grid_stat.cc +++ b/met/src/tools/core/grid_stat/grid_stat.cc @@ -127,7 +127,9 @@ using namespace std; #include #include -#include "omp.h" +#ifdef _OPENMP + #include "omp.h" +#endif #include "grid_stat.h" From 3ff12fe976a738c9f96f20ddc424a87eaf3e3f0b Mon Sep 17 00:00:00 2001 From: Maff Glover Date: Mon, 22 Nov 2021 16:02:58 +0000 Subject: [PATCH 06/27] Feature 1926. Revert tweak to CXX optimisation level with MET_DEVELOPMENT. --- met/configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/met/configure.ac b/met/configure.ac index c8748f5702..f343adde2f 100644 --- a/met/configure.ac +++ b/met/configure.ac @@ -1069,7 +1069,7 @@ AC_SUBST(FC_LIBS, [-lgfortran]) # For development, retain the "-g" option. Otherwise, discard it. AM_COND_IF([ENABLE_DEVELOPMENT], [: ${CFLAGS_OPT="-g -O2"}], [: ${CFLAGS_OPT="-O2"}]) -AM_COND_IF([ENABLE_DEVELOPMENT], [: ${CXXFLAGS_OPT="-g"} ], [: ${CXXFLAGS_OPT="-O3"}] ) +AM_COND_IF([ENABLE_DEVELOPMENT], [: ${CXXFLAGS_OPT="-g"} ], [: ${CXXFLAGS_OPT=""}] ) AM_COND_IF([ENABLE_DEVELOPMENT], [: ${FFLAGS_OPT="-g -O2"}], [: ${FFLAGS_OPT="-O2"}]) AM_COND_IF([ENABLE_OPENMP], [: ${CFLAGS_OMP="-fopenmp"}], [: ${CFLAGS_OMP=""}]) From 9a045e0b264c67ecb72bc7f072b25c5a36fe050f Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 22 Nov 2021 15:27:33 -0700 Subject: [PATCH 07/27] Per #1926, switch the OpenMP configuration option from being like MET_DEVELOPMENT to being like GRIB2, with --enable-grib2." --- met/configure.ac | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/met/configure.ac b/met/configure.ac index f343adde2f..8b9a0e7a52 100644 --- a/met/configure.ac +++ b/met/configure.ac @@ -988,6 +988,30 @@ else AC_MSG_NOTICE([gsi_tools will not be compiled]) fi +# OpenMP + +AC_ARG_ENABLE(openmp, + [AS_HELP_STRING([--enable-openmp], [Enable compilation of OpenMP support])], + [case "${enableval}" in + yes | no ) ENABLE_OPENMP="${enableval}" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-openmp) ;; + esac], + [ENABLE_OPENMP="no"] +) + +AM_CONDITIONAL([ENABLE_OPENMP], [test "x$ENABLE_OPENMP" = "xyes"]) + +if test "x$ENABLE_OPENMP" = "xyes"; then + AC_DEFINE([ENABLE_OPENMP], [], ["build OpenMP support"]) + AC_MSG_NOTICE([OpenMP support will be compiled]) + CPPFLAGS="${CPPFLAGS} -DWITH_OPENMP -fopenmp" + OPENMP_LIBS="-lgomp" +else + AC_MSG_NOTICE([OpenMP support will not be compiled]) + OPENMP_LIBS= +fi +AC_SUBST([OPENMP_LIBS]) + # GRIB2 AC_ARG_ENABLE(grib2, @@ -1053,7 +1077,6 @@ AC_SUBST([PYTHON_LIBS]) # Check for the MET_DEVELOPMENT environment variable AM_CONDITIONAL([ENABLE_DEVELOPMENT], [test -n "$MET_DEVELOPMENT"]) -AM_CONDITIONAL([ENABLE_OPENMP], [test -n "$MET_OPENMP"]) # Add MET_BASE to the CPPFLAGS @@ -1068,17 +1091,9 @@ AC_SUBST(FC_LIBS, [-lgfortran]) # For Intel compilers, "-g" slows down runtimes considerably (MET #1778). # For development, retain the "-g" option. Otherwise, discard it. -AM_COND_IF([ENABLE_DEVELOPMENT], [: ${CFLAGS_OPT="-g -O2"}], [: ${CFLAGS_OPT="-O2"}]) -AM_COND_IF([ENABLE_DEVELOPMENT], [: ${CXXFLAGS_OPT="-g"} ], [: ${CXXFLAGS_OPT=""}] ) -AM_COND_IF([ENABLE_DEVELOPMENT], [: ${FFLAGS_OPT="-g -O2"}], [: ${FFLAGS_OPT="-O2"}]) - -AM_COND_IF([ENABLE_OPENMP], [: ${CFLAGS_OMP="-fopenmp"}], [: ${CFLAGS_OMP=""}]) -AM_COND_IF([ENABLE_OPENMP], [: ${CXXFLAGS_OMP="-fopenmp"}], [: ${CXXFLAGS_OMP=""}]) -AM_COND_IF([ENABLE_OPENMP], [: ${FFLAGS_OMP="-fopenmp"}], [: ${FFLAGS_OMP=""}]) - -CFLAGS="$CFLAGS_OPT $CFLAGS_OMP" -CXXFLAGS="$CXXFLAGS_OPT $CXXFLAGS_OMP" -FFLAGS="$FFLAGS_OPT $FFLAGS_OMP" +AM_COND_IF([ENABLE_DEVELOPMENT], [: ${CFLAGS="-g -O2"}], [: ${CFLAGS="-O2"}]) +AM_COND_IF([ENABLE_DEVELOPMENT], [: ${CXXFLAGS="-g"} ], [: ${CXXFLAGS=""}] ) +AM_COND_IF([ENABLE_DEVELOPMENT], [: ${FFLAGS="-g -O2"}], [: ${FFLAGS="-O2"}]) # Checks for programs. From 4c176c46490d14d25b7160f18569ac4a924d6144 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 22 Nov 2021 15:29:17 -0700 Subject: [PATCH 08/27] Per #1926, just like --enable-grib2 and --enable-python, add a test placeholder for --enable-openmp so that running make test works. --- met/scripts/.gitignore | 1 + met/scripts/Makefile | 1 + met/scripts/mk/openmp.mk | 20 ++++++++++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 met/scripts/mk/openmp.mk diff --git a/met/scripts/.gitignore b/met/scripts/.gitignore index 1263b3675e..1eafa5e2c2 100644 --- a/met/scripts/.gitignore +++ b/met/scripts/.gitignore @@ -14,6 +14,7 @@ mode_graphics mode mode_time_domain modis +openmp pb2nc pcp_combine plot_data_plane diff --git a/met/scripts/Makefile b/met/scripts/Makefile index 2081261e28..857b647489 100755 --- a/met/scripts/Makefile +++ b/met/scripts/Makefile @@ -65,6 +65,7 @@ include $(MK_DIR)/mode_analysis.mk include $(MK_DIR)/mode_graphics.mk include $(MK_DIR)/mode_time_domain.mk include $(MK_DIR)/modis.mk +include $(MK_DIR)/openmp.mk include $(MK_DIR)/pb2nc.mk include $(MK_DIR)/pcp_combine.mk include $(MK_DIR)/plot_data_plane.mk diff --git a/met/scripts/mk/openmp.mk b/met/scripts/mk/openmp.mk new file mode 100644 index 0000000000..03fe639091 --- /dev/null +++ b/met/scripts/mk/openmp.mk @@ -0,0 +1,20 @@ + + + +######################################################################## + + + ## + ## openmp + ## + ## prerequisites: + ## + + +openmp: + @ touch openmp + + +######################################################################## + + From 7043995c936edd86458a7c0b3aa9567fb554cb7e Mon Sep 17 00:00:00 2001 From: Maff Glover Date: Fri, 26 Nov 2021 16:57:42 +0000 Subject: [PATCH 09/27] Feature #1926: Ensure OpenMP flag is passed to the linker. Revert to _OPENMP for the moment. --- met/configure.ac | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/met/configure.ac b/met/configure.ac index 8b9a0e7a52..d4bd50d234 100644 --- a/met/configure.ac +++ b/met/configure.ac @@ -1004,13 +1004,11 @@ AM_CONDITIONAL([ENABLE_OPENMP], [test "x$ENABLE_OPENMP" = "xyes"]) if test "x$ENABLE_OPENMP" = "xyes"; then AC_DEFINE([ENABLE_OPENMP], [], ["build OpenMP support"]) AC_MSG_NOTICE([OpenMP support will be compiled]) - CPPFLAGS="${CPPFLAGS} -DWITH_OPENMP -fopenmp" - OPENMP_LIBS="-lgomp" + CPPFLAGS="${CPPFLAGS} -fopenmp" + LDFLAGS="${LDFLAGS} -fopenmp" else AC_MSG_NOTICE([OpenMP support will not be compiled]) - OPENMP_LIBS= fi -AC_SUBST([OPENMP_LIBS]) # GRIB2 From 81c98b9e2cf968fdda64ad25630a08b7ef185813 Mon Sep 17 00:00:00 2001 From: mo-mglover <78152252+mo-mglover@users.noreply.github.com> Date: Tue, 30 Nov 2021 09:17:30 +0000 Subject: [PATCH 10/27] Feature #1926. Write thread count via logging, not cout. Co-authored-by: johnhg --- met/src/tools/core/grid_stat/grid_stat.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/met/src/tools/core/grid_stat/grid_stat.cc b/met/src/tools/core/grid_stat/grid_stat.cc index 398a74759e..eea4cba2f1 100644 --- a/met/src/tools/core/grid_stat/grid_stat.cc +++ b/met/src/tools/core/grid_stat/grid_stat.cc @@ -194,7 +194,7 @@ int main(int argc, char *argv[]) { #pragma omp single { // Report number of threads if compiled with OpenMP - std::cout << "Running on " << omp_get_num_threads() << " threads.\n"; + mlog << Debug(2) << "Running on " << omp_get_num_threads() << " threads.\n"; } #endif From 1abb2be421d9a7757aea59f5d84766a4386612f2 Mon Sep 17 00:00:00 2001 From: Maff Glover Date: Tue, 30 Nov 2021 20:52:46 +0000 Subject: [PATCH 11/27] Feature #1926: Add note about --enable-openmp in installation.rst --- met/docs/Users_Guide/installation.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/met/docs/Users_Guide/installation.rst b/met/docs/Users_Guide/installation.rst index 963545ac2f..a2de8328ad 100644 --- a/met/docs/Users_Guide/installation.rst +++ b/met/docs/Users_Guide/installation.rst @@ -196,7 +196,8 @@ Example: To configure MET to install all of the available tools in the "bin" sub .. code-block:: none 1. ./configure --prefix=`pwd` --enable-grib2 --enable-python \ - --enable-modis --enable-mode_graphics --enable-lidar2nc + --enable-modis --enable-mode_graphics --enable-lidar2nc \ + --enable-openmp 2. Type 'make install >& make_install.log &' 3. Type 'tail -f make_install.log' to view the execution of the make. 4. When make is finished, type 'CTRL-C' to quit the tail. @@ -272,6 +273,14 @@ Enable compilation of python interface. Requires $MET_PYTHON_CC and $MET_PYTHON_ Enable compilation of utilities using the LIDAR2NC tool. +.. code-block:: none + + --enable-openmp + +Enable compilation of OpenMP directives within the code. This allows some code +regions to benefit from thread-parallel execution. Runtime environment variable +:code:`OMP_NUM_THREADS` controls the number of threads. + .. code-block:: none --enable-modis From d6969c60dac5094ad1ffe045af08c3dfe4e9fa65 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 2 Dec 2021 14:38:57 -0700 Subject: [PATCH 12/27] Per #1926, adding a call to Grid-Stat to run on 2 threads. This should produce the same result as the previous test named grid_stat_GRIB2_NAM_RTMA. --- test/xml/unit_grid_stat.xml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/xml/unit_grid_stat.xml b/test/xml/unit_grid_stat.xml index 1d562778f7..c31455c0ce 100644 --- a/test/xml/unit_grid_stat.xml +++ b/test/xml/unit_grid_stat.xml @@ -75,6 +75,32 @@ + + &MET_BIN;/grid_stat + + OMP_NUM_THREADS 2 + OUTPUT_PREFIX GRIB2_NAM_RTMA_NP2 + + \ + &DATA_DIR_MODEL;/grib2/nam/nam_2012040900_F012_gRtma.grib2 \ + &DATA_DIR_OBS;/rtma/rtma_2012051712_F000.grib2 \ + &CONFIG_DIR;/GridStatConfig_rtma \ + -outdir &OUTPUT_DIR;/grid_stat -v 1 + + + &OUTPUT_DIR;/grid_stat/grid_stat_GRIB2_NAM_RTMA_NP2_120000L_20120409_120000V.stat + &OUTPUT_DIR;/grid_stat/grid_stat_GRIB2_NAM_RTMA_NP2_120000L_20120409_120000V_fho.txt + &OUTPUT_DIR;/grid_stat/grid_stat_GRIB2_NAM_RTMA_NP2_120000L_20120409_120000V_ctc.txt + &OUTPUT_DIR;/grid_stat/grid_stat_GRIB2_NAM_RTMA_NP2_120000L_20120409_120000V_cts.txt + &OUTPUT_DIR;/grid_stat/grid_stat_GRIB2_NAM_RTMA_NP2_120000L_20120409_120000V_cnt.txt + &OUTPUT_DIR;/grid_stat/grid_stat_GRIB2_NAM_RTMA_NP2_120000L_20120409_120000V_sl1l2.txt + &OUTPUT_DIR;/grid_stat/grid_stat_GRIB2_NAM_RTMA_NP2_120000L_20120409_120000V_nbrctc.txt + &OUTPUT_DIR;/grid_stat/grid_stat_GRIB2_NAM_RTMA_NP2_120000L_20120409_120000V_nbrcts.txt + &OUTPUT_DIR;/grid_stat/grid_stat_GRIB2_NAM_RTMA_NP2_120000L_20120409_120000V_nbrcnt.txt + &OUTPUT_DIR;/grid_stat/grid_stat_GRIB2_NAM_RTMA_NP2_120000L_20120409_120000V_pairs.nc + + + &MET_BIN;/grid_stat From 2b37a69415ffb64c92226a861191e4d3e4a4eb2a Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 2 Dec 2021 14:41:19 -0700 Subject: [PATCH 13/27] Per #1926, updating GridStatConfig_rtma to switch width = 1 to 3. This seems like a pretty obvious typo that was causing a warning message to be written to the log file. While this will change the unit test results, I'm confident it was the original intention of this Grid-Stat configuration. --- test/config/GridStatConfig_rtma | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/config/GridStatConfig_rtma b/test/config/GridStatConfig_rtma index 178a1269b1..2c3d3ba058 100644 --- a/test/config/GridStatConfig_rtma +++ b/test/config/GridStatConfig_rtma @@ -111,7 +111,7 @@ interp = { type = [ { method = NEAREST; width = 1; }, { method = UW_MEAN; width = 3; }, - { method = MIN; width = 1; }, + { method = MIN; width = 3; }, { method = MAX; width = 3; } ]; } From 4b2982f1d330699f31faa76d82d9407c8086941b Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 3 Dec 2021 10:13:27 -0700 Subject: [PATCH 14/27] Per #1926, tweak the openmp log message slightly. --- met/src/tools/core/grid_stat/grid_stat.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/met/src/tools/core/grid_stat/grid_stat.cc b/met/src/tools/core/grid_stat/grid_stat.cc index eea4cba2f1..c0b85f9956 100644 --- a/met/src/tools/core/grid_stat/grid_stat.cc +++ b/met/src/tools/core/grid_stat/grid_stat.cc @@ -194,7 +194,7 @@ int main(int argc, char *argv[]) { #pragma omp single { // Report number of threads if compiled with OpenMP - mlog << Debug(2) << "Running on " << omp_get_num_threads() << " threads.\n"; + mlog << Debug(2) << "OpenMP running on " << omp_get_num_threads() << " thread(s).\n"; } #endif From 3c1a5dbc3ce9462bddf7fcd94e8ff652d4b0b053 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 3 Dec 2021 12:39:58 -0700 Subject: [PATCH 15/27] Per #1926, switch from manual --enable-openmp support in autoconf to calling the AC_OPENMP() macro. And that macro compile WITH openmp support by default. It can be disabled using --disable-openmp. --- met/configure.ac | 29 +++++++---------------------- met/scripts/mk/openmp.mk | 20 -------------------- 2 files changed, 7 insertions(+), 42 deletions(-) delete mode 100644 met/scripts/mk/openmp.mk diff --git a/met/configure.ac b/met/configure.ac index d4bd50d234..7008748079 100644 --- a/met/configure.ac +++ b/met/configure.ac @@ -8,6 +8,13 @@ AC_CONFIG_HEADERS([config.h]) AM_INIT_AUTOMAKE([1.9 foreign]) +# OpenMP + +AC_OPENMP() + +CPPFLAGS="${CPPFLAGS} ${OPENMP_CFLAGS}" +LDFLAGS="${LDFLAGS} ${OPENMP_CFLAGS}" + # # Look for the NetCDF library # @@ -988,28 +995,6 @@ else AC_MSG_NOTICE([gsi_tools will not be compiled]) fi -# OpenMP - -AC_ARG_ENABLE(openmp, - [AS_HELP_STRING([--enable-openmp], [Enable compilation of OpenMP support])], - [case "${enableval}" in - yes | no ) ENABLE_OPENMP="${enableval}" ;; - *) AC_MSG_ERROR(bad value ${enableval} for --enable-openmp) ;; - esac], - [ENABLE_OPENMP="no"] -) - -AM_CONDITIONAL([ENABLE_OPENMP], [test "x$ENABLE_OPENMP" = "xyes"]) - -if test "x$ENABLE_OPENMP" = "xyes"; then - AC_DEFINE([ENABLE_OPENMP], [], ["build OpenMP support"]) - AC_MSG_NOTICE([OpenMP support will be compiled]) - CPPFLAGS="${CPPFLAGS} -fopenmp" - LDFLAGS="${LDFLAGS} -fopenmp" -else - AC_MSG_NOTICE([OpenMP support will not be compiled]) -fi - # GRIB2 AC_ARG_ENABLE(grib2, diff --git a/met/scripts/mk/openmp.mk b/met/scripts/mk/openmp.mk deleted file mode 100644 index 03fe639091..0000000000 --- a/met/scripts/mk/openmp.mk +++ /dev/null @@ -1,20 +0,0 @@ - - - -######################################################################## - - - ## - ## openmp - ## - ## prerequisites: - ## - - -openmp: - @ touch openmp - - -######################################################################## - - From b595de47be09f7769dcdc132cd4f84fc0e4eebfc Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 3 Dec 2021 12:42:02 -0700 Subject: [PATCH 16/27] Per #1926, after switching to the AC_OPENMP() macro, these changes to scripts are no longer needed to enable make test to run. --- met/scripts/.gitignore | 1 - met/scripts/Makefile | 1 - 2 files changed, 2 deletions(-) diff --git a/met/scripts/.gitignore b/met/scripts/.gitignore index 1eafa5e2c2..1263b3675e 100644 --- a/met/scripts/.gitignore +++ b/met/scripts/.gitignore @@ -14,7 +14,6 @@ mode_graphics mode mode_time_domain modis -openmp pb2nc pcp_combine plot_data_plane diff --git a/met/scripts/Makefile b/met/scripts/Makefile index 857b647489..2081261e28 100755 --- a/met/scripts/Makefile +++ b/met/scripts/Makefile @@ -65,7 +65,6 @@ include $(MK_DIR)/mode_analysis.mk include $(MK_DIR)/mode_graphics.mk include $(MK_DIR)/mode_time_domain.mk include $(MK_DIR)/modis.mk -include $(MK_DIR)/openmp.mk include $(MK_DIR)/pb2nc.mk include $(MK_DIR)/pcp_combine.mk include $(MK_DIR)/plot_data_plane.mk From ecaa653cbfa0c93006ef51f87b2d39a804936f56 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 3 Dec 2021 12:48:46 -0700 Subject: [PATCH 17/27] Per #1926, since openmp support will now be compiled by default, switch the docs from explaining --enable-openmp to --disable-openmp. --- met/docs/Users_Guide/installation.rst | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/met/docs/Users_Guide/installation.rst b/met/docs/Users_Guide/installation.rst index a2de8328ad..b4459fed0d 100644 --- a/met/docs/Users_Guide/installation.rst +++ b/met/docs/Users_Guide/installation.rst @@ -196,8 +196,7 @@ Example: To configure MET to install all of the available tools in the "bin" sub .. code-block:: none 1. ./configure --prefix=`pwd` --enable-grib2 --enable-python \ - --enable-modis --enable-mode_graphics --enable-lidar2nc \ - --enable-openmp + --enable-modis --enable-mode_graphics --enable-lidar2nc 2. Type 'make install >& make_install.log &' 3. Type 'tail -f make_install.log' to view the execution of the make. 4. When make is finished, type 'CTRL-C' to quit the tail. @@ -273,14 +272,6 @@ Enable compilation of python interface. Requires $MET_PYTHON_CC and $MET_PYTHON_ Enable compilation of utilities using the LIDAR2NC tool. -.. code-block:: none - - --enable-openmp - -Enable compilation of OpenMP directives within the code. This allows some code -regions to benefit from thread-parallel execution. Runtime environment variable -:code:`OMP_NUM_THREADS` controls the number of threads. - .. code-block:: none --enable-modis @@ -299,6 +290,14 @@ Enable compilation of the MODE-Graphics tool. Requires $MET_CAIRO and $MET_FREET Disable use of BLOCK4 in the compilation. Use this if you have trouble using PrepBUFR files. +.. code-block:: none + + --disable-openmp + +Disable compilation of OpenMP directives within the code which allows some code +regions to benefit from thread-parallel execution. Runtime environment variable +:code:`OMP_NUM_THREADS` controls the number of threads. + Run the configure script with the **-help** argument to see the full list of configuration options. Make Targets From 95945056474a32955cdab6789e233e2a6a1db25e Mon Sep 17 00:00:00 2001 From: Maff Glover Date: Mon, 6 Dec 2021 10:13:35 +0000 Subject: [PATCH 18/27] Feature #1926. Use `default(none)` to force explicit `private` or `shared`. --- met/src/basic/vx_util/data_plane_util.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/met/src/basic/vx_util/data_plane_util.cc b/met/src/basic/vx_util/data_plane_util.cc index 9dca30427c..39fbb67679 100644 --- a/met/src/basic/vx_util/data_plane_util.cc +++ b/met/src/basic/vx_util/data_plane_util.cc @@ -251,7 +251,10 @@ void fractional_coverage(const DataPlane &dp, DataPlane &frac_dp, } } -#pragma omp parallel default(shared) private(x,y,n_vld,n_thr,gp,v) +#pragma omp parallel default(none) \ + shared(mlog, dp, frac_dp, width, wrap_lon, t) \ + shared(use_climo, cmn, csd, vld_t, bad) \ + private(x, y, n_vld, n_thr, gp, v) { // Build the grid template From e8d928bc5b7414de7f7ce33817d6ea9b9c3dceac Mon Sep 17 00:00:00 2001 From: Maff Glover Date: Mon, 6 Dec 2021 15:57:43 +0000 Subject: [PATCH 19/27] Feature #1926: Default to single thread if OMP_NUM_THREADS is left unset. --- met/src/basic/vx_util/Makefile.am | 1 + met/src/basic/vx_util/handle_openmp.cc | 51 +++++++++++++++++++++++ met/src/basic/vx_util/handle_openmp.h | 24 +++++++++++ met/src/tools/core/grid_stat/grid_stat.cc | 14 ++----- 4 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 met/src/basic/vx_util/handle_openmp.cc create mode 100644 met/src/basic/vx_util/handle_openmp.h diff --git a/met/src/basic/vx_util/Makefile.am b/met/src/basic/vx_util/Makefile.am index 57b7a100db..de788d7202 100644 --- a/met/src/basic/vx_util/Makefile.am +++ b/met/src/basic/vx_util/Makefile.am @@ -69,6 +69,7 @@ libvx_util_a_SOURCES = ascii_table.cc ascii_table.h \ GridOffset.h GridOffset.cc \ observation.h observation.cc \ stat_column_defs.h \ + handle_openmp.h handle_openmp.cc \ RectangularTemplate.h RectangularTemplate.cc $(OPT_PYTHON_SOURCES) libvx_util_a_CPPFLAGS = ${MET_CPPFLAGS} diff --git a/met/src/basic/vx_util/handle_openmp.cc b/met/src/basic/vx_util/handle_openmp.cc new file mode 100644 index 0000000000..a0d200d269 --- /dev/null +++ b/met/src/basic/vx_util/handle_openmp.cc @@ -0,0 +1,51 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +/////////////////////////////////////////////////////////////////////////////// + +#include + +#ifdef _OPENMP + #include "omp.h" +#endif + +#include "vx_log.h" +#include "handle_openmp.h" + +/////////////////////////////////////////////////////////////////////////////// + +void init_openmp() { + +#ifdef _OPENMP + + // If OMP_NUM_THREADS was not set, use the OpenMP API to set the thread count + // to 1 thread only. + const char* env_omp_num_threads = std::getenv("OMP_NUM_THREADS"); + if (!env_omp_num_threads) { + mlog << Debug(2) << "OMP_NUM_THREADS is not set." + << " Defaulting to 1 thread. \n"; + omp_set_num_threads(1); + } + +#pragma omp parallel +#pragma omp single + { + mlog << Debug(2) << "OpenMP running on " + << omp_get_num_threads() << " thread(s).\n"; + } + +#else /* _OPENMP */ + + mlog << Debug(2) << "OpenMP disabled.\n"; + +#endif /* _OPENMP */ + +} + +/////////////////////////////////////////////////////////////////////////////// + diff --git a/met/src/basic/vx_util/handle_openmp.h b/met/src/basic/vx_util/handle_openmp.h new file mode 100644 index 0000000000..e2ea2e6a38 --- /dev/null +++ b/met/src/basic/vx_util/handle_openmp.h @@ -0,0 +1,24 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __HANDLE_OPENMP_H___ +#define __HANDLE_OPENMP_H___ + +/////////////////////////////////////////////////////////////////////////////// + +void init_openmp(); + +/////////////////////////////////////////////////////////////////////////////// + +#endif /* __HANDLE_OPENMP_H__ */ + +/////////////////////////////////////////////////////////////////////////////// + + diff --git a/met/src/tools/core/grid_stat/grid_stat.cc b/met/src/tools/core/grid_stat/grid_stat.cc index c0b85f9956..1ac4605fae 100644 --- a/met/src/tools/core/grid_stat/grid_stat.cc +++ b/met/src/tools/core/grid_stat/grid_stat.cc @@ -127,9 +127,7 @@ using namespace std; #include #include -#ifdef _OPENMP - #include "omp.h" -#endif +#include "handle_openmp.h" #include "grid_stat.h" @@ -189,14 +187,8 @@ static bool read_data_plane(VarInfo* info, DataPlane& dp, Met2dDataFile* mtddf, int main(int argc, char *argv[]) { -#ifdef _OPENMP -#pragma omp parallel -#pragma omp single - { - // Report number of threads if compiled with OpenMP - mlog << Debug(2) << "OpenMP running on " << omp_get_num_threads() << " thread(s).\n"; - } -#endif + // Setup OpenMP (if compiled-in) + init_openmp(); // Set handler to be called for memory allocation error set_new_handler(oom); From dd35aa43e97370fb560040e7c94943ff324897f7 Mon Sep 17 00:00:00 2001 From: Maff Glover Date: Mon, 6 Dec 2021 16:15:58 +0000 Subject: [PATCH 20/27] Feature #1926: Add OpenMP initialisation to other tools calling fractional_coverage. --- met/src/tools/core/ensemble_stat/ensemble_stat.cc | 5 +++++ met/src/tools/core/grid_stat/grid_stat.cc | 2 +- met/src/tools/other/gen_ens_prod/gen_ens_prod.cc | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat.cc b/met/src/tools/core/ensemble_stat/ensemble_stat.cc index fe5d160157..ab296a2528 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/met/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -92,6 +92,8 @@ using namespace std; #include "nc_obs_util.h" #include "nc_point_obs_in.h" +#include "handle_openmp.h" + //////////////////////////////////////////////////////////////////////// static void process_command_line (int, char **); @@ -171,6 +173,9 @@ static void set_compress(const StringArray &); int main(int argc, char *argv[]) { + // Set up OpenMP (if enabled) + init_openmp(); + // Set handler to be called for memory allocation error set_new_handler(oom); diff --git a/met/src/tools/core/grid_stat/grid_stat.cc b/met/src/tools/core/grid_stat/grid_stat.cc index 1ac4605fae..10c1916b3e 100644 --- a/met/src/tools/core/grid_stat/grid_stat.cc +++ b/met/src/tools/core/grid_stat/grid_stat.cc @@ -187,7 +187,7 @@ static bool read_data_plane(VarInfo* info, DataPlane& dp, Met2dDataFile* mtddf, int main(int argc, char *argv[]) { - // Setup OpenMP (if compiled-in) + // Set up OpenMP (if enabled) init_openmp(); // Set handler to be called for memory allocation error diff --git a/met/src/tools/other/gen_ens_prod/gen_ens_prod.cc b/met/src/tools/other/gen_ens_prod/gen_ens_prod.cc index c7b48a7331..4c2a13976e 100644 --- a/met/src/tools/other/gen_ens_prod/gen_ens_prod.cc +++ b/met/src/tools/other/gen_ens_prod/gen_ens_prod.cc @@ -45,6 +45,8 @@ using namespace std; #include "nc_obs_util.h" #include "nc_point_obs_in.h" +#include "handle_openmp.h" + //////////////////////////////////////////////////////////////////////// static void process_command_line(int, char **); @@ -81,6 +83,9 @@ static void set_ctrl_file (const StringArray &); int main(int argc, char *argv[]) { + // Set up OpenMP (if enabled) + init_openmp(); + // Set handler to be called for memory allocation error set_new_handler(oom); From e0d7840937de6963f0481cec1a6a249e8dfe6cf7 Mon Sep 17 00:00:00 2001 From: mo-mglover <78152252+mo-mglover@users.noreply.github.com> Date: Thu, 16 Dec 2021 13:52:35 +0000 Subject: [PATCH 21/27] Update met/src/basic/vx_util/handle_openmp.cc Co-authored-by: johnhg --- met/src/basic/vx_util/handle_openmp.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/met/src/basic/vx_util/handle_openmp.cc b/met/src/basic/vx_util/handle_openmp.cc index a0d200d269..56ee65abca 100644 --- a/met/src/basic/vx_util/handle_openmp.cc +++ b/met/src/basic/vx_util/handle_openmp.cc @@ -28,7 +28,7 @@ void init_openmp() { const char* env_omp_num_threads = std::getenv("OMP_NUM_THREADS"); if (!env_omp_num_threads) { mlog << Debug(2) << "OMP_NUM_THREADS is not set." - << " Defaulting to 1 thread. \n"; + << " Defaulting to 1 thread. \n"; omp_set_num_threads(1); } From 78e435427e203b464c335a5d5ad739d630d56b8b Mon Sep 17 00:00:00 2001 From: mo-mglover <78152252+mo-mglover@users.noreply.github.com> Date: Thu, 16 Dec 2021 13:54:00 +0000 Subject: [PATCH 22/27] Update met/src/basic/vx_util/Makefile.am Co-authored-by: johnhg --- met/src/basic/vx_util/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/met/src/basic/vx_util/Makefile.am b/met/src/basic/vx_util/Makefile.am index de788d7202..ae466f0963 100644 --- a/met/src/basic/vx_util/Makefile.am +++ b/met/src/basic/vx_util/Makefile.am @@ -69,7 +69,7 @@ libvx_util_a_SOURCES = ascii_table.cc ascii_table.h \ GridOffset.h GridOffset.cc \ observation.h observation.cc \ stat_column_defs.h \ - handle_openmp.h handle_openmp.cc \ + handle_openmp.h handle_openmp.cc \ RectangularTemplate.h RectangularTemplate.cc $(OPT_PYTHON_SOURCES) libvx_util_a_CPPFLAGS = ${MET_CPPFLAGS} From b2a6c1e774c32f9885d7a3409bcd62ccd4ef91eb Mon Sep 17 00:00:00 2001 From: Maff Glover Date: Thu, 16 Dec 2021 18:35:53 +0000 Subject: [PATCH 23/27] Feature #1926: Lengthen OMP_NUM_THREADS debug message, plus user guide info. --- met/docs/Users_Guide/index.rst | 1 + met/docs/Users_Guide/openmp.rst | 62 ++++++++++++++++++++++++++ met/src/basic/vx_util/handle_openmp.cc | 3 +- 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 met/docs/Users_Guide/openmp.rst diff --git a/met/docs/Users_Guide/index.rst b/met/docs/Users_Guide/index.rst index 5ee52862e0..3d75a577e7 100644 --- a/met/docs/Users_Guide/index.rst +++ b/met/docs/Users_Guide/index.rst @@ -45,6 +45,7 @@ The National Center for Atmospheric Research (NCAR) is sponsored by NSF. The DTC overview installation + openmp data_io config_options config_options_tc diff --git a/met/docs/Users_Guide/openmp.rst b/met/docs/Users_Guide/openmp.rst new file mode 100644 index 0000000000..62b2e55d96 --- /dev/null +++ b/met/docs/Users_Guide/openmp.rst @@ -0,0 +1,62 @@ +.. _openmp: + +OpenMP +====== + +Introduction +____________ + +There are a number of different ways of parallelizing code. OpenMP offers parallelism within a single shared-memory workstation or supercomputer node. The programmer writes OpenMP directives into the code to parallelize particular code regions. + +When a parallelized code region is reached, which we shall hereafter call a parallel region, a number of threads are spawned and work is shared among them. Running on different cores, this reduces the execution time. At the end of the parallel region, the code returns to single-thread execution. + +A limited number of code regions are parallelized in MET. As a consequence, there are limits to the overall speed gains achievable. Only the parallel regions of code will get faster with more threads, leaving the remaining serial portions to dominate the runtime. + +Not all top-level executables use parallelized code. If OpenMP is available, a log message will appear inviting the user to increase the number of threads for faster runtimes. + +Setting the number of threads +_____________________________ + +The number of threads is controlled by the environment variable `OMP_NUM_THREADS`. For example, on a quad core machine, the user might choose to run on 4 threads: + +.. code :: bash + + export OMP_NUM_THREADS=4 + +Alternatively, the variable may be specified as a prefix to the executable itself. For example: + +.. code :: bash + + OMP_NUM_THREADS=4 + +The case where this variable remains unset is handled inside the code, which defaults to a single thread. + +There are choices when deciding how many threads to use. To perform a single run as fast as possible, it would likely be appropriate to use as many threads as there are (physical) cores available on the specific system. However, it is not a cast-iron guarantee that more threads will always translate into more speed. In theory, there is a chance that running across multiple NUMA regions may carry negative performance impacts. This has not been observed in practice, however. + +A lower thread count is appropriate when time-to-solution is not so critical, because cores remain idle when the code is not inside a parallel region. Fewer threads typically means better resource utilization. + +Which code is parallelized? +___________________________ + +Regions of parallelized code are: + + * :code:`fractional_coverage (data_plane_util.cc)` + +Only the following top-level executables can presently benefit from OpenMP parallelization: + + * :code:`grid_stat` + * :code:`ensemble_stat` + * :code:`grid_ens_prod` + +Thread binding +______________ + +It is normally beneficial to bind threads to particular cores, sometimes called *affinitization*. There are a few reasons for this, but at the very least it guarantees that threads remain evenly distributed across the available cores. Otherwise, the operating system may migrate threads between cores during a run. + +OpenMP provides some environment variables to handle this: :code:`OMP_PLACES` and :code:`OMP_PROC_BIND`. We anticipate that the effect of setting only :code:`OMP_PROC_BIND=true` would be neutral-to-positive. + +However, there are sometimes compiler-specific environment variables. Instead, thread affinitization is sometimes handled by MPI launchers, since OpenMP is often used in MPI codes to reduce intra-node communications. + +Where code is running in a production context, it is worth being familiar with the binding / affinitization method on the particular system and building it into any relevant scripting. + + diff --git a/met/src/basic/vx_util/handle_openmp.cc b/met/src/basic/vx_util/handle_openmp.cc index 56ee65abca..f23d09b4ca 100644 --- a/met/src/basic/vx_util/handle_openmp.cc +++ b/met/src/basic/vx_util/handle_openmp.cc @@ -28,7 +28,8 @@ void init_openmp() { const char* env_omp_num_threads = std::getenv("OMP_NUM_THREADS"); if (!env_omp_num_threads) { mlog << Debug(2) << "OMP_NUM_THREADS is not set." - << " Defaulting to 1 thread. \n"; + << " Defaulting to 1 thread." + << " Recommend setting OMP_NUM_THREADS for faster runtimes.\n"; omp_set_num_threads(1); } From f39313d6b343fad464aeaec2ec7d482111afc48f Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Fri, 17 Dec 2021 16:50:25 -0700 Subject: [PATCH 24/27] Set up the Runtime Environment Variables Section and started adding info from openmp.rst --- met/docs/Users_Guide/config_options.rst | 104 +++++++++++++++++++++--- 1 file changed, 94 insertions(+), 10 deletions(-) diff --git a/met/docs/Users_Guide/config_options.rst b/met/docs/Users_Guide/config_options.rst index e6d2349cc4..ce81ee279b 100644 --- a/met/docs/Users_Guide/config_options.rst +++ b/met/docs/Users_Guide/config_options.rst @@ -202,6 +202,12 @@ to use one configuration file rather than maintianing many very similar ones. An error in the syntax of a configuration file will result in an error from the MET tool stating the location of the parsing error. +Runtime Environment Variables +----------------------------- + +MET_BASE +^^^^^^^^ + The MET_BASE variable is defined in the code at compilation time as the path to the MET shared data. These are things like the default configuration files, common polygons and color scales. MET_BASE may be used in the MET configuration @@ -209,6 +215,9 @@ files when specifying paths and the appropriate path will be substituted in. If MET_BASE is defined as an environment variable, its value will be used instead of the one defined at compilation time. +MET_OBS_ERROR_TABLE +^^^^^^^^^^^^^^^^^^^ + The MET_OBS_ERROR_TABLE environment variable can be set to specify the location of an ASCII file defining observation error information. The default table can be found in the installed *share/met/table_files/obs_error_table.txt*. This @@ -236,6 +245,9 @@ values and randomly perturbed ensemble member values. Values less than MIN are reset to the mimimum value and values greater than MAX are reset to the maximum value. A value of NA indicates that the variable is unbounded. +MET_GRIB_TABLES +^^^^^^^^^^^^^^^ + The MET_GRIB_TABLES environment variable can be set to specify the location of custom GRIB tables. It can either be set to a specific file name or to a directory containing custom GRIB tables files. These file names must begin with @@ -289,9 +301,81 @@ References: | `NCEP WMO GRIB2 Documentation `_ | +OMP_NUM_THREADS +^^^^^^^^^^^^^^^ + +**Introduction** + +There are a number of different ways of parallelizing code. OpenMP offers +parallelism within a single shared-memory workstation or supercomputer node. +The programmer writes OpenMP directives into the code to parallelize +particular code regions. + +When a parallelized code region is reached, which we shall hereafter call a +parallel region, a number of threads are spawned and work is shared among them. +Running on different cores, this reduces the execution time. At the end of the +parallel region, the code returns to single-thread execution. + +A limited number of code regions are parallelized in MET. As a consequence, +there are limits to the A limited number of code regions are parallelized +in MET. As a consequence, there are limits to the leaving the remaining +serial portions to dominate the runtime. + +Not all top-level executables use parallelized code. If OpenMP is available, +a log message will appear inviting the user to increase the number of threads +for faster runtimes. + +**Setting the number of threads** + +he number of threads is controlled by the environment variable +*OMP_NUM_THREADS* . For example, on a quad core machine, the user might choose +to run on 4 threads: + +.. code :: bash + + export OMP_NUM_THREADS=4 + +Alternatively, the variable may be specified as a prefix to the executable +itself. For example: + +.. code :: bash + + OMP_NUM_THREADS=4 + +The case where this variable remains unset is handled inside the code, which +defaults to a single thread. + +here are choices when deciding how many threads to use. To perform a single run +as fast as possible, it would likely be appropriate to use as many threads as +there are (physical) cores available on the specific system. However, it is not +a cast-iron guarantee that more threads will always translate into more speed. +In theory, there is a chance that running across multiple NUMA regions may +carry negative performance impacts. This has not been observed in practice, +however. + +A lower thread count is appropriate when time-to-solution is not so critical, +because cores remain idle when the code is not inside a parallel region. Fewer +threads typically means better resource utilization. + +**Which code is parallized?** + +Regions of parallelized code are: + + * :code:`fractional_coverage (data_plane_util.cc)` + +Only the following top-level executables can presently benefit from OpenMP +parallelization: + + * :code:`grid_stat` + * :code:`ensemble_stat` + * :code:`grid_ens_prod` + +**Thread Binding** + +coming soon... Settings common to multiple tools -_________________________________ +--------------------------------- .. _exit_on_warning: @@ -2190,10 +2274,10 @@ are empty. Note: grib_code 11 is equivalent to obs_var "TMP". } Settings specific to individual tools -_____________________________________ +------------------------------------- EnsembleStatConfig_default -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ .. _ens: @@ -2505,7 +2589,7 @@ used for random assignment of ranks when they are tied. } MODEAnalysisConfig_default -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ MODE line options are used to create filters that determine which MODE output lines are read in and processed. The MODE line options are numerous. They @@ -2843,7 +2927,7 @@ MET User's Guide for a description of these attributes. MODEConfig_default -~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ .. _quilt: @@ -3158,7 +3242,7 @@ much more flexible "regrid" option may be used instead. shift_right = 0; PB2NCConfig_default -~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ The PB2NC tool filters out observations from PREPBUFR or BUFR files using the following criteria: @@ -3484,7 +3568,7 @@ stack (most quality controlled) or the bottom of the event stack (most raw). event_stack_flag = TOP; SeriesAnalysisConfig_default -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. _block_size: @@ -3539,7 +3623,7 @@ grid is large. } STATAnalysisConfig_default -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ .. _jobs: @@ -4008,7 +4092,7 @@ confidence intervals computed for the aggregated statistics. vif_flag = FALSE; WaveletStatConfig_default -~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ .. _grid_decomp_flag: @@ -4099,7 +4183,7 @@ similar to the "fcst_raw_plot" described in the "Settings common to multiple tools" section. WWMCARegridConfig_default -~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ .. _to_grid: From 63b1fced17f036688ad9d69c002411c2038fac15 Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Tue, 21 Dec 2021 16:57:50 -0700 Subject: [PATCH 25/27] Updated documenation and fixed typos --- met/docs/Users_Guide/config_options.rst | 31 ++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/met/docs/Users_Guide/config_options.rst b/met/docs/Users_Guide/config_options.rst index ce81ee279b..34be0151bd 100644 --- a/met/docs/Users_Guide/config_options.rst +++ b/met/docs/Users_Guide/config_options.rst @@ -317,8 +317,8 @@ Running on different cores, this reduces the execution time. At the end of the parallel region, the code returns to single-thread execution. A limited number of code regions are parallelized in MET. As a consequence, -there are limits to the A limited number of code regions are parallelized -in MET. As a consequence, there are limits to the leaving the remaining +there are limits to the overall speed gains acheivable. Only the parallel +regions of code will get faster with more threads, leaving the remaining serial portions to dominate the runtime. Not all top-level executables use parallelized code. If OpenMP is available, @@ -327,7 +327,7 @@ for faster runtimes. **Setting the number of threads** -he number of threads is controlled by the environment variable +The number of threads is controlled by the environment variable *OMP_NUM_THREADS* . For example, on a quad core machine, the user might choose to run on 4 threads: @@ -345,13 +345,13 @@ itself. For example: The case where this variable remains unset is handled inside the code, which defaults to a single thread. -here are choices when deciding how many threads to use. To perform a single run +There are choices when deciding how many threads to use. To perform a single run as fast as possible, it would likely be appropriate to use as many threads as there are (physical) cores available on the specific system. However, it is not a cast-iron guarantee that more threads will always translate into more speed. -In theory, there is a chance that running across multiple NUMA regions may -carry negative performance impacts. This has not been observed in practice, -however. +In theory, there is a chance that running across multiple non-uniform memory +access (NUMA) regions may carry negative performance impacts. This has not been +observed in practice, however. A lower thread count is appropriate when time-to-solution is not so critical, because cores remain idle when the code is not inside a parallel region. Fewer @@ -372,7 +372,22 @@ parallelization: **Thread Binding** -coming soon... +It is normally beneficial to bind threads to particular cores, sometimes called +*affinitization*. There are a few reasons for this, but at the very least it +guarantees that threads remain evenly distributed across the available cores. +Otherwise, the operating system may migrate threads between cores during a run. + +OpenMP provides some environment variables to handle this: :code:`OMP_PLACES` +and :code:`OMP_PROC_BIND`. We anticipate that the effect of setting only +:code:`OMP_PROC_BIND=true` would be neutral-to-positive. + +However, there are sometimes compiler-specific environment variables. Instead, +thread affinitization is sometimes handled by MPI launchers, since OpenMP is +often used in MPI codes to reduce intra-node communications. + +Where code is running in a production context, it is worth being familiar with +the binding / affinitization method on the particular system and building it +into any relevant scripting. Settings common to multiple tools --------------------------------- From 3e89e56a5f450606bcdbc5353df0af45094d9a69 Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Tue, 21 Dec 2021 17:01:52 -0700 Subject: [PATCH 26/27] removed openmp.rst in favor of having the information contained in a Runtime Environment Variables section in config_options.rst and removed openmp from index.rst --- met/docs/Users_Guide/index.rst | 1 - met/docs/Users_Guide/openmp.rst | 62 --------------------------------- 2 files changed, 63 deletions(-) delete mode 100644 met/docs/Users_Guide/openmp.rst diff --git a/met/docs/Users_Guide/index.rst b/met/docs/Users_Guide/index.rst index 3d75a577e7..5ee52862e0 100644 --- a/met/docs/Users_Guide/index.rst +++ b/met/docs/Users_Guide/index.rst @@ -45,7 +45,6 @@ The National Center for Atmospheric Research (NCAR) is sponsored by NSF. The DTC overview installation - openmp data_io config_options config_options_tc diff --git a/met/docs/Users_Guide/openmp.rst b/met/docs/Users_Guide/openmp.rst deleted file mode 100644 index 62b2e55d96..0000000000 --- a/met/docs/Users_Guide/openmp.rst +++ /dev/null @@ -1,62 +0,0 @@ -.. _openmp: - -OpenMP -====== - -Introduction -____________ - -There are a number of different ways of parallelizing code. OpenMP offers parallelism within a single shared-memory workstation or supercomputer node. The programmer writes OpenMP directives into the code to parallelize particular code regions. - -When a parallelized code region is reached, which we shall hereafter call a parallel region, a number of threads are spawned and work is shared among them. Running on different cores, this reduces the execution time. At the end of the parallel region, the code returns to single-thread execution. - -A limited number of code regions are parallelized in MET. As a consequence, there are limits to the overall speed gains achievable. Only the parallel regions of code will get faster with more threads, leaving the remaining serial portions to dominate the runtime. - -Not all top-level executables use parallelized code. If OpenMP is available, a log message will appear inviting the user to increase the number of threads for faster runtimes. - -Setting the number of threads -_____________________________ - -The number of threads is controlled by the environment variable `OMP_NUM_THREADS`. For example, on a quad core machine, the user might choose to run on 4 threads: - -.. code :: bash - - export OMP_NUM_THREADS=4 - -Alternatively, the variable may be specified as a prefix to the executable itself. For example: - -.. code :: bash - - OMP_NUM_THREADS=4 - -The case where this variable remains unset is handled inside the code, which defaults to a single thread. - -There are choices when deciding how many threads to use. To perform a single run as fast as possible, it would likely be appropriate to use as many threads as there are (physical) cores available on the specific system. However, it is not a cast-iron guarantee that more threads will always translate into more speed. In theory, there is a chance that running across multiple NUMA regions may carry negative performance impacts. This has not been observed in practice, however. - -A lower thread count is appropriate when time-to-solution is not so critical, because cores remain idle when the code is not inside a parallel region. Fewer threads typically means better resource utilization. - -Which code is parallelized? -___________________________ - -Regions of parallelized code are: - - * :code:`fractional_coverage (data_plane_util.cc)` - -Only the following top-level executables can presently benefit from OpenMP parallelization: - - * :code:`grid_stat` - * :code:`ensemble_stat` - * :code:`grid_ens_prod` - -Thread binding -______________ - -It is normally beneficial to bind threads to particular cores, sometimes called *affinitization*. There are a few reasons for this, but at the very least it guarantees that threads remain evenly distributed across the available cores. Otherwise, the operating system may migrate threads between cores during a run. - -OpenMP provides some environment variables to handle this: :code:`OMP_PLACES` and :code:`OMP_PROC_BIND`. We anticipate that the effect of setting only :code:`OMP_PROC_BIND=true` would be neutral-to-positive. - -However, there are sometimes compiler-specific environment variables. Instead, thread affinitization is sometimes handled by MPI launchers, since OpenMP is often used in MPI codes to reduce intra-node communications. - -Where code is running in a production context, it is worth being familiar with the binding / affinitization method on the particular system and building it into any relevant scripting. - - From 83688f5a24268d89033da746eacf6a6cef734bed Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Tue, 21 Dec 2021 17:07:30 -0700 Subject: [PATCH 27/27] Fixed typo --- met/docs/Users_Guide/config_options.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/met/docs/Users_Guide/config_options.rst b/met/docs/Users_Guide/config_options.rst index 34be0151bd..b56cb65d75 100644 --- a/met/docs/Users_Guide/config_options.rst +++ b/met/docs/Users_Guide/config_options.rst @@ -357,7 +357,7 @@ A lower thread count is appropriate when time-to-solution is not so critical, because cores remain idle when the code is not inside a parallel region. Fewer threads typically means better resource utilization. -**Which code is parallized?** +**Which code is parallelized?** Regions of parallelized code are: