Merge branch 'develop'

CMakeLists.txt

@@ -10,7 +10,7 @@ project (APEX CXX C)
 set (APEX_DESCRIPTION "Autonomic Performance Environment for eXascale" CACHE STRING "APEX project description")
 set (APEX_VERSION_MAJOR 2 CACHE STRING "APEX Major Version")
 set (APEX_VERSION_MINOR 6 CACHE STRING "APEX Minor Version")
-set (APEX_VERSION_PATCH 4 CACHE STRING "APEX Patch Version")
+set (APEX_VERSION_PATCH 5 CACHE STRING "APEX Patch Version")
 set (APEX_HOMEPAGE_URL "http://github.com/UO-OACISS/apex" CACHE STRING "APEX homepage URL")
 
 cmake_policy(VERSION 2.8.12)

cmake/Modules/APEX_DefaultOptions.cmake

@@ -87,9 +87,8 @@ if(DEFINED BUILD_BFD)
 endif()
 
 # Provide some backwards compatability
-if(DEFINED BUILD_OMPT)
-    message(WARNING "BUILD_OMPT is deprecated - please use APEX_BUILD_OMPT")
-    set(APEX_BUILD_OMPT ${BUILD_OMPT} CACHE BOOL "")
+if(DEFINED BUILD_OMPT OR DEFINED APEX_BUILD_OMPT)
+    message(FATAL_ERROR "APEX_BUILD_OMPT is disabled. If OpenMP support is needed, use a compiler with compliant Tool support.")
 endif()
 
 # Provide some backwards compatability
@@ -98,7 +97,6 @@ if(DEFINED BUILD_OTF2)
     set(APEX_BUILD_OTF2 ${BUILD_OTF2} CACHE BOOL "")
 endif()
 
-
 # All CMAKE options for the APEX project...
 option (APEX_BUILD_TESTS "Build APEX tests (for 'make test')" FALSE)
 option (APEX_BUILD_EXAMPLES "Build APEX examples" FALSE)

cmake/Modules/FindBFD.cmake

@@ -54,8 +54,8 @@ if((APEX_BUILD_BFD OR (NOT BFD_FOUND)) AND NOT APPLE)
   set(TMP_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
   set(TMP_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fPIC")
   ExternalProject_Add(project_binutils
-    URL "http://ftp.gnu.org/gnu/binutils/binutils-2.37.tar.bz2"
-    URL_HASH SHA256=67fc1a4030d08ee877a4867d3dcab35828148f87e1fd05da6db585ed5a166bd4
+    URL "http://ftp.gnu.org/gnu/binutils/binutils-2.42.tar.bz2"
+    URL_HASH SHA256=aa54850ebda5064c72cd4ec2d9b056c294252991486350d9a97ab2a6dfdfaf12
     CONFIGURE_COMMAND <SOURCE_DIR>/configure CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=${TMP_C_FLAGS} CXXFLAGS=${TMP_CXX_FLAGS} LDFLAGS=${TMP_LINKER_FLAGS} --prefix=${CMAKE_INSTALL_PREFIX} --disable-dependency-tracking --enable-interwork --disable-multilib --enable-shared --enable-64-bit-bfd --target=${TARGET_ARCH} --enable-install-libiberty --disable-gold --program-prefix=g --disable-nls --disable-ld --disable-lto --disable-gas
     BUILD_COMMAND make MAKEINFO=true -j${MAKEJOBS}
     INSTALL_COMMAND make MAKEINFO=true install

doc/Doxyfile.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "Autonomic Performance Environment for eXascale (APEX)"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 2.6.4
+PROJECT_NUMBER         = 2.6.5
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

doc/webdocs/docs/install.md

@@ -8,7 +8,7 @@ APEX is integrated into the [HPX runtime](https://hpx.stellar-group.org), and is
 -DHPX_WITH_APEX=TRUE
 ```
 
-The `-DHPX_WITH_APEX_TAG=develop` can be used to indicate a specific release version of APEX, or to use a specific GitHub branch of APEX.  We recommend using the default configured version that comes with HPX (currently `v2.6.4`) or the `develop` branch.  Additional CMake flags include:
+The `-DHPX_WITH_APEX_TAG=develop` can be used to indicate a specific release version of APEX, or to use a specific GitHub branch of APEX.  We recommend using the default configured version that comes with HPX (currently `v2.6.5`) or the `develop` branch.  Additional CMake flags include:
 
 * `-DAPEX_WITH_LM_SENSORS=TRUE` to enable [LM sensors](https://hwmon.wiki.kernel.org/lm_sensors) support (assumed to be installed in default system paths)
 * `-DAPEX_WITH_PAPI=TRUE` and `-DPAPI_ROOT=...` to enable [PAPI](https://icl.utk.edu/papi/) support
@@ -23,12 +23,12 @@ The `-DHPX_WITH_APEX_TAG=develop` can be used to indicate a specific release ver
 
 APEX is open source, and available on Github at <http://github.com/UO-OACISS/apex>.
 
-For stability, most users will want to download [the most recent release](https://github.com/UO-OACISS/apex/releases) of APEX (for example, v2.6.4):
+For stability, most users will want to download [the most recent release](https://github.com/UO-OACISS/apex/releases) of APEX (for example, v2.6.5):
 
 ```bash
-wget https://github.com/UO-OACISS/apex/archive/refs/tags/v2.6.4.tar.gz
-tar -xvzf v2.6.4.tar.gz
-cd apex-2.6.4
+wget https://github.com/UO-OACISS/apex/archive/refs/tags/v2.6.5.tar.gz
+tar -xvzf v2.6.5.tar.gz
+cd apex-2.6.5
 ```
 
 Other users may want to work with the most recent code available, in which case you can clone the git repo:
@@ -153,7 +153,7 @@ The process for building APEX is:
 2) Enter the repo directory:
 
 ```bash
-cd apex-2.6.4
+cd apex-2.6.5
 ```
 
 3) configure using CMake:

src/apex/CMakeLists_hpx.cmake

@@ -19,7 +19,7 @@ hpx_info("apex" "Will build APEX")
 
 set (APEX_VERSION_MAJOR 2)
 set (APEX_VERSION_MINOR 6)
-set (APEX_VERSION_PATCH 4)
+set (APEX_VERSION_PATCH 5)
 
 if (NOT APEX_ROOT)
   if (EXISTS ${HPX_SOURCE_DIR}/apex)

src/apex/apex.cpp

@@ -580,6 +580,17 @@ uint64_t init(const char * thread_name, uint64_t comm_rank,
 #else
     enable_memory_wrapper();
 #endif
+    if (apex_options::delay_memory_tracking()) {
+        if (instance->get_node_id() == 0) {
+            std::cout << "Pausing memory tracking until further notice..." << std::endl;
+        }
+        controlMemoryWrapper(false);
+    } else {
+        if (instance->get_node_id() == 0) {
+            std::cout << "Enabling memory tracking!" << std::endl;
+        }
+        controlMemoryWrapper(true);
+    }
 
     // It's now safe to initialize CUDA and/or HIP and/or Level0
     dynamic::cuda::init();
@@ -1659,6 +1670,7 @@ void finalize_plugins(void) {
 
 std::string dump(bool reset, bool finalizing) {
     in_apex prevent_deadlocks;
+    static size_t index{0};
     // if APEX is disabled, do nothing.
     if (apex_options::disable() == true ||
         (!finalizing && apex_options::use_final_output_only()))
@@ -1676,6 +1688,15 @@ std::string dump(bool reset, bool finalizing) {
     dynamic::cuda::flush();
     dynamic::roctracer::flush();
     dynamic::level0::flush();
+    /* only track after N calls to apex::dump() */
+    index = index + 1;
+    if (apex_options::delay_memory_tracking() &&
+        index > apex_options::delay_memory_iterations()) {
+        if (instance->get_node_id() == 0) {
+            std::cout << "Enabling memory tracking!" << std::endl;
+        }
+        controlMemoryWrapper(true);
+    } 
     if (_notify_listeners) {
         dump_event_data data(instance->get_node_id(),
             thread_instance::get_id(), reset);

src/apex/apex_bfd.cpp

@@ -32,9 +32,6 @@
 #define PACKAGE_VERSION
 #endif
 #include <bfd.h>
-#if APEX_BFD >= 022300
-#include <elf-bfd.h>
-#endif
 #include <dirent.h>
 #include <stdint.h>
 #include <cctype>
@@ -135,7 +132,7 @@ struct ApexBfdModule
       return (bfdOpen = false);
     }
 
-#if APEX_BFD >= 022200
+#if defined(BFD_DECOMPRESS)
     // Decompress sections
     bfdImage->flags |= BFD_DECOMPRESS;
 #endif
@@ -1025,7 +1022,7 @@ void Apex_bfd_internal_locateAddress(bfd * bfdptr,
   // ApexBfdInfo fields without an extra copy.  This also means
   // that the pointers in ApexBfdInfo must never be deleted
   // since they point directly into the module's BFD.
-#if (APEX_BFD >= 022200)
+#if defined(bfd_find_nearest_line_discriminator)
   data.found = bfd_find_nearest_line_discriminator(bfdptr, section,
       data.module->syms, (data.info.probeAddr - vma),
       &data.info.filename, &data.info.funcname,

src/apex/apex_mpi.cpp

@@ -183,6 +183,7 @@ void  _symbol( MPI_Fint *ierr ) { \
     /* Get the total bytes transferred, record it, and return it
        to be used for bandwidth calculation */
     inline double getBytesTransferred(int count, MPI_Datatype datatype, const char * function) {
+        apex::in_apex prevent_memory_tracking;
         int typesize = 0;
         PMPI_Type_size( datatype, &typesize );
         double bytes = (double)(typesize) * (double)(count);
@@ -192,6 +193,7 @@ void  _symbol( MPI_Fint *ierr ) { \
         return bytes;
     }
     inline double getBytesTransferred2(const int count, MPI_Datatype datatype, MPI_Comm comm, const char * function) {
+        apex::in_apex prevent_memory_tracking;
         int typesize = 0;
         int commsize = 0;
         PMPI_Type_size( datatype, &typesize );
@@ -203,6 +205,7 @@ void  _symbol( MPI_Fint *ierr ) { \
         return bytes;
     }
     inline double getBytesTransferred3(const int * count, MPI_Datatype datatype, MPI_Comm comm, const char * function) {
+        apex::in_apex prevent_memory_tracking;
         int typesize = 0;
         int commsize = 0;
         PMPI_Type_size( datatype, &typesize );
@@ -216,22 +219,8 @@ void  _symbol( MPI_Fint *ierr ) { \
         apex::sample_value(name, bytes);
         return bytes;
     }
-    inline bool checkAvailableMemory(double bytes_requested) {
-        std::array<double,2> available{apex::getAvailableMemory()};
-        //std::cout << "Available: " << available[0] << ", " << available[1];
-        //std::cout << " Requested: " << bytes_requested << std::endl;
-        if (bytes_requested >
-            (apex::apex_options::validate_mpi_memory_usage_fraction()*available[0]) ||
-            bytes_requested >
-            (apex::apex_options::validate_mpi_memory_usage_fraction()*available[1])) {
-            std::cerr << "Warning! Requesting too much memory!" << std::endl;
-            std::cerr << "Expect a failure! Here's a backtrace:" << std::endl;
-            apex_print_backtrace();
-            return false;
-        }
-        return true;
-    }
-    inline void getBandwidth(double bytes, std::shared_ptr<apex::task_wrapper> task, const char * function) {
+     inline void getBandwidth(double bytes, std::shared_ptr<apex::task_wrapper> task, const char * function) {
+        apex::in_apex prevent_memory_tracking;
         if ((task != nullptr) && (task->prof != nullptr)) {
             std::string name("BW (Bytes/second) : ");
             name.append(function);
@@ -244,9 +233,6 @@ void  _symbol( MPI_Fint *ierr ) { \
         int tag, MPI_Comm comm, MPI_Request *request) {
         /* Get the byte count */
         double bytes = getBytesTransferred(count, datatype, "MPI_Isend");
-        if (apex::apex_options::validate_mpi_memory_usage()) {
-            checkAvailableMemory(bytes);
-        }
         /* start the timer */
         MPI_START_TIMER
         apex::recordMetric("Send Bytes", bytes);
@@ -276,9 +262,6 @@ void  _symbol( void * buf, MPI_Fint * count, MPI_Fint * datatype, MPI_Fint * des
         /* Get the byte count */
         double bytes = getBytesTransferred(count, datatype, "MPI_Irecv");
         MPI_START_TIMER
-        if (apex::apex_options::validate_mpi_memory_usage()) {
-            checkAvailableMemory(bytes);
-        }
         apex::recordMetric("Recv Bytes", bytes);
         int retval = PMPI_Irecv(buf, count, datatype, source, tag, comm,
             request);
@@ -305,9 +288,6 @@ void  _symbol( void * buf, MPI_Fint * count, MPI_Fint * datatype, MPI_Fint * sou
         int tag, MPI_Comm comm){
         /* Get the byte count */
         double bytes = getBytesTransferred(count, datatype, "MPI_Send");
-        if (apex::apex_options::validate_mpi_memory_usage()) {
-            checkAvailableMemory(bytes);
-        }
         /* start the timer */
         MPI_START_TIMER
         apex::recordMetric("Send Bytes", bytes);
@@ -334,9 +314,6 @@ void  _symbol( void * buf, MPI_Fint * count, MPI_Fint * datatype, MPI_Fint * des
         int source, int tag, MPI_Comm comm, MPI_Status *status){
         /* Get the byte count */
         double bytes = getBytesTransferred(count, datatype, "MPI_Recv");
-        if (apex::apex_options::validate_mpi_memory_usage()) {
-            checkAvailableMemory(bytes);
-        }
         MPI_START_TIMER
         apex::recordMetric("Recv Bytes", bytes);
         int retval = PMPI_Recv(buf, count, datatype, source, tag, comm, status);
@@ -374,9 +351,6 @@ void  _symbol( void * buf, MPI_Fint * count, MPI_Fint * datatype, MPI_Fint * sou
         /* Get the byte count */
         double sbytes = getBytesTransferred(sendcount, sendtype, "MPI_Gather sendbuf");
         double rbytes = amIroot(comm, root) ? getBytesTransferred2(recvcount, recvtype, comm, "MPI_Gather recvbuf") : 0.0;
-        if (apex::apex_options::validate_mpi_memory_usage()) {
-            checkAvailableMemory(sbytes+rbytes);
-        }
         MPI_START_TIMER
         apex::recordMetric("Send Bytes", sbytes);
         apex::recordMetric("Recv Bytes", rbytes);
@@ -408,9 +382,6 @@ void _symbol(void * sendbuf, MPI_Fint *sendcnt, MPI_Fint *sendtype, void * recvb
         /* Get the byte count */
         double sbytes = getBytesTransferred(count, datatype, "MPI_Allreduce sendbuf");
         double rbytes = getBytesTransferred2(count, datatype, comm, "MPI_Allreduce recvbuf");
-        if (apex::apex_options::validate_mpi_memory_usage()) {
-            checkAvailableMemory(sbytes+rbytes);
-        }
         MPI_START_TIMER
         apex::recordMetric("Send Bytes", sbytes);
         apex::recordMetric("Recv Bytes", rbytes);
@@ -440,9 +411,6 @@ void _symbol(void * sendbuf, void * recvbuf, MPI_Fint *count, MPI_Fint *datatype
         /* Get the byte count */
         double sbytes = getBytesTransferred(count, datatype, "MPI_Reduce sendbuf");
         double rbytes = amIroot(comm, root) ? getBytesTransferred2(count, datatype, comm, "MPI_Reduce recvbuf") : 0.0;
-        if (apex::apex_options::validate_mpi_memory_usage()) {
-            checkAvailableMemory(sbytes+rbytes);
-        }
         MPI_START_TIMER
         apex::recordMetric("Send Bytes", sbytes);
         apex::recordMetric("Recv Bytes", rbytes);
@@ -474,9 +442,6 @@ void _symbol(void * sendbuf, void * recvbuf, MPI_Fint *count, MPI_Fint *datatype
         //PMPI_Comm_rank(comm, &commrank);
         /* Get the byte count */
         double sbytes = getBytesTransferred(count, datatype, "MPI_Bcast");
-        if (apex::apex_options::validate_mpi_memory_usage()) {
-            checkAvailableMemory(sbytes);
-        }
         MPI_START_TIMER
         //if (root == commrank) {
             apex::recordMetric("Send Bytes", sbytes);
@@ -534,9 +499,6 @@ void _symbol(MPI_Fint *count, MPI_Fint * array_of_requests, MPI_Fint *ierr) { \
         /* Get the byte count */
         double sbytes = getBytesTransferred(sendcount, sendtype, "MPI_Alltoall sendbuf");
         double rbytes = getBytesTransferred2(recvcount, recvtype, comm, "MPI_Alltoall recvbuf");
-        if (apex::apex_options::validate_mpi_memory_usage()) {
-            checkAvailableMemory(sbytes+rbytes);
-        }
         MPI_START_TIMER
         apex::recordMetric("Send Bytes", sbytes);
         apex::recordMetric("Recv Bytes", rbytes);
@@ -567,9 +529,6 @@ MPI_Fint *recvcnt, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *ierr) { \
         /* Get the byte count */
         double sbytes = getBytesTransferred(sendcount, sendtype, "MPI_Allgather sendbuf");
         double rbytes = getBytesTransferred2(recvcount, recvtype, comm, "MPI_Allgather recvbuf");
-        if (apex::apex_options::validate_mpi_memory_usage()) {
-            checkAvailableMemory(sbytes+rbytes);
-        }
         MPI_START_TIMER
         apex::recordMetric("Send Bytes", sbytes);
         apex::recordMetric("Recv Bytes", rbytes);
@@ -601,10 +560,7 @@ void _symbol(void * sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, void * rec
         MPI_Datatype datatype_recv, MPI_Comm communicator) {
         /* Get the byte count */
         double sbytes = getBytesTransferred(count_send, datatype_send, "MPI_Allgatherv sendbuf");
-        double rbytes = getBytesTransferred3(counts_recv, datatype_recv, communicator, "MPI_Allgatherv recvbuf");
-        if (apex::apex_options::validate_mpi_memory_usage()) {
-            checkAvailableMemory(sbytes+rbytes);
-        }
+        double rbytes = 0; //getBytesTransferred3(counts_recv, datatype_recv, communicator, "MPI_Allgatherv recvbuf");
         MPI_START_TIMER
         apex::recordMetric("Send Bytes", sbytes);
         apex::recordMetric("Recv Bytes", rbytes);
@@ -636,10 +592,7 @@ void _symbol(void * sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, void * rec
         MPI_Datatype recvtype, int root, MPI_Comm comm) {
         /* Get the byte count */
         double sbytes = getBytesTransferred(sendcount, sendtype, "MPI_Gatherv sendbuf");
-        double rbytes = amIroot(comm, root) ? getBytesTransferred3(recvcounts, recvtype, comm, "MPI_Gatherv recvbuf") : 0;
-        if (apex::apex_options::validate_mpi_memory_usage()) {
-            checkAvailableMemory(sbytes+rbytes);
-        }
+        double rbytes = 0; //amIroot(comm, root) ? getBytesTransferred3(recvcounts, recvtype, comm, "MPI_Gatherv recvbuf") : 0;
         MPI_START_TIMER
         apex::recordMetric("Send Bytes", sbytes);
         apex::recordMetric("Recv Bytes", rbytes);
@@ -671,9 +624,6 @@ void _symbol(void * sendbuf, MPI_Fint *sendcnt, MPI_Fint *sendtype, void * recvb
         /* Get the byte count */
         double sbytes = getBytesTransferred(sendcount, sendtype, "MPI_Sendrecv sendbuf");
         double rbytes = getBytesTransferred(recvcount, recvtype, "MPI_Sendrecv recvbuf");
-        if (apex::apex_options::validate_mpi_memory_usage()) {
-            checkAvailableMemory(sbytes+rbytes);
-        }
         MPI_START_TIMER
         apex::recordMetric("Send Bytes", sbytes);
         apex::recordMetric("Recv Bytes", rbytes);

src/apex/apex_types.h

@@ -329,6 +329,8 @@ inline unsigned int sc_nprocessors_onln(void)
     macro (APEX_PIN_APEX_THREADS, pin_apex_threads, bool, true, "Pin APEX asynchronous threads to the last core/PU on the system.") \
     macro (APEX_TRACK_CPU_MEMORY, track_cpu_memory, bool, false, "Track all malloc/free/new/delete calls to CPU memory and report leaks.") \
     macro (APEX_TRACK_GPU_MEMORY, track_gpu_memory, bool, false, "Track all malloc/free/new/delete calls to GPU memory and report leaks.") \
+    macro (APEX_DELAY_MEMORY_TRACKING, delay_memory_tracking, bool, false, "Delay memory tracking until explicitly enabled.") \
+    macro (APEX_DELAY_MEMORY_ITERATIONS, delay_memory_iterations, int, 1, "Delay memory tracking until after N calls to apex::dump().") \
     macro (APEX_TASK_SCATTERPLOT, task_scatterplot, bool, false, "Periodically sample APEX tasks, generating a scatterplot of time distributions.") \
     macro (APEX_TIME_TOP_LEVEL_OS_THREADS, top_level_os_threads, bool, false, "When registering threads, measure their lifetimes.") \
     macro (APEX_POLICY_DRAIN_TIMEOUT, policy_drain_timeout, int, 1000, "Internal usage only.") \
@@ -354,7 +356,7 @@ inline unsigned int sc_nprocessors_onln(void)
     macro (APEX_JUPYTER_SUPPORT, use_jupyter_support, int, false, "Internal use only.") \
     macro (APEX_KOKKOS_VERBOSE, use_kokkos_verbose, bool, false, "Enable verbose Kokkos autotuning and profiling messages.") \
     macro (APEX_KOKKOS_COUNTERS, use_kokkos_counters, bool, false, "Enable Kokkos counters.") \
-    macro (APEX_KOKKOS_TUNING, use_kokkos_tuning, bool, true, "Enable Kokkos autotuning.") \
+    macro (APEX_KOKKOS_TUNING, use_kokkos_tuning, bool, false, "Enable Kokkos autotuning.") \
     macro (APEX_KOKKOS_TUNING_WINDOW, kokkos_tuning_window, int, 5, "Minimum number of tests per candidate while autotuning.") \
     macro (APEX_KOKKOS_PROFILING_FENCES, use_kokkos_profiling_fences, bool, false, "Force Kokkos to fence after all Kokkos kernel launches (recommended, but not required).") \
     macro (APEX_START_DELAY_SECONDS, start_delay_seconds, int, 0, "Delay collection of APEX data for N seconds.") \