ch4/posix: decrease shm_limit_counter when freeing comm obj

src/mpi/coll/src/coll_impl.c

@@ -27,7 +27,7 @@
     - name        : MPIR_CVAR_COLLECTIVE_FALLBACK
       category    : COLLECTIVE
       type        : enum
-      default     : error
+      default     : silent
       class       : none
       verbosity   : MPI_T_VERBOSITY_USER_BASIC
       scope       : MPI_T_SCOPE_ALL_EQ

src/mpid/ch4/shm/posix/posix_init.c

@@ -257,6 +257,7 @@ int MPIDI_POSIX_coll_init(int rank, int size)
 int MPIDI_POSIX_coll_finalize(void)
 {
     int mpi_errno = MPI_SUCCESS;
+    static MPL_atomic_uint64_t MPIDI_POSIX_dummy_shm_limit_counter;
 
     MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_POSIX_COLL_FINALIZE);
     MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_POSIX_COLL_FINALIZE);
@@ -265,6 +266,11 @@ int MPIDI_POSIX_coll_finalize(void)
      * per node for intra-node collectives */
     mpi_errno = MPIDU_Init_shm_free(MPIDI_POSIX_global.shm_ptr);
 
+    /* MPIDI_POSIX_global.shm_ptr is freed but will be referenced during builtin
+     * comm free; here we set MPIDI_POSIX_shm_limit_counter as dummy counter to
+     * avoid segmentation fault */
+    MPIDI_POSIX_shm_limit_counter = &MPIDI_POSIX_dummy_shm_limit_counter;
+
     if (MPIDI_global.shm.posix.csel_root) {
         mpi_errno = MPIR_Csel_free(MPIDI_global.shm.posix.csel_root);
         MPIR_ERR_CHECK(mpi_errno);

src/mpid/ch4/shm/posix/release_gather/release_gather.c

@@ -265,8 +265,6 @@ int MPIDI_POSIX_mpi_release_gather_comm_init(MPIR_Comm * comm_ptr,
                 MPIR_Bcast_impl(&fallback, 1, MPI_INT, 0, comm_ptr, &errflag);
                 MPIR_ERR_SETANDJUMP(mpi_errno_ret, MPI_ERR_NO_MEM, "**nomem");
             } else {
-                /* More shm can be created, update the shared counter */
-                MPL_atomic_fetch_add_uint64(MPIDI_POSIX_shm_limit_counter, memory_to_be_allocated);
                 fallback = 0;
                 mpi_errno = MPIR_Bcast_impl(&fallback, 1, MPI_INT, 0, comm_ptr, &errflag);
                 if (mpi_errno) {
@@ -376,6 +374,10 @@ int MPIDI_POSIX_mpi_release_gather_comm_init(MPIR_Comm * comm_ptr,
         release_gather_info_ptr->reduce_buf_addr = NULL;
         release_gather_info_ptr->child_reduce_buf_addr = NULL;
 
+        RELEASE_GATHER_FIELD(comm_ptr, flags_shm_size) = flags_shm_size;
+        /* update the shared counter */
+        if (rank == 0)
+            MPL_atomic_fetch_add_uint64(MPIDI_POSIX_shm_limit_counter, flags_shm_size);
         mpi_errno =
             MPIDU_shm_alloc(comm_ptr, flags_shm_size,
                             (void **) &(release_gather_info_ptr->flags_addr), &mapfail_flag);
@@ -411,6 +413,12 @@ int MPIDI_POSIX_mpi_release_gather_comm_init(MPIR_Comm * comm_ptr,
     }
 
     if (initialize_bcast_buf) {
+        RELEASE_GATHER_FIELD(comm_ptr, bcast_shm_size) =
+            MPIR_CVAR_BCAST_INTRANODE_BUFFER_TOTAL_SIZE;
+        if (rank == 0)
+            MPL_atomic_fetch_add_uint64(MPIDI_POSIX_shm_limit_counter,
+                                        RELEASE_GATHER_FIELD(comm_ptr, bcast_shm_size));
+
         /* Allocate the shared memory for bcast buffer */
         mpi_errno =
             MPIDU_shm_alloc(comm_ptr, MPIR_CVAR_BCAST_INTRANODE_BUFFER_TOTAL_SIZE,
@@ -432,6 +440,12 @@ int MPIDI_POSIX_mpi_release_gather_comm_init(MPIR_Comm * comm_ptr,
         RELEASE_GATHER_FIELD(comm_ptr, child_reduce_buf_addr) =
             MPL_malloc(num_ranks * sizeof(void *), MPL_MEM_COLL);
 
+        RELEASE_GATHER_FIELD(comm_ptr, reduce_shm_size) =
+            MPIR_CVAR_BCAST_INTRANODE_BUFFER_TOTAL_SIZE;
+        if (rank == 0)
+            MPL_atomic_fetch_add_uint64(MPIDI_POSIX_shm_limit_counter,
+                                        RELEASE_GATHER_FIELD(comm_ptr, reduce_shm_size));
+
         mpi_errno =
             MPIDU_shm_alloc(comm_ptr, num_ranks * MPIR_CVAR_REDUCE_INTRANODE_BUFFER_TOTAL_SIZE,
                             (void **) &(RELEASE_GATHER_FIELD(comm_ptr, reduce_buf_addr)),
@@ -480,6 +494,11 @@ int MPIDI_POSIX_mpi_release_gather_comm_free(MPIR_Comm * comm_ptr)
         goto fn_exit;
     }
 
+    /* decrease shm memory limit counter */
+    if (comm_ptr->rank == 0)
+        MPL_atomic_fetch_sub_uint64(MPIDI_POSIX_shm_limit_counter,
+                                    RELEASE_GATHER_FIELD(comm_ptr, flags_shm_size));
+
     /* destroy and detach shared memory used for flags */
     mpi_errno = MPIDU_shm_free(RELEASE_GATHER_FIELD(comm_ptr, flags_addr));
     if (mpi_errno) {
@@ -492,6 +511,9 @@ int MPIDI_POSIX_mpi_release_gather_comm_free(MPIR_Comm * comm_ptr)
     }
 
     if (RELEASE_GATHER_FIELD(comm_ptr, bcast_buf_addr) != NULL) {
+        if (comm_ptr->rank == 0)
+            MPL_atomic_fetch_sub_uint64(MPIDI_POSIX_shm_limit_counter,
+                                        RELEASE_GATHER_FIELD(comm_ptr, bcast_shm_size));
         /* destroy and detach shared memory used for bcast buffer */
         mpi_errno = MPIDU_shm_free(RELEASE_GATHER_FIELD(comm_ptr, bcast_buf_addr));
         if (mpi_errno) {
@@ -505,6 +527,9 @@ int MPIDI_POSIX_mpi_release_gather_comm_free(MPIR_Comm * comm_ptr)
     }
 
     if (RELEASE_GATHER_FIELD(comm_ptr, reduce_buf_addr) != NULL) {
+        if (comm_ptr->rank == 0)
+            MPL_atomic_fetch_sub_uint64(MPIDI_POSIX_shm_limit_counter,
+                                        RELEASE_GATHER_FIELD(comm_ptr, reduce_shm_size));
         /* destroy and detach shared memory used for reduce buffers */
         mpi_errno = MPIDU_shm_free(RELEASE_GATHER_FIELD(comm_ptr, reduce_buf_addr));
         if (mpi_errno) {

src/mpid/ch4/shm/posix/release_gather/release_gather_types.h

@@ -26,7 +26,10 @@ typedef struct MPIDI_POSIX_release_gather_comm_t {
     int num_collective_calls;
 
     MPIR_Treealgo_tree_t bcast_tree, reduce_tree;
-    int flags_shm_size;
+    /* shm mem allocated to this comm */
+    uint64_t flags_shm_size;
+    uint64_t bcast_shm_size;
+    uint64_t reduce_shm_size;
     uint64_t gather_state, release_state;
 
     void *flags_addr, *bcast_buf_addr, *reduce_buf_addr;