diff --git a/maint/tuning/coll/ch4/generic.json b/maint/tuning/coll/ch4/generic.json index 3e8c11de150..aa94ba85d5b 100644 --- a/maint/tuning/coll/ch4/generic.json +++ b/maint/tuning/coll/ch4/generic.json @@ -150,7 +150,7 @@ { "comm_type=intra": { - "composition=MPIDI_Alltoall_intra_composition_alpha":{} + "composition=MPIDI_Alltoall_intra_composition_beta":{} } }, "collective=alltoallv": @@ -171,7 +171,7 @@ { "comm_type=intra": { - "composition=MPIDI_Allgather_intra_composition_alpha":{} + "composition=MPIDI_Allgather_intra_composition_beta":{} } }, "collective=allgatherv": diff --git a/src/mpid/ch4/src/ch4_coll.h b/src/mpid/ch4/src/ch4_coll.h index 3c96bc84672..f456466f7f1 100644 --- a/src/mpid/ch4/src/ch4_coll.h +++ b/src/mpid/ch4/src/ch4_coll.h @@ -10,7 +10,96 @@ #include "ch4_proc.h" #include "ch4_coll_impl.h" -MPL_STATIC_INLINE_PREFIX int MPID_Barrier(MPIR_Comm * comm, MPIR_Errflag_t * errflag) +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_BARRIER_COMPOSITION + category : COLLECTIVE + type : int + default : 0 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Select composition (inter_node + intra_node) for Barrier + 0 Auto selection + 1 NM + SHM + 2 NM only + + - name : MPIR_CVAR_BCAST_COMPOSITION + category : COLLECTIVE + type : int + default : 0 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Select composition (inter_node + intra_node) for Bcast + 0 Auto selection + 1 NM + SHM with explicit send-recv between rank 0 and root + 2 NM + SHM without the explicit send-recv + 3 NM only + + - name : MPIR_CVAR_ALLREDUCE_COMPOSITION + category : COLLECTIVE + type : int + default : 0 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Select composition (inter_node + intra_node) for Allreduce + 0 Auto selection + 1 NM + SHM with reduce + bcast + 2 NM only composition + 3 SHM only composition + + - name : MPIR_CVAR_ALLGATHER_COMPOSITION + category : COLLECTIVE + type : int + default : 0 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Select composition (inter_node + intra_node) for Allgather + 0 Auto selection + 1 Multi leaders based inter node + intra node composition + 2 NM only composition + + - name : MPIR_CVAR_ALLTOALL_COMPOSITION + category : COLLECTIVE + type : int + default : 0 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Select composition (inter_node + intra_node) for Alltoall + 0 Auto selection + 1 Multi leaders based inter node + intra node composition + 2 NM only composition + + - name : MPIR_CVAR_REDUCE_COMPOSITION + category : COLLECTIVE + type : int + default : 0 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Select composition (inter_node + intra_node) for Reduce + 0 Auto selection + 1 NM + SHM with explicit send-recv between rank 0 and root + 2 NM + SHM without the explicit send-recv + 3 NM only + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + +MPL_STATIC_INLINE_PREFIX int MPIDI_Barrier_allcomm_composition_json(MPIR_Comm * comm, + MPIR_Errflag_t * errflag) { int mpi_errno = MPI_SUCCESS; const MPIDI_Csel_container_s *cnt = NULL; @@ -20,8 +109,6 @@ MPL_STATIC_INLINE_PREFIX int MPID_Barrier(MPIR_Comm * comm, MPIR_Errflag_t * err .comm_ptr = comm, }; - MPIR_FUNC_ENTER; - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); if (cnt == NULL) { @@ -43,6 +130,48 @@ MPL_STATIC_INLINE_PREFIX int MPID_Barrier(MPIR_Comm * comm, MPIR_Errflag_t * err MPIR_ERR_CHECK(mpi_errno); + fn_exit: + return mpi_errno; + fn_fail: + goto fn_exit; +} + +MPL_STATIC_INLINE_PREFIX int MPID_Barrier(MPIR_Comm * comm, MPIR_Errflag_t * errflag) +{ + int mpi_errno = MPI_SUCCESS; + + MPIR_FUNC_ENTER; + + + switch (MPIR_CVAR_BARRIER_COMPOSITION) { + case 1: + MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, + (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) && + (comm->hierarchy_kind == + MPIR_COMM_HIERARCHY_KIND__PARENT), mpi_errno, + "Barrier composition alpha cannot be applied.\n"); + mpi_errno = MPIDI_Barrier_intra_composition_alpha(comm, errflag); + break; + case 2: + MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, + comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, + "Barrier composition beta cannot be applied.\n"); + mpi_errno = MPIDI_Barrier_intra_composition_beta(comm, errflag); + break; + default: + mpi_errno = MPIDI_Barrier_allcomm_composition_json(comm, errflag); + break; + } + + MPIR_ERR_CHECK(mpi_errno); + goto fn_exit; + + fallback: + if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) + mpi_errno = MPIR_Barrier_impl(comm, errflag); + else + mpi_errno = MPIDI_Barrier_intra_composition_beta(comm, errflag); + fn_exit: MPIR_FUNC_EXIT; return mpi_errno; @@ -50,8 +179,10 @@ MPL_STATIC_INLINE_PREFIX int MPID_Barrier(MPIR_Comm * comm, MPIR_Errflag_t * err goto fn_exit; } -MPL_STATIC_INLINE_PREFIX int MPID_Bcast(void *buffer, MPI_Aint count, MPI_Datatype datatype, - int root, MPIR_Comm * comm, MPIR_Errflag_t * errflag) +MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_allcomm_composition_json(void *buffer, MPI_Aint count, + MPI_Datatype datatype, int root, + MPIR_Comm * comm, + MPIR_Errflag_t * errflag) { int mpi_errno = MPI_SUCCESS; MPIR_Csel_coll_sig_s coll_sig = { @@ -65,8 +196,6 @@ MPL_STATIC_INLINE_PREFIX int MPID_Bcast(void *buffer, MPI_Aint count, MPI_Dataty const MPIDI_Csel_container_s *cnt = NULL; - MPIR_FUNC_ENTER; - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); if (cnt == NULL) { @@ -94,6 +223,61 @@ MPL_STATIC_INLINE_PREFIX int MPID_Bcast(void *buffer, MPI_Aint count, MPI_Dataty MPIR_ERR_CHECK(mpi_errno); + fn_exit: + return mpi_errno; + fn_fail: + goto fn_exit; +} + +MPL_STATIC_INLINE_PREFIX int MPID_Bcast(void *buffer, MPI_Aint count, MPI_Datatype datatype, + int root, MPIR_Comm * comm, MPIR_Errflag_t * errflag) +{ + int mpi_errno = MPI_SUCCESS; + + MPIR_FUNC_ENTER; + + switch (MPIR_CVAR_BCAST_COMPOSITION) { + case 1: + MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, + (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) && + (comm->hierarchy_kind == + MPIR_COMM_HIERARCHY_KIND__PARENT), mpi_errno, + "Bcast composition alpha cannot be applied.\n"); + mpi_errno = + MPIDI_Bcast_intra_composition_alpha(buffer, count, datatype, root, comm, errflag); + break; + case 2: + MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, + (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) && + (comm->hierarchy_kind == + MPIR_COMM_HIERARCHY_KIND__PARENT), mpi_errno, + "Bcast composition beta cannot be applied.\n"); + mpi_errno = + MPIDI_Bcast_intra_composition_beta(buffer, count, datatype, root, comm, errflag); + break; + case 3: + MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, + comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, + "Bcast composition gamma cannot be applied.\n"); + mpi_errno = + MPIDI_Bcast_intra_composition_gamma(buffer, count, datatype, root, comm, errflag); + break; + default: + mpi_errno = + MPIDI_Bcast_allcomm_composition_json(buffer, count, datatype, root, comm, errflag); + break; + } + + MPIR_ERR_CHECK(mpi_errno); + goto fn_exit; + + fallback: + if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) + mpi_errno = MPIR_Bcast_impl(buffer, count, datatype, root, comm, errflag); + else + mpi_errno = + MPIDI_Bcast_intra_composition_gamma(buffer, count, datatype, root, comm, errflag); + fn_exit: MPIR_FUNC_EXIT; return mpi_errno; @@ -101,9 +285,11 @@ MPL_STATIC_INLINE_PREFIX int MPID_Bcast(void *buffer, MPI_Aint count, MPI_Dataty goto fn_exit; } -MPL_STATIC_INLINE_PREFIX int MPID_Allreduce(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Errflag_t * errflag) +MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_allcomm_composition_json(const void *sendbuf, + void *recvbuf, MPI_Aint count, + MPI_Datatype datatype, + MPI_Op op, MPIR_Comm * comm, + MPIR_Errflag_t * errflag) { int mpi_errno = MPI_SUCCESS; const MPIDI_Csel_container_s *cnt = NULL; @@ -119,8 +305,6 @@ MPL_STATIC_INLINE_PREFIX int MPID_Allreduce(const void *sendbuf, void *recvbuf, .u.allreduce.op = op, }; - MPIR_FUNC_ENTER; - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); if (cnt == NULL) { @@ -151,6 +335,72 @@ MPL_STATIC_INLINE_PREFIX int MPID_Allreduce(const void *sendbuf, void *recvbuf, MPIR_ERR_CHECK(mpi_errno); + fn_exit: + return mpi_errno; + fn_fail: + goto fn_exit; +} + +MPL_STATIC_INLINE_PREFIX int MPID_Allreduce(const void *sendbuf, void *recvbuf, MPI_Aint count, + MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, + MPIR_Errflag_t * errflag) +{ + int mpi_errno = MPI_SUCCESS; + int is_commutative = -1; + + MPIR_FUNC_ENTER; + + is_commutative = MPIR_Op_is_commutative(op); + + switch (MPIR_CVAR_ALLREDUCE_COMPOSITION) { + case 1: + MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, + (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) && + (comm->hierarchy_kind == + MPIR_COMM_HIERARCHY_KIND__PARENT) && + is_commutative, mpi_errno, + "Allreduce composition alpha cannot be applied.\n"); + mpi_errno = + MPIDI_Allreduce_intra_composition_alpha(sendbuf, recvbuf, count, datatype, op, comm, + errflag); + break; + case 2: + MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, + comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, + "Allreduce composition beta cannot be applied.\n"); + mpi_errno = + MPIDI_Allreduce_intra_composition_beta(sendbuf, recvbuf, count, datatype, op, comm, + errflag); + break; + case 3: + MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, + (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) && + (comm->node_comm != NULL) && + (MPIR_Comm_size(comm) == + MPIR_Comm_size(comm->node_comm)), mpi_errno, + "Allreduce composition gamma cannot be applied.\n"); + mpi_errno = + MPIDI_Allreduce_intra_composition_gamma(sendbuf, recvbuf, count, datatype, op, comm, + errflag); + break; + default: + mpi_errno = + MPIDI_Allreduce_allcomm_composition_json(sendbuf, recvbuf, count, datatype, op, + comm, errflag); + break; + } + + MPIR_ERR_CHECK(mpi_errno); + goto fn_exit; + + fallback: + if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) + mpi_errno = MPIR_Allreduce_impl(sendbuf, recvbuf, count, datatype, op, comm, errflag); + else + mpi_errno = + MPIDI_Allreduce_intra_composition_beta(sendbuf, recvbuf, count, datatype, op, comm, + errflag); + fn_exit: MPIR_FUNC_EXIT; return mpi_errno; @@ -158,10 +408,15 @@ MPL_STATIC_INLINE_PREFIX int MPID_Allreduce(const void *sendbuf, void *recvbuf, goto fn_exit; } -MPL_STATIC_INLINE_PREFIX int MPID_Allgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Errflag_t * errflag) + +MPL_STATIC_INLINE_PREFIX int MPIDI_Allgather_allcomm_composition_json(const void *sendbuf, + MPI_Aint sendcount, + MPI_Datatype sendtype, + void *recvbuf, + MPI_Aint recvcount, + MPI_Datatype recvtype, + MPIR_Comm * comm, + MPIR_Errflag_t * errflag) { int mpi_errno = MPI_SUCCESS; const MPIDI_Csel_container_s *cnt = NULL; @@ -178,8 +433,6 @@ MPL_STATIC_INLINE_PREFIX int MPID_Allgather(const void *sendbuf, MPI_Aint sendco .u.allgather.recvtype = recvtype, }; - MPIR_FUNC_ENTER; - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); if (cnt == NULL) { @@ -191,11 +444,10 @@ MPL_STATIC_INLINE_PREFIX int MPID_Allgather(const void *sendbuf, MPI_Aint sendco } switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgather_intra_composition_alpha: + case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgather_intra_composition_beta: mpi_errno = - MPIDI_Allgather_intra_composition_alpha(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, - comm, errflag); + MPIDI_Allgather_intra_composition_beta(sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, comm, errflag); break; default: MPIR_Assert(0); @@ -204,7 +456,51 @@ MPL_STATIC_INLINE_PREFIX int MPID_Allgather(const void *sendbuf, MPI_Aint sendco MPIR_ERR_CHECK(mpi_errno); fn_exit: - MPIR_FUNC_EXIT; + return mpi_errno; + fn_fail: + goto fn_exit; +} + +MPL_STATIC_INLINE_PREFIX int MPID_Allgather(const void *sendbuf, MPI_Aint sendcount, + MPI_Datatype sendtype, void *recvbuf, + MPI_Aint recvcount, MPI_Datatype recvtype, + MPIR_Comm * comm, MPIR_Errflag_t * errflag) +{ + int mpi_errno = MPI_SUCCESS; + + MPIR_FUNC_ENTER; + + switch (MPIR_CVAR_ALLGATHER_COMPOSITION) { + case 2: + MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, + comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, + "Allgather composition beta cannot be applied.\n"); + mpi_errno = + MPIDI_Allgather_intra_composition_beta(sendbuf, sendcount, sendtype, recvbuf, + recvcount, recvtype, comm, errflag); + break; + default: + mpi_errno = MPIDI_Allgather_allcomm_composition_json(sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, comm, + errflag); + break; + } + + MPIR_ERR_CHECK(mpi_errno); + goto fn_exit; + + fallback: + if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) + mpi_errno = + MPIR_Allgather_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, + errflag); + else + mpi_errno = + MPIDI_Allgather_intra_composition_beta(sendbuf, sendcount, sendtype, recvbuf, recvcount, + recvtype, comm, errflag); + + fn_exit: + MPIR_FUNC_ENTER; return mpi_errno; fn_fail: goto fn_exit; @@ -474,10 +770,14 @@ MPL_STATIC_INLINE_PREFIX int MPID_Gatherv(const void *sendbuf, MPI_Aint sendcoun goto fn_exit; } -MPL_STATIC_INLINE_PREFIX int MPID_Alltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Errflag_t * errflag) +MPL_STATIC_INLINE_PREFIX int MPIDI_Alltoall_allcomm_composition_json(const void *sendbuf, + MPI_Aint sendcount, + MPI_Datatype sendtype, + void *recvbuf, + MPI_Aint recvcount, + MPI_Datatype recvtype, + MPIR_Comm * comm, + MPIR_Errflag_t * errflag) { int mpi_errno = MPI_SUCCESS; const MPIDI_Csel_container_s *cnt = NULL; @@ -494,8 +794,6 @@ MPL_STATIC_INLINE_PREFIX int MPID_Alltoall(const void *sendbuf, MPI_Aint sendcou .u.alltoall.recvtype = recvtype, }; - MPIR_FUNC_ENTER; - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); if (cnt == NULL) { @@ -506,10 +804,10 @@ MPL_STATIC_INLINE_PREFIX int MPID_Alltoall(const void *sendbuf, MPI_Aint sendcou } switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoall_intra_composition_alpha: + case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoall_intra_composition_beta: mpi_errno = - MPIDI_Alltoall_intra_composition_alpha(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, errflag); + MPIDI_Alltoall_intra_composition_beta(sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, comm, errflag); break; default: MPIR_Assert(0); @@ -518,7 +816,50 @@ MPL_STATIC_INLINE_PREFIX int MPID_Alltoall(const void *sendbuf, MPI_Aint sendcou MPIR_ERR_CHECK(mpi_errno); fn_exit: - MPIR_FUNC_EXIT; + return mpi_errno; + fn_fail: + goto fn_exit; +} + +MPL_STATIC_INLINE_PREFIX int MPID_Alltoall(const void *sendbuf, MPI_Aint sendcount, + MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, + MPI_Datatype recvtype, MPIR_Comm * comm, + MPIR_Errflag_t * errflag) +{ + int mpi_errno = MPI_SUCCESS; + + MPIR_FUNC_ENTER; + + switch (MPIR_CVAR_ALLTOALL_COMPOSITION) { + case 2: + MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, + comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, + "Alltoall composition beta cannot be applied.\n"); + mpi_errno = + MPIDI_Alltoall_intra_composition_beta(sendbuf, sendcount, sendtype, recvbuf, + recvcount, recvtype, comm, errflag); + break; + default: + mpi_errno = MPIDI_Alltoall_allcomm_composition_json(sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, comm, + errflag); + break; + } + + MPIR_ERR_CHECK(mpi_errno); + goto fn_exit; + + fallback: + if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) + mpi_errno = + MPIR_Alltoall_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, + errflag); + else + mpi_errno = MPIDI_Alltoall_intra_composition_beta(sendbuf, sendcount, sendtype, recvbuf, + recvcount, recvtype, comm, errflag); + + fn_exit: + MPIR_FUNC_ENTER; return mpi_errno; fn_fail: goto fn_exit; @@ -635,9 +976,11 @@ MPL_STATIC_INLINE_PREFIX int MPID_Alltoallw(const void *sendbuf, const MPI_Aint goto fn_exit; } -MPL_STATIC_INLINE_PREFIX int MPID_Reduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - int root, MPIR_Comm * comm, MPIR_Errflag_t * errflag) +MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_allcomm_composition_json(const void *sendbuf, + void *recvbuf, MPI_Aint count, + MPI_Datatype datatype, MPI_Op op, + int root, MPIR_Comm * comm, + MPIR_Errflag_t * errflag) { int mpi_errno = MPI_SUCCESS; const MPIDI_Csel_container_s *cnt = NULL; @@ -654,8 +997,6 @@ MPL_STATIC_INLINE_PREFIX int MPID_Reduce(const void *sendbuf, void *recvbuf, .u.reduce.root = root, }; - MPIR_FUNC_ENTER; - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); if (cnt == NULL) { @@ -686,6 +1027,67 @@ MPL_STATIC_INLINE_PREFIX int MPID_Reduce(const void *sendbuf, void *recvbuf, MPIR_ERR_CHECK(mpi_errno); + fn_exit: + return mpi_errno; + fn_fail: + goto fn_exit; +} + +MPL_STATIC_INLINE_PREFIX int MPID_Reduce(const void *sendbuf, void *recvbuf, + MPI_Aint count, MPI_Datatype datatype, MPI_Op op, + int root, MPIR_Comm * comm, MPIR_Errflag_t * errflag) +{ + int mpi_errno = MPI_SUCCESS; + + MPIR_FUNC_ENTER; + + switch (MPIR_CVAR_REDUCE_COMPOSITION) { + case 1: + MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, comm->comm_kind == MPIR_COMM_KIND__INTRACOMM + && comm->hierarchy_kind == + MPIR_COMM_HIERARCHY_KIND__PARENT && + MPIR_Op_is_commutative(op), mpi_errno, + "Reduce composition alpha cannot be applied.\n"); + mpi_errno = + MPIDI_Reduce_intra_composition_alpha(sendbuf, recvbuf, count, datatype, op, root, + comm, errflag); + break; + case 2: + MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, comm->comm_kind == MPIR_COMM_KIND__INTRACOMM + && comm->hierarchy_kind == + MPIR_COMM_HIERARCHY_KIND__PARENT && + MPIR_Op_is_commutative(op), mpi_errno, + "Reduce composition beta cannot be applied.\n"); + mpi_errno = + MPIDI_Reduce_intra_composition_beta(sendbuf, recvbuf, count, datatype, op, root, + comm, errflag); + break; + case 3: + MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, + comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, + "Reduce composition gamma cannot be applied.\n"); + mpi_errno = + MPIDI_Reduce_intra_composition_gamma(sendbuf, recvbuf, count, datatype, op, root, + comm, errflag); + break; + default: + mpi_errno = + MPIDI_Reduce_allcomm_composition_json(sendbuf, recvbuf, count, datatype, op, + root, comm, errflag); + break; + } + + MPIR_ERR_CHECK(mpi_errno); + goto fn_exit; + + fallback: + if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) + mpi_errno = MPIR_Reduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm, errflag); + else + mpi_errno = + MPIDI_Reduce_intra_composition_gamma(sendbuf, recvbuf, count, datatype, op, root, comm, + errflag); + fn_exit: MPIR_FUNC_EXIT; return mpi_errno; diff --git a/src/mpid/ch4/src/ch4_coll_impl.h b/src/mpid/ch4/src/ch4_coll_impl.h index a145d3bfbb2..58132b232d6 100644 --- a/src/mpid/ch4/src/ch4_coll_impl.h +++ b/src/mpid/ch4/src/ch4_coll_impl.h @@ -569,14 +569,14 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_intra_composition_gamma(const void *se goto fn_exit; } -MPL_STATIC_INLINE_PREFIX int MPIDI_Alltoall_intra_composition_alpha(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag) +MPL_STATIC_INLINE_PREFIX int MPIDI_Alltoall_intra_composition_beta(const void *sendbuf, + MPI_Aint sendcount, + MPI_Datatype sendtype, + void *recvbuf, + MPI_Aint recvcount, + MPI_Datatype recvtype, + MPIR_Comm * comm_ptr, + MPIR_Errflag_t * errflag) { int mpi_errno = MPI_SUCCESS; @@ -642,14 +642,14 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Alltoallw_intra_composition_alpha(const void goto fn_exit; } -MPL_STATIC_INLINE_PREFIX int MPIDI_Allgather_intra_composition_alpha(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag) +MPL_STATIC_INLINE_PREFIX int MPIDI_Allgather_intra_composition_beta(const void *sendbuf, + MPI_Aint sendcount, + MPI_Datatype sendtype, + void *recvbuf, + MPI_Aint recvcount, + MPI_Datatype recvtype, + MPIR_Comm * comm_ptr, + MPIR_Errflag_t * errflag) { int mpi_errno = MPI_SUCCESS; diff --git a/src/mpid/ch4/src/ch4_csel_container.h b/src/mpid/ch4/src/ch4_csel_container.h index 0d6dd7938f2..cd5144d3964 100644 --- a/src/mpid/ch4/src/ch4_csel_container.h +++ b/src/mpid/ch4/src/ch4_csel_container.h @@ -20,9 +20,11 @@ typedef struct { MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allreduce_intra_composition_beta, MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allreduce_intra_composition_gamma, MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoall_intra_composition_alpha, + MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoall_intra_composition_beta, MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoallv_intra_composition_alpha, MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoallw_intra_composition_alpha, MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgather_intra_composition_alpha, + MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgather_intra_composition_beta, MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgatherv_intra_composition_alpha, MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Gather_intra_composition_alpha, MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Gatherv_intra_composition_alpha, diff --git a/src/mpid/ch4/src/ch4_init.c b/src/mpid/ch4/src/ch4_init.c index cfafe7565bf..f3403b11338 100644 --- a/src/mpid/ch4/src/ch4_init.c +++ b/src/mpid/ch4/src/ch4_init.c @@ -142,6 +142,8 @@ static void *create_container(struct json_object *obj) else if (!strcmp(ckey, "composition=MPIDI_Alltoall_intra_composition_alpha")) cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoall_intra_composition_alpha; + else if (!strcmp(ckey, "composition=MPIDI_Alltoall_intra_composition_beta")) + cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoall_intra_composition_beta; else if (!strcmp(ckey, "composition=MPIDI_Alltoallv_intra_composition_alpha")) cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoallv_intra_composition_alpha; @@ -151,6 +153,9 @@ static void *create_container(struct json_object *obj) else if (!strcmp(ckey, "composition=MPIDI_Allgather_intra_composition_alpha")) cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgather_intra_composition_alpha; + else if (!strcmp(ckey, "composition=MPIDI_Allgather_intra_composition_beta")) + cnt->id = + MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgather_intra_composition_beta; else if (!strcmp(ckey, "composition=MPIDI_Allgatherv_intra_composition_alpha")) cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgatherv_intra_composition_alpha;