Skip to content

Commit

Permalink
Merge pull request open-mpi#3711 from gpaulsen/rel_v2.0.x_pr3500_pr3635
Browse files Browse the repository at this point in the history 
Rel v2.0.x pr3500 pr3635
  • Loading branch information
@hppritcha
hppritcha authored Jul 11, 2017
2 parents 18f1659 + 7dd2bc0 commit 25919a1
Showing 1 changed file with 61 additions and 60 deletions.
121 changes: 61 additions & 60 deletions ompi/mpi/c/sendrecv_replace.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2010 The University of Tennessee and The University
* Copyright (c) 2004-2017 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
Expand All @@ -12,6 +12,7 @@
* Copyright (c) 2010-2012 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2017 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -48,10 +49,10 @@ int MPI_Sendrecv_replace(void * buf, int count, MPI_Datatype datatype,
int rc = MPI_SUCCESS;

MEMCHECKER(
memchecker_datatype(datatype);
memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
memchecker_comm(comm);
);
memchecker_datatype(datatype);
memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
memchecker_comm(comm);
);

if ( MPI_PARAM_CHECK ) {
rc = MPI_SUCCESS;
Expand All @@ -74,66 +75,66 @@ int MPI_Sendrecv_replace(void * buf, int count, MPI_Datatype datatype,

/* simple case */
if ( source == MPI_PROC_NULL || dest == MPI_PROC_NULL || count == 0 ) {
rc = PMPI_Sendrecv(buf,count,datatype,dest,sendtag,buf,count,datatype,source,recvtag,comm,status);
rc = PMPI_Sendrecv(buf, count, datatype, dest, sendtag, buf, count, datatype, source, recvtag, comm, status);

return rc;
} else {

opal_convertor_t convertor;
struct iovec iov;
unsigned char recv_data[2048];
size_t packed_size, max_data;
uint32_t iov_count;
ompi_status_public_t recv_status;
ompi_proc_t* proc = ompi_comm_peer_lookup(comm,source);
if(proc == NULL) {
rc = MPI_ERR_RANK;
OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
}

/* initialize convertor to unpack recv buffer */
OBJ_CONSTRUCT(&convertor, opal_convertor_t);
opal_convertor_copy_and_prepare_for_recv( proc->super.proc_convertor, &(datatype->super),
count, buf, 0, &convertor );

/* setup a buffer for recv */
opal_convertor_get_packed_size( &convertor, &packed_size );
if( packed_size > sizeof(recv_data) ) {
rc = PMPI_Alloc_mem(packed_size, MPI_INFO_NULL, &iov.iov_base);
if(OMPI_SUCCESS != rc) {
OMPI_ERRHANDLER_RETURN(OMPI_ERR_OUT_OF_RESOURCE, comm, MPI_ERR_BUFFER, FUNC_NAME);
}
} else {
iov.iov_base = (caddr_t)recv_data;
}

/* recv into temporary buffer */
rc = PMPI_Sendrecv( buf, count, datatype, dest, sendtag, iov.iov_base, packed_size,
MPI_BYTE, source, recvtag, comm, &recv_status );
if (rc != MPI_SUCCESS) {
if(packed_size > sizeof(recv_data))
PMPI_Free_mem(iov.iov_base);
OBJ_DESTRUCT(&convertor);
OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
}

/* unpack into users buffer */
iov.iov_len = recv_status._ucount;
iov_count = 1;
max_data = recv_status._ucount;
opal_convertor_unpack(&convertor, &iov, &iov_count, &max_data );
}

/* return status to user */
if(status != MPI_STATUS_IGNORE) {
*status = recv_status;
}
/**
* If we look for an optimal solution, then we should receive the data into a temporary buffer
* and once the send completes we would unpack back into the original buffer. However, if the
* sender is unknown, this approach can only be implementing by receiving with the recv datatype
* (potentially non-contiguous) and thus the allocated memory will be larger than the size of the
* datatype. A simpler, but potentially less efficient approach is to work on the data we have
* control of, aka the sent data, and pack it into a contiguous buffer before posting the receive.
* Once the send completes, we free it.
*/
opal_convertor_t convertor;
unsigned char packed_data[2048];
struct iovec iov = { .iov_base = packed_data, .iov_len = sizeof(packed_data) };
size_t packed_size, max_data;
uint32_t iov_count;
ompi_status_public_t recv_status;
ompi_proc_t* proc = ompi_comm_peer_lookup(comm, dest);
if(proc == NULL) {
rc = MPI_ERR_RANK;
OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
}

/* release resources */
if(packed_size > sizeof(recv_data)) {
PMPI_Free_mem(iov.iov_base);
/* initialize convertor to unpack recv buffer */
OBJ_CONSTRUCT(&convertor, opal_convertor_t);
opal_convertor_copy_and_prepare_for_send( proc->super.proc_convertor, &(datatype->super),
count, buf, 0, &convertor );

/* setup a buffer for recv */
opal_convertor_get_packed_size( &convertor, &packed_size );
if( packed_size > sizeof(packed_data) ) {
rc = PMPI_Alloc_mem(packed_size, MPI_INFO_NULL, &iov.iov_base);
if(OMPI_SUCCESS != rc) {
rc = OMPI_ERR_OUT_OF_RESOURCE;
goto cleanup_and_return;
}
OBJ_DESTRUCT(&convertor);
iov.iov_len = packed_size;
}
max_data = packed_size;
iov_count = 1;
rc = opal_convertor_pack(&convertor, &iov, &iov_count, &max_data);

/* recv into temporary buffer */
rc = PMPI_Sendrecv( iov.iov_base, packed_size, MPI_PACKED, dest, sendtag, buf, count,
datatype, source, recvtag, comm, &recv_status );

cleanup_and_return:
/* return status to user */
if(status != MPI_STATUS_IGNORE) {
*status = recv_status;
}

return MPI_SUCCESS;
/* release resources */
if(packed_size > sizeof(packed_data)) {
PMPI_Free_mem(iov.iov_base);
}
OBJ_DESTRUCT(&convertor);

OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
}

0 comments on commit 25919a1

Please sign in to comment.