Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tpetra: Added a test for multiple asynchronous transfers with cyclic maps #10310

Closed
wants to merge 2 commits into from
Closed

Tpetra: Added a test for multiple asynchronous transfers with cyclic maps #10310

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dcb5c2d
Added a test for multiple asynchronous transfers
tasmith4 Mar 8, 2022
133b713
Tpetra: Add an async transfer test for a group of MultiVectors with c…
tasmith4 Mar 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
229 changes: 229 additions & 0 deletions packages/tpetra/core/test/ImportExport2/AsyncTransfer_UnitTests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,186 @@ namespace {
};


template <typename Scalar, typename LO, typename GO>
class MultiVectorGroupTransferFixture {
private:
using map_type = Map<LO, GO>;
using mv_type = MultiVector<Scalar, LO, GO>;

public:
MultiVectorGroupTransferFixture(FancyOStream& o, bool& s)
: out(o),
success(s),
comm(getDefaultComm()),
numProcs(comm->getSize()),
myRank(comm->getRank()),
numMVs(4)
{ }

~MultiVectorGroupTransferFixture() { }

void setup(int collectRank) {
setupMaps(collectRank);
setupMultiVectors();
}

template <typename TransferMethod>
void performTransfer(const TransferMethod& transfer) {
transfer(sourceMVs, targetMVs);
}

template <typename ReferenceSolution>
void checkResults(const ReferenceSolution& referenceSolution) {
for (int i=0; i<numMVs; i++) {
RCP<const mv_type> referenceMV = referenceSolution.generateWithClassicalCodePath(sourceMVs[i], targetMap);
compareMultiVectors(targetMVs[i], referenceMV);
}
}

private:
void setupMaps(int collectRank) {
const GO indexBase = 0;
const global_size_t INVALID = OrdinalTraits<global_size_t>::invalid();

const size_t sourceNumLocalElements = 3;
const size_t totalElements = numProcs*sourceNumLocalElements;
const size_t targetNumLocalElements = (myRank == collectRank) ? totalElements : 0;

sourceMap = rcp(new map_type(INVALID, sourceNumLocalElements, indexBase, comm));
targetMap = rcp(new map_type(INVALID, targetNumLocalElements, indexBase, comm));
}

void setupMultiVectors() {
for (int i=0; i<numMVs; i++) {
sourceMVs.push_back(rcp(new mv_type(sourceMap, 1)));
sourceMVs[i]->randomize();

targetMVs.push_back(rcp(new mv_type(targetMap, 1)));
targetMVs[i]->putScalar(ScalarTraits<Scalar>::zero());
}
}

void compareMultiVectors(RCP<const mv_type> resultMV, RCP<const mv_type> referenceMV) {
auto data = resultMV->getLocalViewHost(Tpetra::Access::ReadOnly);
auto referenceData = referenceMV->getLocalViewHost(Tpetra::Access::ReadOnly);

TEST_EQUALITY(data.size(), referenceData.size());
for (LO localRow = 0; localRow < as<LO>(data.size()); localRow++) {
TEST_EQUALITY(data(localRow, 0), referenceData(localRow, 0));
}
}

FancyOStream& out;
bool& success;

RCP<const Comm<int>> comm;
const int numProcs;
const int myRank;

const int numMVs;

RCP<const map_type> sourceMap;
RCP<const map_type> targetMap;

std::vector<RCP<mv_type>> sourceMVs;
std::vector<RCP<mv_type>> targetMVs;
};


template <typename Scalar, typename LO, typename GO>
class MultiVectorCyclicGroupTransferFixture {
private:
using map_type = Map<LO, GO>;
using mv_type = MultiVector<Scalar, LO, GO>;

public:
MultiVectorCyclicGroupTransferFixture(FancyOStream& o, bool& s)
: out(o),
success(s),
comm(getDefaultComm()),
numProcs(comm->getSize()),
myRank(comm->getRank()),
numMVs(4)
{ }

~MultiVectorCyclicGroupTransferFixture() { }

void setup(int collectRank) {
setupMaps(collectRank);
setupMultiVectors();
}

template <typename TransferMethod>
void performTransfer(const TransferMethod& transfer) {
transfer(sourceMVs, targetMVs);
for (int i=0; i<numMVs; i++) {
TEST_EQUALITY(targetMVs[i]->importsAreAliased(), false);
}
}

template <typename ReferenceSolution>
void checkResults(const ReferenceSolution& referenceSolution) {
for (int i=0; i<numMVs; i++) {
RCP<const mv_type> referenceMV = referenceSolution.generateWithClassicalCodePath(sourceMVs[i], targetMap);
compareMultiVectors(targetMVs[i], referenceMV);
}
}

private:
void setupMaps(int collectRank) {
const GO indexBase = 0;
const global_size_t INVALID = OrdinalTraits<global_size_t>::invalid();

const size_t sourceNumLocalElements = 3;
const size_t totalElements = numProcs*sourceNumLocalElements;
const size_t targetNumLocalElements = (myRank == collectRank) ? totalElements : 0;

Teuchos::Array<GO> sourceEntries(sourceNumLocalElements);
for (size_t i=0; i<sourceNumLocalElements; i++) {
sourceEntries[i] = i*numProcs + myRank;
}

sourceMap = rcp(new map_type(INVALID, sourceEntries, indexBase, comm));
targetMap = rcp(new map_type(INVALID, targetNumLocalElements, indexBase, comm));
}

void setupMultiVectors() {
for (int i=0; i<numMVs; i++) {
sourceMVs.push_back(rcp(new mv_type(sourceMap, 1)));
sourceMVs[i]->randomize();

targetMVs.push_back(rcp(new mv_type(targetMap, 1)));
targetMVs[i]->putScalar(ScalarTraits<Scalar>::zero());
}
}

void compareMultiVectors(RCP<const mv_type> resultMV, RCP<const mv_type> referenceMV) {
auto data = resultMV->getLocalViewHost(Tpetra::Access::ReadOnly);
auto referenceData = referenceMV->getLocalViewHost(Tpetra::Access::ReadOnly);

TEST_EQUALITY(data.size(), referenceData.size());
for (LO localRow = 0; localRow < as<LO>(data.size()); localRow++) {
TEST_EQUALITY(data(localRow, 0), referenceData(localRow, 0));
}
}

FancyOStream& out;
bool& success;

RCP<const Comm<int>> comm;
const int numProcs;
const int myRank;

const int numMVs;

RCP<const map_type> sourceMap;
RCP<const map_type> targetMap;

std::vector<RCP<mv_type>> sourceMVs;
std::vector<RCP<mv_type>> targetMVs;
};


template <typename Scalar, typename LO, typename GO>
class DiagonalCrsMatrixTransferFixture {
private:
Expand Down Expand Up @@ -911,6 +1091,53 @@ namespace {
}


template <typename Packet, typename LO, typename GO>
class ForwardImportGroup {
private:
using DistObjectRCP = RCP<MultiVector<Packet, LO, GO>>;

public:
void operator()(std::vector<DistObjectRCP> sources, std::vector<DistObjectRCP> targets) const {
Import<LO, GO> importer(sources[0]->getMap(), targets[0]->getMap());

for (unsigned i=0; i<sources.size(); i++) {
targets[i]->beginImport(*sources[i], importer, INSERT);
}

unsigned completedImports = 0;
std::vector<bool> completedImport(false, sources.size());
while (completedImports < completedImport.size()) {
for (unsigned i=0; i<sources.size(); i++) {
if (completedImport[i]) continue;
if (targets[i]->transferArrived()) {
targets[i]->endImport(*sources[i], importer, INSERT);
completedImport[i] = true;
completedImports++;
}
}
}
}
};

TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL( AsyncForwardImport, MultiVectorGroup_rank0, Scalar, LO, GO )
{
MultiVectorGroupTransferFixture<Scalar, LO, GO> fixture(out, success);

fixture.setup(0);
fixture.performTransfer(ForwardImportGroup<Scalar, LO, GO>());
fixture.checkResults(ReferenceImportMultiVector<Scalar, LO, GO>());
}


TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL( AsyncForwardImport, MultiVectorCyclicGroup_rank0, Scalar, LO, GO )
{
MultiVectorCyclicGroupTransferFixture<Scalar, LO, GO> fixture(out, success);

fixture.setup(0);
fixture.performTransfer(ForwardImportGroup<Scalar, LO, GO>());
fixture.checkResults(ReferenceImportMultiVector<Scalar, LO, GO>());
}

//
// INSTANTIATIONS
//
Expand Down Expand Up @@ -940,6 +1167,8 @@ namespace {
TEUCHOS_UNIT_TEST_TEMPLATE_3_INSTANT( TransferArrived, CrsMatrix_forwardExportTrue, SC, LO, GO ) \
TEUCHOS_UNIT_TEST_TEMPLATE_3_INSTANT( TransferArrived, MultiVector_forwardImportFalse, SC, LO, GO ) \
TEUCHOS_UNIT_TEST_TEMPLATE_3_INSTANT( TransferArrived, MultiVector_forwardExportFalse, SC, LO, GO ) \
TEUCHOS_UNIT_TEST_TEMPLATE_3_INSTANT( AsyncForwardImport, MultiVectorGroup_rank0, SC, LO, GO ) \
TEUCHOS_UNIT_TEST_TEMPLATE_3_INSTANT( AsyncForwardImport, MultiVectorCyclicGroup_rank0, SC, LO, GO ) \

TPETRA_ETI_MANGLING_TYPEDEFS()

Expand Down