Skip to content

Commit

Permalink
have parallel mesh test passing
Browse files Browse the repository at this point in the history
  • Loading branch information
Matthew Bettencourt committed Jul 30, 2021
1 parent f020c7b commit 5590ce5
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 23 deletions.
7 changes: 4 additions & 3 deletions packages/panzer/adapters-stk/src/Panzer_STK_LocalMeshUtilities.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,13 +119,14 @@ buildGhostedVertices(const Tpetra::Import<int,panzer::GlobalOrdinal,panzer::Tpet
// copy multivector into ghstd vertex structure
Kokkos::DynRankView<double,PHX::Device> ghstd_vertices("ghstd_vertices",ghstd_cell_cnt,vertices_per_cell,space_dim);
{
auto ghstd_vertices_view = ghstd_vertices_mv->getLocalViewHost(Tpetra::Access::ReadOnly);
for(size_t i=0;i<ghstd_cell_cnt;i++) {
auto ghstd_vertices_view = ghstd_vertices_mv->getLocalViewDevice(Tpetra::Access::ReadOnly);
Kokkos::parallel_for(ghstd_cell_cnt, KOKKOS_LAMBDA (size_t i) {
int l = 0;
for(int j=0;j<vertices_per_cell;j++)
for(int k=0;k<space_dim;k++,l++)
ghstd_vertices(i,j,k) = ghstd_vertices_view(i,l);
}
} );
Kokkos::fence();
}

return ghstd_vertices;
Expand Down
48 changes: 30 additions & 18 deletions packages/panzer/adapters-stk/test/local_mesh/parallel_local_mesh_utilities.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,47 +104,53 @@ TEUCHOS_UNIT_TEST(parallelLocalMeshUtilities, 1D_mesh)

{
const auto & block = mesh_info->element_blocks.at("eblock-0");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Element Block eblock-0" << std::endl;

// Block should have some basic stuff working
TEST_EQUALITY(block.num_owned_cells, 2);
TEST_EQUALITY(block.num_ghstd_cells, 1);
TEST_EQUALITY(block.num_virtual_cells, 1);
TEST_EQUALITY((int) block.global_cells(0), 0);
TEST_EQUALITY((int) block.global_cells(1), 1);
TEST_EQUALITY((int) block.global_cells(2), 2);
TEST_EQUALITY((int) block.global_cells(3), 6);
TEST_EQUALITY((int) global_cells_h(0), 0);
TEST_EQUALITY((int) global_cells_h(1), 1);
TEST_EQUALITY((int) global_cells_h(2), 2);
TEST_EQUALITY((int) global_cells_h(3), 6);
TEST_ASSERT(block.has_connectivity);
}

{
const auto & block = mesh_info->element_blocks.at("eblock-1");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Element Block eblock-1" << std::endl;

// Block should be empty
TEST_EQUALITY(block.num_owned_cells, 2);
TEST_EQUALITY(block.num_ghstd_cells, 2);
TEST_EQUALITY(block.num_virtual_cells, 0);
TEST_EQUALITY((int) block.global_cells(0), 3);
TEST_EQUALITY((int) block.global_cells(1), 4);
TEST_EQUALITY((int) block.global_cells(2), 5);
TEST_EQUALITY((int) block.global_cells(3), 2);
TEST_EQUALITY((int) global_cells_h(0), 3);
TEST_EQUALITY((int) global_cells_h(1), 4);
TEST_EQUALITY((int) global_cells_h(2), 5);
TEST_EQUALITY((int) global_cells_h(3), 2);
TEST_ASSERT(block.has_connectivity);
}

{
const auto & block = mesh_info->sidesets.at("eblock-0").at("left");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Sideset eblock-0 left" << std::endl;

// Block should have some basic stuff working
TEST_EQUALITY(block.num_owned_cells, 1);
TEST_EQUALITY(block.num_ghstd_cells, 0);
TEST_EQUALITY(block.num_virtual_cells, 1);
TEST_EQUALITY((int) block.global_cells(0), 0);
TEST_EQUALITY((int) block.global_cells(1), 6);
TEST_EQUALITY((int) global_cells_h(0), 0);
TEST_EQUALITY((int) global_cells_h(1), 6);
TEST_ASSERT(block.has_connectivity);
}

Expand All @@ -153,47 +159,53 @@ TEUCHOS_UNIT_TEST(parallelLocalMeshUtilities, 1D_mesh)
} else {
{
const auto & block = mesh_info->element_blocks.at("eblock-0");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Element Block eblock-0" << std::endl;

// Block should have some basic stuff working
TEST_EQUALITY(block.num_owned_cells, 1);
TEST_EQUALITY(block.num_ghstd_cells, 2);
TEST_EQUALITY(block.num_virtual_cells, 0);
TEST_EQUALITY((int) block.global_cells(0), 2);
TEST_EQUALITY((int) block.global_cells(1), 3);
TEST_EQUALITY((int) block.global_cells(2), 1);
TEST_EQUALITY((int) global_cells_h(0), 2);
TEST_EQUALITY((int) global_cells_h(1), 3);
TEST_EQUALITY((int) global_cells_h(2), 1);
TEST_ASSERT(block.has_connectivity);
}

{
const auto & block = mesh_info->element_blocks.at("eblock-1");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Element Block eblock-1" << std::endl;

// Block should be empty
TEST_EQUALITY(block.num_owned_cells, 1);
TEST_EQUALITY(block.num_ghstd_cells, 1);
TEST_EQUALITY(block.num_virtual_cells, 1);
TEST_EQUALITY((int) block.global_cells(0), 5);
TEST_EQUALITY((int) block.global_cells(1), 4);
TEST_EQUALITY((int) block.global_cells(2), 7);
TEST_EQUALITY((int) global_cells_h(0), 5);
TEST_EQUALITY((int) global_cells_h(1), 4);
TEST_EQUALITY((int) global_cells_h(2), 7);
TEST_ASSERT(block.has_connectivity);
}

// TEUCHOS_ASSERT(mesh_info->sidesets.find("eblock-0") == mesh_info->sidesets.end());

{
const auto & block = mesh_info->sidesets.at("eblock-1").at("right");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Sideset eblock-1 right" << std::endl;

// Block should have some basic stuff working
TEST_EQUALITY(block.num_owned_cells, 1);
TEST_EQUALITY(block.num_ghstd_cells, 0);
TEST_EQUALITY(block.num_virtual_cells, 1);
TEST_EQUALITY((int) block.global_cells(0), 5);
TEST_EQUALITY((int) block.global_cells(1), 7);
TEST_EQUALITY((int) global_cells_h(0), 5);
TEST_EQUALITY((int) global_cells_h(1), 7);
TEST_ASSERT(block.has_connectivity);
}
}
Expand Down
5 changes: 3 additions & 2 deletions packages/panzer/disc-fe/src/Panzer_LocalPartitioningUtilities.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,13 +400,14 @@ buildGhostedCellOneRing(const Teuchos::RCP<const Teuchos::Comm<int> > & comm,
// build an array containing only the ghstd cells
int indx = 0;
PHX::View<panzer::GlobalOrdinal*> ghstd_cells("ghstd_cells",ghstd_cells_set.size());
auto ghstd_cells_h = Kokkos::create_mirror_view(ghstd_cells);
for(auto global_cell_index : ghstd_cells_set) {
ghstd_cells(indx) = global_cell_index;
ghstd_cells_h(indx) = global_cell_index;
indx++;
}

// print_view_1D("ghstd_cells",ghstd_cells);

Kokkos::deep_copy(ghstd_cells, ghstd_cells_h);
return ghstd_cells;
}

Expand Down

0 comments on commit 5590ce5

Please sign in to comment.