From 5590ce545814b0c1e8b61faacc2272c5163da78f Mon Sep 17 00:00:00 2001 From: Matthew Bettencourt Date: Fri, 30 Jul 2021 07:36:57 -0600 Subject: [PATCH] have parallel mesh test passing --- .../src/Panzer_STK_LocalMeshUtilities.cpp | 7 +-- .../parallel_local_mesh_utilities.cpp | 48 ++++++++++++------- .../src/Panzer_LocalPartitioningUtilities.cpp | 5 +- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/packages/panzer/adapters-stk/src/Panzer_STK_LocalMeshUtilities.cpp b/packages/panzer/adapters-stk/src/Panzer_STK_LocalMeshUtilities.cpp index 6d6eb1dc1506..ce74b9c82020 100644 --- a/packages/panzer/adapters-stk/src/Panzer_STK_LocalMeshUtilities.cpp +++ b/packages/panzer/adapters-stk/src/Panzer_STK_LocalMeshUtilities.cpp @@ -119,13 +119,14 @@ buildGhostedVertices(const Tpetra::Import ghstd_vertices("ghstd_vertices",ghstd_cell_cnt,vertices_per_cell,space_dim); { - auto ghstd_vertices_view = ghstd_vertices_mv->getLocalViewHost(Tpetra::Access::ReadOnly); - for(size_t i=0;igetLocalViewDevice(Tpetra::Access::ReadOnly); + Kokkos::parallel_for(ghstd_cell_cnt, KOKKOS_LAMBDA (size_t i) { int l = 0; for(int j=0;jelement_blocks.at("eblock-0"); + auto global_cells_h = Kokkos::create_mirror_view(block.global_cells); + Kokkos::deep_copy(global_cells_h, block.global_cells); out << "Element Block eblock-0" << std::endl; @@ -111,15 +113,17 @@ TEUCHOS_UNIT_TEST(parallelLocalMeshUtilities, 1D_mesh) TEST_EQUALITY(block.num_owned_cells, 2); TEST_EQUALITY(block.num_ghstd_cells, 1); TEST_EQUALITY(block.num_virtual_cells, 1); - TEST_EQUALITY((int) block.global_cells(0), 0); - TEST_EQUALITY((int) block.global_cells(1), 1); - TEST_EQUALITY((int) block.global_cells(2), 2); - TEST_EQUALITY((int) block.global_cells(3), 6); + TEST_EQUALITY((int) global_cells_h(0), 0); + TEST_EQUALITY((int) global_cells_h(1), 1); + TEST_EQUALITY((int) global_cells_h(2), 2); + TEST_EQUALITY((int) global_cells_h(3), 6); TEST_ASSERT(block.has_connectivity); } { const auto & block = mesh_info->element_blocks.at("eblock-1"); + auto global_cells_h = Kokkos::create_mirror_view(block.global_cells); + Kokkos::deep_copy(global_cells_h, block.global_cells); out << "Element Block eblock-1" << std::endl; @@ -127,15 +131,17 @@ TEUCHOS_UNIT_TEST(parallelLocalMeshUtilities, 1D_mesh) TEST_EQUALITY(block.num_owned_cells, 2); TEST_EQUALITY(block.num_ghstd_cells, 2); TEST_EQUALITY(block.num_virtual_cells, 0); - TEST_EQUALITY((int) block.global_cells(0), 3); - TEST_EQUALITY((int) block.global_cells(1), 4); - TEST_EQUALITY((int) block.global_cells(2), 5); - TEST_EQUALITY((int) block.global_cells(3), 2); + TEST_EQUALITY((int) global_cells_h(0), 3); + TEST_EQUALITY((int) global_cells_h(1), 4); + TEST_EQUALITY((int) global_cells_h(2), 5); + TEST_EQUALITY((int) global_cells_h(3), 2); TEST_ASSERT(block.has_connectivity); } { const auto & block = mesh_info->sidesets.at("eblock-0").at("left"); + auto global_cells_h = Kokkos::create_mirror_view(block.global_cells); + Kokkos::deep_copy(global_cells_h, block.global_cells); out << "Sideset eblock-0 left" << std::endl; @@ -143,8 +149,8 @@ TEUCHOS_UNIT_TEST(parallelLocalMeshUtilities, 1D_mesh) TEST_EQUALITY(block.num_owned_cells, 1); TEST_EQUALITY(block.num_ghstd_cells, 0); TEST_EQUALITY(block.num_virtual_cells, 1); - TEST_EQUALITY((int) block.global_cells(0), 0); - TEST_EQUALITY((int) block.global_cells(1), 6); + TEST_EQUALITY((int) global_cells_h(0), 0); + TEST_EQUALITY((int) global_cells_h(1), 6); TEST_ASSERT(block.has_connectivity); } @@ -153,6 +159,8 @@ TEUCHOS_UNIT_TEST(parallelLocalMeshUtilities, 1D_mesh) } else { { const auto & block = mesh_info->element_blocks.at("eblock-0"); + auto global_cells_h = Kokkos::create_mirror_view(block.global_cells); + Kokkos::deep_copy(global_cells_h, block.global_cells); out << "Element Block eblock-0" << std::endl; @@ -160,14 +168,16 @@ TEUCHOS_UNIT_TEST(parallelLocalMeshUtilities, 1D_mesh) TEST_EQUALITY(block.num_owned_cells, 1); TEST_EQUALITY(block.num_ghstd_cells, 2); TEST_EQUALITY(block.num_virtual_cells, 0); - TEST_EQUALITY((int) block.global_cells(0), 2); - TEST_EQUALITY((int) block.global_cells(1), 3); - TEST_EQUALITY((int) block.global_cells(2), 1); + TEST_EQUALITY((int) global_cells_h(0), 2); + TEST_EQUALITY((int) global_cells_h(1), 3); + TEST_EQUALITY((int) global_cells_h(2), 1); TEST_ASSERT(block.has_connectivity); } { const auto & block = mesh_info->element_blocks.at("eblock-1"); + auto global_cells_h = Kokkos::create_mirror_view(block.global_cells); + Kokkos::deep_copy(global_cells_h, block.global_cells); out << "Element Block eblock-1" << std::endl; @@ -175,9 +185,9 @@ TEUCHOS_UNIT_TEST(parallelLocalMeshUtilities, 1D_mesh) TEST_EQUALITY(block.num_owned_cells, 1); TEST_EQUALITY(block.num_ghstd_cells, 1); TEST_EQUALITY(block.num_virtual_cells, 1); - TEST_EQUALITY((int) block.global_cells(0), 5); - TEST_EQUALITY((int) block.global_cells(1), 4); - TEST_EQUALITY((int) block.global_cells(2), 7); + TEST_EQUALITY((int) global_cells_h(0), 5); + TEST_EQUALITY((int) global_cells_h(1), 4); + TEST_EQUALITY((int) global_cells_h(2), 7); TEST_ASSERT(block.has_connectivity); } @@ -185,6 +195,8 @@ TEUCHOS_UNIT_TEST(parallelLocalMeshUtilities, 1D_mesh) { const auto & block = mesh_info->sidesets.at("eblock-1").at("right"); + auto global_cells_h = Kokkos::create_mirror_view(block.global_cells); + Kokkos::deep_copy(global_cells_h, block.global_cells); out << "Sideset eblock-1 right" << std::endl; @@ -192,8 +204,8 @@ TEUCHOS_UNIT_TEST(parallelLocalMeshUtilities, 1D_mesh) TEST_EQUALITY(block.num_owned_cells, 1); TEST_EQUALITY(block.num_ghstd_cells, 0); TEST_EQUALITY(block.num_virtual_cells, 1); - TEST_EQUALITY((int) block.global_cells(0), 5); - TEST_EQUALITY((int) block.global_cells(1), 7); + TEST_EQUALITY((int) global_cells_h(0), 5); + TEST_EQUALITY((int) global_cells_h(1), 7); TEST_ASSERT(block.has_connectivity); } } diff --git a/packages/panzer/disc-fe/src/Panzer_LocalPartitioningUtilities.cpp b/packages/panzer/disc-fe/src/Panzer_LocalPartitioningUtilities.cpp index ddc5b0f368ec..fdd84c54870a 100644 --- a/packages/panzer/disc-fe/src/Panzer_LocalPartitioningUtilities.cpp +++ b/packages/panzer/disc-fe/src/Panzer_LocalPartitioningUtilities.cpp @@ -400,13 +400,14 @@ buildGhostedCellOneRing(const Teuchos::RCP > & comm, // build an array containing only the ghstd cells int indx = 0; PHX::View ghstd_cells("ghstd_cells",ghstd_cells_set.size()); + auto ghstd_cells_h = Kokkos::create_mirror_view(ghstd_cells); for(auto global_cell_index : ghstd_cells_set) { - ghstd_cells(indx) = global_cell_index; + ghstd_cells_h(indx) = global_cell_index; indx++; } // print_view_1D("ghstd_cells",ghstd_cells); - + Kokkos::deep_copy(ghstd_cells, ghstd_cells_h); return ghstd_cells; }