Skip to content

Commit

Permalink
panzer: Face to elem (trilinos#9509)
Browse files Browse the repository at this point in the history 
* have first unit test passing

* have test running, but not passing

* A bit of work on workset building w/o uvm

* Have use of UVM reduced

* More progress

* Have one mesh test passing

* have parallel mesh test passin

* Have all mesh tests passing

* Added missing fence

* Removed warning

* Fixed this ref

Co-authored-by: Matthew Bettencourt <[email protected]>
  • Loading branch information
@bathmatt
bathmatt and Matthew Bettencourt authored Aug 2, 2021
1 parent 0fc6674 commit 286bab8
Show file tree
Hide file tree
Showing 13 changed files with 504 additions and 298 deletions.
156 changes: 93 additions & 63 deletions packages/panzer/adapters-stk/src/Panzer_STK_LocalMeshUtilities.cpp
View file

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion packages/panzer/adapters-stk/src/Panzer_STK_SurfaceNodeNormals.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,8 @@ namespace panzer_stk {
elementEntities.push_back(*parentElement); // notice this is size 1!
PHX::MDField<double,panzer::Cell,panzer::NODE,panzer::Dim> vertices
= af.buildStaticArray<double,Cell,NODE,Dim>("",elementEntities.size(), parentTopology->getVertexCount(), mesh->getDimension());
mesh->getElementVerticesNoResize(elementEntities,elementBlockName,vertices);
auto vert_view = vertices.get_view();
mesh->getElementVerticesNoResize(elementEntities,elementBlockName,vert_view);

panzer::CellData sideCellData(1,*sideID,parentTopology); // this is size 1 because elementEntties is size 1!
RCP<panzer::IntegrationRule> ir = Teuchos::rcp(new panzer::IntegrationRule(cubDegree,sideCellData));
Expand Down
5 changes: 3 additions & 2 deletions packages/panzer/adapters-stk/src/evaluators/Panzer_STK_GatherRefCoords_impl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,9 @@ evaluateFields(typename Traits::EvalData workset)
std::vector<stk::mesh::Entity> selected_elements;
for(std::size_t cell=0;cell<localCellIds.size();cell++)
selected_elements.push_back(localElements[localCellIds[cell]]);

mesh_->getElementVertices_FromCoordsNoResize(selected_elements,coordField_);

auto coordField = coordField_.get_view();
mesh_->getElementVertices_FromCoordsNoResize(selected_elements,coordField);
}

#endif
8 changes: 6 additions & 2 deletions packages/panzer/adapters-stk/src/stk_interface/Panzer_STK_Interface.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1721,6 +1721,7 @@ void STK_Interface::getElementVertices_FromCoords(const std::vector<stk::mesh::E

// allocate space
vertices = Kokkos::createDynRankView(vertices,"vertices",elements.size(),masterVertexCount,getDimension());
auto vertices_h = Kokkos::create_mirror_view(vertices);

// loop over each requested element
unsigned dim = getDimension();
Expand All @@ -1745,9 +1746,10 @@ void STK_Interface::getElementVertices_FromCoords(const std::vector<stk::mesh::E

// set each dimension of the coordinate
for(unsigned d=0;d<dim;d++)
vertices(cell,node,d) = coord[d];
vertices_h(cell,node,d) = coord[d];
}
}
Kokkos::deep_copy(vertices, vertices_h);
}

template <typename ArrayT>
Expand All @@ -1768,6 +1770,7 @@ void STK_Interface::getElementVertices_FromCoordsNoResize(const std::vector<stk:

// loop over each requested element
unsigned dim = getDimension();
auto vertices_h = Kokkos::create_mirror_view(vertices);
for(std::size_t cell=0;cell<elements.size();cell++) {
stk::mesh::Entity element = elements[cell];
TEUCHOS_ASSERT(element!=0);
Expand All @@ -1789,9 +1792,10 @@ void STK_Interface::getElementVertices_FromCoordsNoResize(const std::vector<stk:

// set each dimension of the coordinate
for(unsigned d=0;d<dim;d++)
vertices(cell,node,d) = coord[d];
vertices_h(cell,node,d) = coord[d];
}
}
Kokkos::deep_copy(vertices, vertices_h);
}

template <typename ArrayT>
Expand Down
17 changes: 10 additions & 7 deletions packages/panzer/adapters-stk/test/face_to_elem/Panzer_Relations.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,22 +122,23 @@ FaceToElems::FaceToElems(Teuchos::RCP<panzer::ConnManager> conn) :


face_to_elem_ = PHX::View<GlobalOrdinal*[2]>("FaceToElems::face_to_elem_",face_map->getNodeNumElements());
auto face_to_elem_h = Kokkos::create_mirror_view(face_to_elem_);
num_boundary_faces_=0;
for (int i(0); i < face_to_elem_.extent_int(0); ++i)
{
typename Graph::nonconst_global_inds_host_view_type indices("indices", 2);
size_t num_ent;
graph_overlap.getGlobalRowCopy(face_map->getGlobalElement(i), indices, num_ent);
assert(num_ent == 2 || num_ent == 1);
face_to_elem_(i,0) = indices(0);
face_to_elem_h(i,0) = indices(0);
if ( num_ent == 2)
face_to_elem_(i,1) = indices(1);
face_to_elem_h(i,1) = indices(1);
else {
face_to_elem_(i,1) = -1;
face_to_elem_h(i,1) = -1;
num_boundary_faces_++;
}
}

Kokkos::deep_copy(face_to_elem_, face_to_elem_h);

// Now we can get the nodal values
std::vector<std::vector<int>> face_to_node;
Expand All @@ -156,21 +157,23 @@ FaceToElems::FaceToElems(Teuchos::RCP<panzer::ConnManager> conn) :

}
face_to_node_ = PHX::View<GlobalOrdinal**>("face_to_node", face_to_elem_.extent(0), face_to_node[0].size());
Kokkos::deep_copy(face_to_node_, -1);
auto face_to_node_h = Kokkos::create_mirror_view(face_to_node_);
Kokkos::deep_copy(face_to_node_h, -1);
for (int ielem=0;ielem< static_cast<int>(elem_to_face_.size()); ++ielem) {
const auto * connectivity = conn_->getConnectivity(ielem);
for (int iface=0; iface <static_cast<int>(elem_to_face_[ielem].size()); ++iface ) {
for (int inode(0); inode < face_to_node_.extent_int(1); ++inode)
{
GlobalOrdinal g_face = elem_to_face_[ielem][iface];
LocalOrdinal l_face = face_map->getLocalElement(g_face);
face_to_node_(l_face, inode) = connectivity[face_to_node[iface][inode]];
face_to_node_h(l_face, inode) = connectivity[face_to_node[iface][inode]];
}
}
}
for (int i(0); i < face_to_node_.extent_int(0); ++i)
for (int j(0); j < face_to_node_.extent_int(1); ++j)
TEUCHOS_ASSERT(face_to_node_(i,j) >=0);
TEUCHOS_ASSERT(face_to_node_h(i,j) >=0);
Kokkos::deep_copy(face_to_node_, face_to_node_h);
}

void FaceToElems::setNormals(Teuchos::RCP<std::vector<panzer::Workset> > worksets){
Expand Down
48 changes: 30 additions & 18 deletions packages/panzer/adapters-stk/test/local_mesh/parallel_local_mesh_utilities.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,47 +104,53 @@ TEUCHOS_UNIT_TEST(parallelLocalMeshUtilities, 1D_mesh)

{
const auto & block = mesh_info->element_blocks.at("eblock-0");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Element Block eblock-0" << std::endl;

// Block should have some basic stuff working
TEST_EQUALITY(block.num_owned_cells, 2);
TEST_EQUALITY(block.num_ghstd_cells, 1);
TEST_EQUALITY(block.num_virtual_cells, 1);
TEST_EQUALITY((int) block.global_cells(0), 0);
TEST_EQUALITY((int) block.global_cells(1), 1);
TEST_EQUALITY((int) block.global_cells(2), 2);
TEST_EQUALITY((int) block.global_cells(3), 6);
TEST_EQUALITY((int) global_cells_h(0), 0);
TEST_EQUALITY((int) global_cells_h(1), 1);
TEST_EQUALITY((int) global_cells_h(2), 2);
TEST_EQUALITY((int) global_cells_h(3), 6);
TEST_ASSERT(block.has_connectivity);
}

{
const auto & block = mesh_info->element_blocks.at("eblock-1");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Element Block eblock-1" << std::endl;

// Block should be empty
TEST_EQUALITY(block.num_owned_cells, 2);
TEST_EQUALITY(block.num_ghstd_cells, 2);
TEST_EQUALITY(block.num_virtual_cells, 0);
TEST_EQUALITY((int) block.global_cells(0), 3);
TEST_EQUALITY((int) block.global_cells(1), 4);
TEST_EQUALITY((int) block.global_cells(2), 5);
TEST_EQUALITY((int) block.global_cells(3), 2);
TEST_EQUALITY((int) global_cells_h(0), 3);
TEST_EQUALITY((int) global_cells_h(1), 4);
TEST_EQUALITY((int) global_cells_h(2), 5);
TEST_EQUALITY((int) global_cells_h(3), 2);
TEST_ASSERT(block.has_connectivity);
}

{
const auto & block = mesh_info->sidesets.at("eblock-0").at("left");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Sideset eblock-0 left" << std::endl;

// Block should have some basic stuff working
TEST_EQUALITY(block.num_owned_cells, 1);
TEST_EQUALITY(block.num_ghstd_cells, 0);
TEST_EQUALITY(block.num_virtual_cells, 1);
TEST_EQUALITY((int) block.global_cells(0), 0);
TEST_EQUALITY((int) block.global_cells(1), 6);
TEST_EQUALITY((int) global_cells_h(0), 0);
TEST_EQUALITY((int) global_cells_h(1), 6);
TEST_ASSERT(block.has_connectivity);
}

Expand All @@ -153,47 +159,53 @@ TEUCHOS_UNIT_TEST(parallelLocalMeshUtilities, 1D_mesh)
} else {
{
const auto & block = mesh_info->element_blocks.at("eblock-0");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Element Block eblock-0" << std::endl;

// Block should have some basic stuff working
TEST_EQUALITY(block.num_owned_cells, 1);
TEST_EQUALITY(block.num_ghstd_cells, 2);
TEST_EQUALITY(block.num_virtual_cells, 0);
TEST_EQUALITY((int) block.global_cells(0), 2);
TEST_EQUALITY((int) block.global_cells(1), 3);
TEST_EQUALITY((int) block.global_cells(2), 1);
TEST_EQUALITY((int) global_cells_h(0), 2);
TEST_EQUALITY((int) global_cells_h(1), 3);
TEST_EQUALITY((int) global_cells_h(2), 1);
TEST_ASSERT(block.has_connectivity);
}

{
const auto & block = mesh_info->element_blocks.at("eblock-1");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Element Block eblock-1" << std::endl;

// Block should be empty
TEST_EQUALITY(block.num_owned_cells, 1);
TEST_EQUALITY(block.num_ghstd_cells, 1);
TEST_EQUALITY(block.num_virtual_cells, 1);
TEST_EQUALITY((int) block.global_cells(0), 5);
TEST_EQUALITY((int) block.global_cells(1), 4);
TEST_EQUALITY((int) block.global_cells(2), 7);
TEST_EQUALITY((int) global_cells_h(0), 5);
TEST_EQUALITY((int) global_cells_h(1), 4);
TEST_EQUALITY((int) global_cells_h(2), 7);
TEST_ASSERT(block.has_connectivity);
}

// TEUCHOS_ASSERT(mesh_info->sidesets.find("eblock-0") == mesh_info->sidesets.end());

{
const auto & block = mesh_info->sidesets.at("eblock-1").at("right");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Sideset eblock-1 right" << std::endl;

// Block should have some basic stuff working
TEST_EQUALITY(block.num_owned_cells, 1);
TEST_EQUALITY(block.num_ghstd_cells, 0);
TEST_EQUALITY(block.num_virtual_cells, 1);
TEST_EQUALITY((int) block.global_cells(0), 5);
TEST_EQUALITY((int) block.global_cells(1), 7);
TEST_EQUALITY((int) global_cells_h(0), 5);
TEST_EQUALITY((int) global_cells_h(1), 7);
TEST_ASSERT(block.has_connectivity);
}
}
Expand Down
48 changes: 30 additions & 18 deletions packages/panzer/adapters-stk/test/local_mesh/parallel_periodic_local_mesh_utilities.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,8 @@ TEUCHOS_UNIT_TEST(parallelPeriodicLocalMeshUtilities, 1D_mesh)

{
const auto & block = mesh_info->element_blocks.at("eblock-0");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Element Block eblock-0" << std::endl;

Expand All @@ -183,15 +185,17 @@ TEUCHOS_UNIT_TEST(parallelPeriodicLocalMeshUtilities, 1D_mesh)
TEST_EQUALITY(block.num_owned_cells, 2);
TEST_EQUALITY(block.num_ghstd_cells, 2);
TEST_EQUALITY(block.num_virtual_cells, 0);
TEST_EQUALITY((int) block.global_cells(0), 0);
TEST_EQUALITY((int) block.global_cells(1), 1);
TEST_EQUALITY((int) block.global_cells(2), 2);
TEST_EQUALITY((int) block.global_cells(3), 5);
TEST_EQUALITY((int) global_cells_h(0), 0);
TEST_EQUALITY((int) global_cells_h(1), 1);
TEST_EQUALITY((int) global_cells_h(2), 2);
TEST_EQUALITY((int) global_cells_h(3), 5);
TEST_ASSERT(block.has_connectivity);
}

{
const auto & block = mesh_info->element_blocks.at("eblock-1");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Element Block eblock-1" << std::endl;

Expand All @@ -215,15 +219,17 @@ TEUCHOS_UNIT_TEST(parallelPeriodicLocalMeshUtilities, 1D_mesh)
TEST_EQUALITY(block.num_owned_cells, 2);
TEST_EQUALITY(block.num_ghstd_cells, 2);
TEST_EQUALITY(block.num_virtual_cells, 0);
TEST_EQUALITY((int) block.global_cells(0), 3);
TEST_EQUALITY((int) block.global_cells(1), 4);
TEST_EQUALITY((int) block.global_cells(2), 5);
TEST_EQUALITY((int) block.global_cells(3), 2);
TEST_EQUALITY((int) global_cells_h(0), 3);
TEST_EQUALITY((int) global_cells_h(1), 4);
TEST_EQUALITY((int) global_cells_h(2), 5);
TEST_EQUALITY((int) global_cells_h(3), 2);
TEST_ASSERT(block.has_connectivity);
}

{
const auto & block = mesh_info->sidesets.at("eblock-0").at("left");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Sideset eblock-0 left" << std::endl;

Expand All @@ -237,8 +243,8 @@ TEUCHOS_UNIT_TEST(parallelPeriodicLocalMeshUtilities, 1D_mesh)
TEST_EQUALITY(block.num_owned_cells, 1);
TEST_EQUALITY(block.num_ghstd_cells, 1);
TEST_EQUALITY(block.num_virtual_cells, 0);
TEST_EQUALITY((int) block.global_cells(0), 0);
TEST_EQUALITY((int) block.global_cells(1), 5);
TEST_EQUALITY((int) global_cells_h(0), 0);
TEST_EQUALITY((int) global_cells_h(1), 5);
TEST_ASSERT(block.has_connectivity);
}

Expand All @@ -247,6 +253,8 @@ TEUCHOS_UNIT_TEST(parallelPeriodicLocalMeshUtilities, 1D_mesh)
} else {
{
const auto & block = mesh_info->element_blocks.at("eblock-0");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Element Block eblock-0" << std::endl;

Expand All @@ -270,14 +278,16 @@ TEUCHOS_UNIT_TEST(parallelPeriodicLocalMeshUtilities, 1D_mesh)
TEST_EQUALITY(block.num_owned_cells, 1);
TEST_EQUALITY(block.num_ghstd_cells, 2);
TEST_EQUALITY(block.num_virtual_cells, 0);
TEST_EQUALITY((int) block.global_cells(0), 2);
TEST_EQUALITY((int) block.global_cells(1), 3);
TEST_EQUALITY((int) block.global_cells(2), 1);
TEST_EQUALITY((int) global_cells_h(0), 2);
TEST_EQUALITY((int) global_cells_h(1), 3);
TEST_EQUALITY((int) global_cells_h(2), 1);
TEST_ASSERT(block.has_connectivity);
}

{
const auto & block = mesh_info->element_blocks.at("eblock-1");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Element Block eblock-1" << std::endl;

Expand All @@ -301,16 +311,18 @@ TEUCHOS_UNIT_TEST(parallelPeriodicLocalMeshUtilities, 1D_mesh)
TEST_EQUALITY(block.num_owned_cells, 1);
TEST_EQUALITY(block.num_ghstd_cells, 2);
TEST_EQUALITY(block.num_virtual_cells, 0);
TEST_EQUALITY((int) block.global_cells(0), 5);
TEST_EQUALITY((int) block.global_cells(1), 0);
TEST_EQUALITY((int) block.global_cells(2), 4);
TEST_EQUALITY((int) global_cells_h(0), 5);
TEST_EQUALITY((int) global_cells_h(1), 0);
TEST_EQUALITY((int) global_cells_h(2), 4);
TEST_ASSERT(block.has_connectivity);
}

// TEUCHOS_ASSERT(mesh_info->sidesets.find("eblock-0") == mesh_info->sidesets.end());

{
const auto & block = mesh_info->sidesets.at("eblock-1").at("right");
auto global_cells_h = Kokkos::create_mirror_view(block.global_cells);
Kokkos::deep_copy(global_cells_h, block.global_cells);

out << "Sideset eblock-1 right" << std::endl;

Expand All @@ -324,8 +336,8 @@ TEUCHOS_UNIT_TEST(parallelPeriodicLocalMeshUtilities, 1D_mesh)
TEST_EQUALITY(block.num_owned_cells, 1);
TEST_EQUALITY(block.num_ghstd_cells, 1);
TEST_EQUALITY(block.num_virtual_cells, 0);
TEST_EQUALITY((int) block.global_cells(0), 5);
TEST_EQUALITY((int) block.global_cells(1), 0);
TEST_EQUALITY((int) global_cells_h(0), 5);
TEST_EQUALITY((int) global_cells_h(1), 0);
TEST_ASSERT(block.has_connectivity);
}
}
Expand Down
Loading

0 comments on commit 286bab8

Please sign in to comment.