Skip to content

Commit

Permalink
Merge Pull Request #9632 from trilinos/Trilinos/tangent-gather
Browse files Browse the repository at this point in the history 
Automatically Merged using Trilinos Pull Request AutoTester
PR Title: panzer: Tangent gather
PR Author: bathmatt
  • Loading branch information
@trilinos-autotester
trilinos-autotester authored Aug 31, 2021
2 parents 9323eee + 3faef8b commit 0e60079
Show file tree
Hide file tree
Showing 4 changed files with 113 additions and 110 deletions.
78 changes: 52 additions & 26 deletions packages/panzer/adapters-stk/test/evaluator_tests/gather_solution.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@

// Epetra
#include "Epetra_MpiComm.h"
// Kokkos
#include "Kokkos_View_Fad.hpp"

// Panzer
#include "PanzerAdaptersSTK_config.hpp"
Expand Down Expand Up @@ -492,28 +494,36 @@ namespace panzer {
TEST_EQUALITY(fieldData2_q1.extent(1),4);
TEST_EQUALITY(fieldData1_q1.size(),Teuchos::as<unsigned int>(4*4/numProcs));
TEST_EQUALITY(fieldData2_q1.size(),Teuchos::as<unsigned int>(4*4/numProcs));

auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view());
auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view());
Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view());
Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view());

for(unsigned int i=0;i<fieldData1_q1.extent(0);i++)
for(unsigned int j=0;j<fieldData1_q1.extent(1);j++)
TEST_EQUALITY(fieldData1_q1(i,j),123.0+myRank);
TEST_EQUALITY(fieldData1_q1_h(i,j),123.0+myRank);

for(unsigned int i=0;i<fieldData2_q1.extent(0);i++)
for(unsigned int j=0;j<fieldData2_q1.extent(1);j++)
TEST_EQUALITY(fieldData2_q1(i,j),123.0+myRank);
TEST_EQUALITY(fieldData2_q1_h(i,j),123.0+myRank);
}
{
PHX::MDField<panzer::Traits::Residual::ScalarT,panzer::Cell,panzer::BASIS>
fieldData_qedge1(fieldName_qedge1,basis_qedge1->functional);

fm.getFieldData<panzer::Traits::Residual>(fieldData_qedge1);


auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view());
Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view());

TEST_EQUALITY(fieldData_qedge1.extent(0),Teuchos::as<unsigned int>(4/numProcs));
TEST_EQUALITY(fieldData_qedge1.extent(1),4);
TEST_EQUALITY(fieldData_qedge1.size(),Teuchos::as<unsigned int>(4*4/numProcs));

for(unsigned int cell=0;cell<fieldData_qedge1.extent(0);++cell)
for(unsigned int pt=0;pt<fieldData_qedge1.extent(1);pt++)
TEST_EQUALITY(fieldData_qedge1(cell,pt),123.0+myRank);
TEST_EQUALITY(fieldData_qedge1_h(cell,pt),123.0+myRank);
}

// test Jacobian fields
Expand All @@ -526,16 +536,21 @@ namespace panzer {
fm.getFieldData<panzer::Traits::Jacobian>(fieldData1_q1);
fm.getFieldData<panzer::Traits::Jacobian>(fieldData2_q1);

auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view());
auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view());
Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view());
Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view());

for(unsigned int cell=0;cell<fieldData1_q1.extent(0);++cell) {
for(unsigned int pt=0;pt<fieldData1_q1.extent(1);pt++) {
TEST_EQUALITY(fieldData1_q1(cell,pt),123.0+myRank);
TEST_EQUALITY(fieldData1_q1(cell,pt).availableSize(),12);
TEST_EQUALITY(fieldData1_q1_h(cell,pt),123.0+myRank);
TEST_EQUALITY(fieldData1_q1_h(cell,pt).availableSize(),12);
}
}
for(unsigned int cell=0;cell<fieldData2_q1.extent(0);++cell) {
for(unsigned int pt=0;pt<fieldData2_q1.extent(1);pt++) {
TEST_EQUALITY(fieldData2_q1(cell,pt),123.0+myRank);
TEST_EQUALITY(fieldData2_q1(cell,pt).availableSize(),12);
TEST_EQUALITY(fieldData2_q1_h(cell,pt),123.0+myRank);
TEST_EQUALITY(fieldData2_q1_h(cell,pt).availableSize(),12);
}
}
}
Expand All @@ -545,10 +560,13 @@ namespace panzer {

fm.getFieldData<panzer::Traits::Jacobian>(fieldData_qedge1);

auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view());
Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view());

for(unsigned int cell=0;cell<fieldData_qedge1.extent(0);++cell) {
for(unsigned int pt=0;pt<fieldData_qedge1.extent(1);++pt) {
TEST_EQUALITY(fieldData_qedge1(cell,pt),123.0+myRank);
TEST_EQUALITY(fieldData_qedge1(cell,pt).availableSize(),12);
TEST_EQUALITY(fieldData_qedge1_h(cell,pt),123.0+myRank);
TEST_EQUALITY(fieldData_qedge1_h(cell,pt).availableSize(),12);
}
}
}
Expand All @@ -563,33 +581,38 @@ namespace panzer {
fm.getFieldData<panzer::Traits::Tangent>(fieldData1_q1);
fm.getFieldData<panzer::Traits::Tangent>(fieldData2_q1);

auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view());
auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view());
Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view());
Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view());

for(unsigned int cell=0;cell<fieldData1_q1.extent(0);++cell) {
for(unsigned int pt=0;pt<fieldData1_q1.extent(1);pt++) {
if (enable_tangents) {
TEST_EQUALITY(fieldData1_q1(cell,pt).val(),123.0+myRank);
TEST_EQUALITY(fieldData1_q1(cell,pt).availableSize(),num_tangent);
TEST_EQUALITY(fieldData1_q1_h(cell,pt).val(),123.0+myRank);
TEST_EQUALITY(fieldData1_q1_h(cell,pt).availableSize(),num_tangent);
for (int i=0; i<num_tangent; ++i)
TEST_EQUALITY(fieldData1_q1(cell,pt).dx(i),0.123+myRank+i);
TEST_EQUALITY(fieldData1_q1_h(cell,pt).dx(i),0.123+myRank+i);
}
else {
TEST_EQUALITY(fieldData1_q1(cell,pt),123.0+myRank);
TEST_EQUALITY(fieldData1_q1(cell,pt).availableSize(),0);
TEST_EQUALITY(fieldData1_q1_h(cell,pt),123.0+myRank);
TEST_EQUALITY(fieldData1_q1_h(cell,pt).availableSize(),0);
}
}
}
for(unsigned int cell=0;cell<fieldData2_q1.extent(0);++cell) {
for(unsigned int pt=0;pt<fieldData2_q1.extent(1);pt++) {
if (enable_tangents) {
TEST_EQUALITY(fieldData2_q1(cell,pt).val(),123.0+myRank);
TEST_EQUALITY(fieldData2_q1(cell,pt).availableSize(),num_tangent);
TEST_EQUALITY(fieldData2_q1_h(cell,pt).val(),123.0+myRank);
TEST_EQUALITY(fieldData2_q1_h(cell,pt).availableSize(),num_tangent);
for (int i=0; i<num_tangent; ++i) {
TEST_EQUALITY(fieldData1_q1(cell,pt).dx(i),0.123+myRank+i);
TEST_EQUALITY(fieldData2_q1(cell,pt).dx(i),0.123+myRank+i);
TEST_EQUALITY(fieldData2_q1_h(cell,pt).dx(i),0.123+myRank+i);
TEST_EQUALITY(fieldData2_q1_h(cell,pt).dx(i),0.123+myRank+i);
}
}
else {
TEST_EQUALITY(fieldData2_q1(cell,pt),123.0+myRank);
TEST_EQUALITY(fieldData2_q1(cell,pt).availableSize(),0);
TEST_EQUALITY(fieldData2_q1_h(cell,pt),123.0+myRank);
TEST_EQUALITY(fieldData2_q1_h(cell,pt).availableSize(),0);
}
}
}
Expand All @@ -600,17 +623,20 @@ namespace panzer {

fm.getFieldData<panzer::Traits::Tangent>(fieldData_qedge1);

auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view());
Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view());

for(unsigned int cell=0;cell<fieldData_qedge1.extent(0);++cell) {
for(unsigned int pt=0;pt<fieldData_qedge1.extent(1);++pt) {
if (enable_tangents) {
TEST_EQUALITY(fieldData_qedge1(cell,pt).val(),123.0+myRank);
TEST_EQUALITY(fieldData_qedge1(cell,pt).availableSize(),num_tangent);
TEST_EQUALITY(fieldData_qedge1_h(cell,pt).val(),123.0+myRank);
TEST_EQUALITY(fieldData_qedge1_h(cell,pt).availableSize(),num_tangent);
for (int i=0; i<num_tangent; ++i)
TEST_EQUALITY(fieldData_qedge1(cell,pt).dx(i),0.123+myRank+i);
TEST_EQUALITY(fieldData_qedge1_h(cell,pt).dx(i),0.123+myRank+i);
}
else {
TEST_EQUALITY(fieldData_qedge1(cell,pt),123.0+myRank);
TEST_EQUALITY(fieldData_qedge1(cell,pt).availableSize(),0);
TEST_EQUALITY(fieldData_qedge1_h(cell,pt),123.0+myRank);
TEST_EQUALITY(fieldData_qedge1_h(cell,pt).availableSize(),0);
}
}
}
Expand Down
118 changes: 45 additions & 73 deletions packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Epetra_impl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -472,86 +472,58 @@ evaluateFields(
// NOTE: A reordering of these loops will likely improve performance. The
// "getGIDFieldOffsets may be expensive. However the "getElementGIDs"
// can be cheaper. However the lookup for LIDs may be more expensive!
if (x_.is_null())
{
// Gather operation for each cell in the workset.
for (int cell(0); cell < numCells; ++cell)
{
size_t cellLocalId(localCellIds[cell]);
auto LIDs = globalIndexer_->getElementLIDs(cellLocalId);

// Loop over the fields to be gathered.
for (int fieldInd(0); fieldInd < numFields; ++fieldInd)
{
MDField<ScalarT, Cell, NODE>& field = gatherFields_[fieldInd];
int fieldNum(fieldIds_[fieldInd]);
const vector<int>& elmtOffset =
globalIndexer_->getGIDFieldOffsets(blockId, fieldNum);
int numBases(elmtOffset.size());

// Loop over the basis functions and fill the fields.
for (int basis(0); basis < numBases; ++basis)
{
int offset(elmtOffset[basis]), lid(LIDs[offset]);
field(cell, basis) = (*xEvRoGed_)[lid];
} // end loop over the basis functions
} // end loop over the fields to be gathered
} // end loop over localCellIds
}
else // if (not x_.is_null())
auto LIDs = globalIndexer_->getLIDs();
auto LIDs_h = Kokkos::create_mirror_view(LIDs);
Kokkos::deep_copy(LIDs_h, LIDs);
// Loop over the fields to be gathered.
for (int fieldInd(0); fieldInd < numFields; ++fieldInd)
{
MDField<ScalarT, Cell, NODE>& field = gatherFields_[fieldInd];
auto field_h = Kokkos::create_mirror_view(field.get_static_view());
int fieldNum(fieldIds_[fieldInd]);
const vector<int>& elmtOffset =
globalIndexer_->getGIDFieldOffsets(blockId, fieldNum);
int numBases(elmtOffset.size());
// Gather operation for each cell in the workset.
for (int cell(0); cell < numCells; ++cell)
{
size_t cellLocalId(localCellIds[cell]);
auto LIDs = globalIndexer_->getElementLIDs(cellLocalId);

// Loop over the fields to be gathered.
for (int fieldInd(0); fieldInd < numFields; ++fieldInd)
size_t cellLocalId(localCellIds[cell]);
// Loop over the basis functions and fill the fields.
for (int basis(0); basis < numBases; ++basis)
{
MDField<ScalarT, Cell, NODE>& field = gatherFields_[fieldInd];
int fieldNum(fieldIds_[fieldInd]);
const vector<int>& elmtOffset =
globalIndexer_->getGIDFieldOffsets(blockId, fieldNum);
int numBases(elmtOffset.size());

// Loop over the basis functions and fill the fields.
for (int basis(0); basis < numBases; ++basis)
{
int offset(elmtOffset[basis]), lid(LIDs[offset]);
field(cell, basis) = (*x_)[lid];
} // end loop over the basis functions
} // end loop over the fields to be gathered
int offset(elmtOffset[basis]), lid(LIDs_h(cellLocalId, offset));
if (x_.is_null())
field_h(cell, basis) = (*xEvRoGed_)[lid];
else
field_h(cell, basis) = (*x_)[lid];
} // end loop over the basis functions
} // end loop over localCellIds
} // end if (x_.is_null()) or not

// Deal with the tangent fields.
if (hasTangentFields_)
{
// Loop over the cells in the workset.
for (int cell(0); cell < numCells; ++cell)
{
// Loop over the fields to be gathered.
for (int fieldInd(0); fieldInd < numFields; ++fieldInd)
{
MDField<ScalarT, Cell, NODE>& field = gatherFields_[fieldInd];
int fieldNum(fieldIds_[fieldInd]);
const vector<int>& elmtOffset =
globalIndexer_->getGIDFieldOffsets(blockId, fieldNum);
int numBases(elmtOffset.size());

// Loop over the basis functions.
for (int basis(0); basis < numBases; ++basis)
{
// Loop over the tangent fields and fill them in.
int numTangentFields(tangentFields_[fieldInd].size());
for (int i(0); i < numTangentFields; ++i)
field(cell, basis).fastAccessDx(i) =
tangentFields_[fieldInd][i](cell, basis).val();
} // end loop over the basis functions
} // end loop over the fields to be gathered
} // end loop over the cells in the workset
} // end if (hasTangentFields_)
// Deal with the tangent fields.
if (hasTangentFields_) {
// Loop over the tangent fields and fill them in.
int numTangentFields(tangentFields_[fieldInd].size());
for (int i(0); i < numTangentFields; ++i){
auto tf = Kokkos::create_mirror_view(tangentFields_[fieldInd][i].get_static_view());
Kokkos::deep_copy(tf, tangentFields_[fieldInd][i].get_static_view());
// Loop over the cells in the workset.
for (int cell(0); cell < numCells; ++cell) {
// Loop over the fields to be gathered.
int fieldNum(fieldIds_[fieldInd]);
const vector<int>& elmtOffset =
globalIndexer_->getGIDFieldOffsets(blockId, fieldNum);
int numBases(elmtOffset.size());

// Loop over the basis functions.
for (int basis(0); basis < numBases; ++basis){
field_h(cell, basis).fastAccessDx(i) =
tf(cell, basis).val();
} // end loop over the basis functions
} // end loop over the cells in the workset
} // end loop over numTangentFields
} // end if (hasTangentFields_)
Kokkos::deep_copy(field.get_static_view(), field_h);
} // end loop over the fields to be gathered
} // end of evaluateFields() (Tangent Specialization)

///////////////////////////////////////////////////////////////////////////////
Expand Down
2 changes: 2 additions & 0 deletions packages/panzer/disc-fe/src/evaluators/Panzer_GatherTangent_Epetra.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@
// ***********************************************************************
// @HEADER

#include "Kokkos_View_Fad.hpp"

#include "PanzerDiscFE_config.hpp"

#ifdef HAVE_PANZER_EXPLICIT_INSTANTIATION
Expand Down
25 changes: 14 additions & 11 deletions packages/panzer/disc-fe/src/evaluators/Panzer_GatherTangent_Epetra_impl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,15 +217,17 @@ evaluateFields(
// can be cheaper. However the lookup for LIDs may be more expensive!

// Gather operation for each cell in the workset.
for (int cell(0); cell < numCells; ++cell)
auto LIDs = globalIndexer_->getLIDs();
auto LIDs_h = Kokkos::create_mirror_view(LIDs);
Kokkos::deep_copy(LIDs_h, LIDs);
// Loop over the fields to be gathered.
for (int fieldIndex(0); fieldIndex < numFields; ++fieldIndex)
{
size_t cellLocalId(localCellIds[cell]);
auto LIDs = globalIndexer_->getElementLIDs(cellLocalId);

// Loop over the fields to be gathered.
for (int fieldIndex(0); fieldIndex < numFields; ++fieldIndex)
MDField<ScalarT, Cell, NODE>& field = gatherFields_[fieldIndex];
auto field_h = Kokkos::create_mirror_view(field.get_static_view());
for (int cell(0); cell < numCells; ++cell)
{
MDField<ScalarT, Cell, NODE>& field = gatherFields_[fieldIndex];
size_t cellLocalId(localCellIds[cell]);
int fieldNum(fieldIds_[fieldIndex]);
const vector<int>& elmtOffset =
globalIndexer_->getGIDFieldOffsets(blockId, fieldNum);
Expand All @@ -234,11 +236,12 @@ evaluateFields(
// Loop over the basis functions and fill the fields.
for (int basis(0); basis < numBases; ++basis)
{
int offset(elmtOffset[basis]), lid(LIDs[offset]);
field(cell, basis) = (*dxdpEvRoGed_)[lid];
int offset(elmtOffset[basis]), lid(LIDs_h(cellLocalId, offset));
field_h(cell, basis) = (*dxdpEvRoGed_)[lid];
} // end loop over the basis functions
} // end loop over the fields to be gathered
} // end loop over the cells in the workset
} // end loop over the cells in the workset
Kokkos::deep_copy(field.get_static_view(), field_h);
} // end loop over the fields to be gathered
} // end of evaluateFields()

#endif // __Panzer_GatherTangent_Epetra_impl_hpp__

0 comments on commit 0e60079

Please sign in to comment.