Panzer: fix periodic bc matcher for HDiv gid passing 32bit int limit

packages/panzer/adapters-stk/src/stk_interface/Panzer_STK_PeriodicBC_Matcher.cpp

@@ -188,7 +188,7 @@ getLocalSideIdsAndCoords(const STK_Interface & mesh,
 
    stk::mesh::EntityRank rank;
    const STK_Interface::VectorFieldType * field = 0;
-   unsigned int offset = 0;
+   stk::mesh::EntityId offset = 0;
    if(type_ == "coord"){
      rank = mesh.getNodeRank();
      field = & mesh.getCoordinatesField();
@@ -270,7 +270,7 @@ getSideIdsAndCoords(const STK_Interface & mesh,
 
    std::vector<std::size_t> & local_side_ids = *sidePair.first;
    std::vector<Teuchos::Tuple<double,3> > & local_side_coords = *sidePair.second;
-   int nodeCount = local_side_ids.size();
+   std::size_t nodeCount = local_side_ids.size();
 
    // build local Tpetra objects
    auto computeInternally = Teuchos::OrdinalTraits<Tpetra::global_size_t>::invalid();
@@ -294,7 +294,7 @@ getSideIdsAndCoords(const STK_Interface & mesh,
    // (these are "distributed" or "dist" objects)
    //////////////////////////////////////////////////////////////
 
-   int dist_nodeCount = idMap_->getGlobalNumElements();
+   std::size_t dist_nodeCount = idMap_->getGlobalNumElements();
 
    // build global Tpetra objects
    RCP<Map> distMap_ = rcp(new Map(dist_nodeCount,0,comm,Tpetra::LocallyReplicated));

packages/panzer/adapters-stk/test/periodic_bcs/CMakeLists.txt

@@ -25,3 +25,17 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST(
 #   COMM serial mpi
 #   NUM_MPI_PROCS 1
 #   )
+
+TRIBITS_ADD_EXECUTABLE_AND_TEST(
+  periodic_32bit_int_limit
+  SOURCES periodic_32bit_int_limit.cpp ${UNIT_TEST_DRIVER}
+  COMM serial mpi
+  NUM_MPI_PROCS 2
+  )
+
+TRIBITS_COPY_FILES_TO_BINARY_DIR(periodic_32bit_int_limit_files
+  SOURCE_FILES periodic_32bit_int_limit.jou periodic_32bit_int_limit.exo README.txt
+  SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/"
+  DEST_DIR "${CMAKE_CURRENT_BINARY_DIR}/"
+  EXEDEPS periodic_32bit_int_limit
+  )

packages/panzer/adapters-stk/test/periodic_bcs/README.txt

@@ -0,0 +1,18 @@
+
+For the test "periodic_32bit_int_limit":
+
+The exodus mesh must be generated using cubit with the journal file
+peridoic_32bit_int_limit.jou. Then the gids must be shifted above the
+32bit int limit using the SEACAS "grepos" tool/executable. Run grepos
+with:
+
+grepos -64 periodic_32bit_int_limit_UNSHIFTED.exo periodic_32bit_int_limit.exo
+
+Once in grepos do the following to shift the
+indices:
+
+increment nodemap 7000000000
+increment elemmap 7000000000
+exit
+
+On exit, the new file will be written with the shifted indices.

packages/panzer/adapters-stk/test/periodic_bcs/periodic_32bit_int_limit.cpp

@@ -0,0 +1,186 @@
+#include "Teuchos_ConfigDefs.hpp"
+#include "Teuchos_UnitTestHarness.hpp"
+#include "Teuchos_GlobalMPISession.hpp"
+#include "Teuchos_RCP.hpp"
+#include "Teuchos_ParameterList.hpp"
+
+#include "Kokkos_Core.hpp"
+
+#include "Intrepid2_Basis.hpp"
+#include "Intrepid2_HGRAD_HEX_C1_FEM.hpp"
+#include "Intrepid2_HCURL_HEX_I1_FEM.hpp"
+#include "Intrepid2_HDIV_HEX_I1_FEM.hpp"
+#include "Intrepid2_HVOL_C0_FEM.hpp"
+
+#include "Panzer_STK_MeshFactory.hpp"
+#include "Panzer_STK_Interface.hpp"
+#include "Panzer_STK_ExodusReaderFactory.hpp"
+#include "Panzer_STKConnManager.hpp"
+#include "Panzer_IntrepidFieldPattern.hpp"
+#include "Panzer_DOFManager.hpp"
+
+using namespace std;
+using Teuchos::RCP;
+using Teuchos::rcp;
+
+TEUCHOS_UNIT_TEST(periodic_bcs, 32_bit_int_limit)
+{
+  // This test requires 64 bit integers for the GIDs of Tpetra
+  // objects. If the GIDs are not 64 bit then disable the test.
+  if ( not( (std::numeric_limits<panzer::GlobalOrdinal>::max() >= std::numeric_limits<long long int>::max()) || 
+            (std::numeric_limits<panzer::GlobalOrdinal>::max() >= std::numeric_limits<unsigned long long int>::max()) ) )
+    {
+      std::cout << "\nWARNING: Panzer_AdaptersSTK::periodic_32bit_int_limit test is disabled since Tpetra"
+                << " was not configured with a 64 bit GID type.\n\n";
+    return;
+  }
+
+  // To run in parallel we need to set an ioss property to split
+  // exodus file across mpi ranks.
+  setenv("IOSS_PROPERTIES", "DECOMPOSITION_METHOD=rib", 1);
+
+  Teuchos::RCP<Teuchos::MpiComm<int>> Comm = Teuchos::rcp( new Teuchos::MpiComm<int>(MPI_COMM_WORLD) );
+  Kokkos::initialize();
+
+  using topo_RCP = Teuchos::RCP<const shards::CellTopology>;
+  using basis_RCP = Teuchos::RCP<Intrepid2::Basis<PHX::Device::execution_space,double,double>>;
+
+  {
+    // ==========================================================
+    // Create a mesh from the file defined by the user
+    // ==========================================================
+    Teuchos::RCP<panzer_stk::STK_MeshFactory> mesh_factory = Teuchos::rcp(new panzer_stk::STK_ExodusReaderFactory());
+
+    std::string input_file_name = "periodic_32bit_int_limit.exo";
+    RCP<Teuchos::ParameterList> pl = rcp(new Teuchos::ParameterList);
+    pl->set("File Name",input_file_name);
+
+    // Optional: include periodic BCs
+    if (true) {
+      RCP<Teuchos::ParameterList> pbc = rcp(new Teuchos::ParameterList);
+      pbc->set("Count",2);
+      pbc->set("Periodic Condition 1","xz-all 1e-12,3D: top;bottom");
+      pbc->set("Periodic Condition 2","yz-all 1e-12,3D: left;right");
+      pl->sublist("Periodic BCs").setParameters( *pbc );
+    }
+
+    mesh_factory->setParameterList(pl);
+    Teuchos::RCP<panzer_stk::STK_Interface> mesh = mesh_factory->buildUncommitedMesh(*(Comm->getRawMpiComm()));
+
+    mesh_factory->completeMeshConstruction(*mesh,*(Comm->getRawMpiComm()));
+
+    bool is_print_process = false;
+    if (Comm->getRank() == 0)
+      is_print_process = true;
+
+    if (is_print_process)
+      mesh->printMetaData(std::cout);
+
+    vector<string> blocknames;
+    mesh->getElementBlockNames(blocknames);
+
+    // ==========================================================
+    // Define the connectivity manager
+    // ==========================================================
+
+    Teuchos::RCP<panzer::ConnManager> conn = Teuchos::rcp(new panzer_stk::STKConnManager(mesh));
+
+    // ==========================================================
+    // Test out a few DOF managers
+    // ==========================================================
+
+    {
+      if (is_print_process) {
+        out << "================================================" << std::endl;
+        out << " *** Testing one HGRAD variable *** " << std::endl;
+      }
+      Teuchos::RCP<panzer::DOFManager> DOF = Teuchos::rcp(new panzer::DOFManager());
+      DOF->setConnManager(conn,*(Comm->getRawMpiComm()));
+      DOF->setOrientationsRequired(true);
+
+      for (size_t b=0; b<blocknames.size(); b++) {
+        topo_RCP cellTopo = mesh->getCellTopology(blocknames[b]);
+        basis_RCP basis = Teuchos::rcp(new Intrepid2::Basis_HGRAD_HEX_C1_FEM<PHX::Device::execution_space,double,double>() );
+        Teuchos::RCP<const panzer::Intrepid2FieldPattern> Pattern = Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis));
+        DOF->addField(blocknames[b], "T", Pattern, panzer::FieldType::CG);
+      }
+
+      DOF->buildGlobalUnknowns();
+      if (is_print_process) {
+        DOF->printFieldInformation(out);
+        out << "================================================" << std::endl << std::endl;
+      }
+    }
+
+    {
+      if (is_print_process) {
+        out << "================================================" << std::endl;
+        out << " *** Testing one HCURL variable *** " << std::endl;
+      }
+      Teuchos::RCP<panzer::DOFManager> DOF = Teuchos::rcp(new panzer::DOFManager());
+      DOF->setConnManager(conn,*(Comm->getRawMpiComm()));
+      DOF->setOrientationsRequired(true);
+
+      for (size_t b=0; b<blocknames.size(); b++) {
+        topo_RCP cellTopo = mesh->getCellTopology(blocknames[b]);
+        basis_RCP basis = Teuchos::rcp(new Intrepid2::Basis_HCURL_HEX_I1_FEM<PHX::Device::execution_space,double,double>() );
+        Teuchos::RCP<const panzer::Intrepid2FieldPattern> Pattern = Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis));
+        DOF->addField(blocknames[b], "E", Pattern, panzer::FieldType::CG);
+      }
+
+      DOF->buildGlobalUnknowns();
+      if (is_print_process) {
+        DOF->printFieldInformation(out);
+        out << "================================================" << std::endl << std::endl;
+      }
+    }
+
+    {
+      if (is_print_process) {
+        out << "================================================" << std::endl;
+        out << " *** Testing one HDIV variable *** " << std::endl;
+      }
+      Teuchos::RCP<panzer::DOFManager> DOF = Teuchos::rcp(new panzer::DOFManager());
+      DOF->setConnManager(conn,*(Comm->getRawMpiComm()));
+      DOF->setOrientationsRequired(true);
+
+      for (size_t b=0; b<blocknames.size(); b++) {
+        topo_RCP cellTopo = mesh->getCellTopology(blocknames[b]);
+        basis_RCP basis = Teuchos::rcp(new Intrepid2::Basis_HDIV_HEX_I1_FEM<PHX::Device::execution_space,double,double>() );
+        Teuchos::RCP<const panzer::Intrepid2FieldPattern> Pattern = Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis));
+        DOF->addField(blocknames[b], "B", Pattern, panzer::FieldType::CG);
+      }
+
+      DOF->buildGlobalUnknowns();
+      if (is_print_process) {
+        DOF->printFieldInformation(out);
+        out << "================================================" << std::endl << std::endl;
+      }
+    }
+
+    {
+      if (is_print_process) {
+        out << "================================================" << std::endl;
+        out << " *** Testing one HVOL variable *** " << std::endl;
+      }
+      Teuchos::RCP<panzer::DOFManager> DOF = Teuchos::rcp(new panzer::DOFManager());
+      DOF->setConnManager(conn,*(Comm->getRawMpiComm()));
+      DOF->setOrientationsRequired(true);
+
+      for (size_t b=0; b<blocknames.size(); b++) {
+        topo_RCP cellTopo = mesh->getCellTopology(blocknames[b]);
+        basis_RCP basis = Teuchos::rcp(new Intrepid2::Basis_HVOL_C0_FEM<PHX::Device::execution_space,double,double>(*cellTopo) );
+        Teuchos::RCP<const panzer::Intrepid2FieldPattern> Pattern = Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis));
+        DOF->addField(blocknames[b], "p", Pattern, panzer::FieldType::CG);
+      }
+
+      DOF->buildGlobalUnknowns();
+      if (is_print_process) {
+        DOF->printFieldInformation(out);
+        out << "================================================" << std::endl << std::endl;
+      }
+    }
+  }
+
+  Kokkos::finalize();
+}

packages/panzer/adapters-stk/test/periodic_bcs/periodic_32bit_int_limit.jou

@@ -0,0 +1,42 @@
+reset
+reset aprepro
+
+#{x1 = -1.0}
+#{x2 = 1.0}
+#{y1 = -1.0}
+#{y2 = 1.0}
+#{z1 = -1.0}
+#{z2 = 1.0}
+#{unitscale = 1e-6}
+#{intnum = 2}
+
+create brick x {x2-x1} y {y2-y1} z {z2-z1}
+
+move vol 1 midpoint location x {(x1+x2)/2} y {(y1+y2)/2} z {(z1+z2)/2}
+
+curve 1,2 interval {intnum}
+curve 9 interval {intnum}
+
+mesh curve 1,2,9
+
+mesh vol 1
+
+block 1 add vol 1
+block 1 name 'air'
+
+block all element type hex
+
+sideset 1  add   surface 5
+sideset 1  name  'top'
+sideset 2  add   surface 3
+sideset 2  name  'bottom'
+sideset 3  add   surface 6
+sideset 3  name  'left'
+sideset 4  add   surface 4
+sideset 4  name  'right'
+
+volume all scale {unitscale}
+
+set exodus netcdf4 off
+set large exodus on
+export mesh "periodic_32bit_int_limit_UNSHIFTED.exo" overwrite