Merge 'trilinos/Trilinos:develop' (c610d9d) into 'tcad-charon/Trilino…

…s:develop' (d17489d).

* trilinos-develop: (29 commits)
  Tpetra: Correcting unecessary extraction of remotes for A matrix in Jacobi
  Tpetra: MMM Modifications to avoid remotemap construction in serial
  Ifpack2: pass correct "symmetric" flag to MTSGS setup
  MueLu: Adding timer granularity to SaP
  Fix const_cast in Tpetra, Amesos and Ifpack2
  SEACAS: Remove locale setting
  Teuchos utils: fix another stacked timer plotting bug
  tpetra:  In trilinos#6598, @mhoemmen recommended this change of offset
  Tpetra::CrsMatrix: Add Kokkos kernel labels; expose debug code
  Tpetra::CrsMatrix: Remove values2D_
  Tpetra::CrsGraph: Remove gblInds2D_
  Tpetra::CrsGraph: Remove lclInds2D_
  Tpetra::CrsMatrix: Remove unused method allocateValues2D
  Tpetra: Use verbosePrintCountThreshold in copyOffsets
  Tpetra::Details::Behavior: Add longRowMinNumEntries
  Tpetra::Details::Behavior: Factor out size_t reading
  SEACAS: go back to lib:fmt 6.0.0 until fix issue on vortex xl/cuda build
  Disable Teko_testdriver_tpetra_MPI_4 in all atdm 'waterman' builds (trilinos#6463)
  zoltan2:  add missing include file for non-ETI builds
  Tpetra: Missed ifdef guard
  ...

cmake/std/atdm/waterman/tweaks/Tweaks.cmake

@@ -6,6 +6,9 @@
 ATDM_SET_ENABLE(PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2_DISABLE ON)
 ATDM_SET_ENABLE(PanzerAdaptersIOSS_tIOSSConnManager3_MPI_3_DISABLE ON)
 
+# Disable randomly timing out test in all 'waterman' builds (#6463)
+ATDM_SET_ENABLE(Teko_testdriver_tpetra_MPI_4_DISABLE ON)
+
 IF (Trilinos_ENABLE_DEBUG)
 
   # STEQR() test fails on IBM Power systems with current TPL setup (#2410, #6166)

cmake/std/sems/PullRequestGCC4.9.3TestingEnv.sh

@@ -17,22 +17,8 @@ module load sems-netcdf/4.4.1/exo_parallel
 module load sems-parmetis/4.0.3/parallel
 module load sems-scotch/6.0.3/nopthread_64bit_parallel
 module load sems-superlu/4.3/base
-
-# Load the SEMS CMake Module
-# - One of the SEMS modules will load CMake 3.4.x also,
-#   so this will pull in the SEMS cmake 3.10.3 version
-#   for Trilinos compatibility.
-module load sems-cmake/3.10.3
-
-# Using CMake and Ninja modules from the ATDM project space.
-# SEMS does not yet supply a recent enough version of CMake
-# for the single configure/build/test capability. We are also
-# using a custom version of Ninja (with Fortran support not
-# available in main-line Ninja) to significantly speed up
-# compile and link times.
-module load atdm-env
-module load atdm-cmake/3.11.1
-module load atdm-ninja_fortran/1.7.2
+module load sems-cmake/3.12.2
+module load sems-ninja_fortran/1.8.2
 
 # add the OpenMP environment variable we need
 export OMP_NUM_THREADS=2

cmake/std/sems/PullRequestGCC4.9.3TestingEnvSERIAL.sh

@@ -16,22 +16,8 @@ module load sems-netcdf/4.4.1/exo
 module load sems-metis/5.1.0/base
 # module load sems-scotch/6.0.3/nopthread_64bit_parallel
 module load sems-superlu/4.3/base
-
-# Load the SEMS CMake Module
-# - One of the SEMS modules will load CMake 3.4.x also,
-#   so this will pull in the SEMS cmake 3.10.3 version
-#   for Trilinos compatibility.
-module load sems-cmake/3.10.3
-
-# Using CMake and Ninja modules from the ATDM project space.
-# SEMS does not yet supply a recent enough version of CMake
-# for the single configure/build/test capability. We are also
-# using a custom version of Ninja (with Fortran support not
-# available in main-line Ninja) to significantly speed up
-# compile and link times.
-module load atdm-env
-module load atdm-cmake/3.11.1
-module load atdm-ninja_fortran/1.7.2
+module load sems-cmake/3.12.2
+module load sems-ninja_fortran/1.8.2
 
 # add the OpenMP environment variable we need
 export OMP_NUM_THREADS=2

cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh

@@ -17,21 +17,8 @@ module load sems-netcdf/4.4.1/exo_parallel
 module load sems-parmetis/4.0.3/parallel
 module load sems-scotch/6.0.3/nopthread_64bit_parallel
 module load sems-superlu/4.3/base
-
-# Load the SEMS CMake Module
-# - One of the SEMS modules will load CMake 3.4.x also,
-#   so this will pull in the SEMS cmake 3.10.3 version
-#   for Trilinos compatibility.
-module load sems-cmake/3.10.3
-
-# Using CMake and Ninja modules from the ATDM project space.
-# SEMS does not yet supply a recent enough version of CMake
-# for the single configure/build/test capability. We are also
-# using a custom version of Ninja (with Fortran support not
-# available in main-line Ninja) to significantly speed up
-# compile and link times.
-module load atdm-env
-module load atdm-ninja_fortran/1.7.2
+module load sems-cmake/3.12.2
+module load sems-ninja_fortran/1.8.2
 
 # add the OpenMP environment variable we need
 export OMP_NUM_THREADS=2

cmake/std/sems/PullRequestGCC7.3.0TestingEnv.sh

@@ -17,22 +17,8 @@ module load sems-netcdf/4.4.1/exo_parallel
 module load sems-parmetis/4.0.3/parallel
 module load sems-scotch/6.0.3/nopthread_64bit_parallel
 module load sems-superlu/4.3/base
-
-# Load the SEMS CMake Module
-# - One of the SEMS modules will load CMake 3.4.x also,
-#   so this will pull in the SEMS cmake 3.10.3 version
-#   for Trilinos compatibility.
-module load sems-cmake/3.10.3
-
-# Using CMake and Ninja modules from the ATDM project space.
-# SEMS does not yet supply a recent enough version of CMake
-# for the single configure/build/test capability. We are also
-# using a custom version of Ninja (with Fortran support not
-# available in main-line Ninja) to significantly speed up
-# compile and link times.
-module load atdm-env
-module load atdm-cmake/3.11.1
-module load atdm-ninja_fortran/1.7.2
+module load sems-cmake/3.12.2
+module load sems-ninja_fortran/1.8.2
 
 # add the OpenMP environment variable we need
 export OMP_NUM_THREADS=2

cmake/std/sems/PullRequestIntel17.0.1TestingEnv.sh

@@ -23,22 +23,8 @@ module load sems-netcdf/4.4.1/exo_parallel
 module load sems-parmetis/4.0.3/parallel
 module load sems-scotch/6.0.3/nopthread_64bit_parallel
 module load sems-superlu/4.3/base
-
-# Load the SEMS CMake Module
-# - One of the SEMS modules will load CMake 3.4.x also,
-#   so this will pull in the SEMS cmake 3.10.3 version
-#   for Trilinos compatibility.
-module load sems-cmake/3.10.3
-
-# Using CMake and Ninja modules from the ATDM project space.
-# SEMS does not yet supply a recent enough version of CMake
-# for the single configure/build/test capability. We are also
-# using a custom version of Ninja (with Fortran support not
-# available in main-line Ninja) to significantly speed up
-# compile and link times.
-module load atdm-env
-module load atdm-cmake/3.11.1
-module load atdm-ninja_fortran/1.7.2
+module load sems-cmake/3.12.2
+module load sems-ninja_fortran/1.8.2
 
 # add the OpenMP environment variable we need
 export OMP_NUM_THREADS=2

packages/amesos2/src/Amesos2_SolverCore_def.hpp

@@ -205,7 +205,7 @@ SolverCore<ConcreteSolver,Matrix,Vector>::solve(const Teuchos::Ptr<Vector> X,
   if( !status_.numericFactorizationDone() ){
     // This casting-away of constness is probably OK because this
     // function is meant to be "logically const"
-    const_cast<type*>(this)->numericFactorization();
+    const_cast<type&>(*this).numericFactorization();
   }
 
   static_cast<const solver_type*>(this)->solve_impl(Teuchos::outArg(*x), Teuchos::ptrInArg(*b));

packages/ifpack2/src/Ifpack2_Relaxation_def.hpp

@@ -686,9 +686,6 @@ void Relaxation<MatrixType>::initialize ()
       }
       local_matrix_type kcsr = crsMat->getLocalMatrix ();
 
-      bool is_symmetric = (PrecType_ == Ifpack2::Details::MTSGS);
-      is_symmetric = is_symmetric || is_matrix_structurally_symmetric_;
-
       using KokkosSparse::Experimental::gauss_seidel_symbolic;
       gauss_seidel_symbolic<mt_kernel_handle_type,
                             lno_row_view_t,
@@ -697,7 +694,7 @@ void Relaxation<MatrixType>::initialize ()
                                                  A_->getNodeNumCols (),
                                                  kcsr.graph.row_map,
                                                  kcsr.graph.entries,
-                                                 is_symmetric);
+                                                 is_matrix_structurally_symmetric_);
     }
   } // timing of initialize stops here
 
@@ -1300,7 +1297,6 @@ void Relaxation<MatrixType>::compute ()
          "when the input matrix is a Tpetra::CrsMatrix.");
       local_matrix_type kcsr = crsMat->getLocalMatrix ();
 
-      const bool is_symmetric = (PrecType_ == Ifpack2::Details::MTSGS);
       auto diagView_2d = Diagonal_->getLocalViewDevice ();
       auto diagView_1d = Kokkos::subview (diagView_2d, Kokkos::ALL (), 0);
       using KokkosSparse::Experimental::gauss_seidel_numeric;
@@ -1314,7 +1310,7 @@ void Relaxation<MatrixType>::compute ()
                                                    kcsr.graph.entries,
                                                    kcsr.values,
                                                    diagView_1d,
-                                                   is_symmetric);
+                                                   is_matrix_structurally_symmetric_);
     }
   } // end TimeMonitor scope
 
@@ -1543,7 +1539,7 @@ ApplyInverseGS (const Tpetra::MultiVector<scalar_type,local_ordinal_type,global_
   const crs_matrix_type* crsMat =
     dynamic_cast<const crs_matrix_type*> (A_.getRawPtr ());
   if (blockCrsMat != nullptr)  {
-    const_cast<this_type*> (this)->ApplyInverseGS_BlockCrsMatrix (*blockCrsMat, X, Y);
+    const_cast<this_type&> (*this).ApplyInverseGS_BlockCrsMatrix (*blockCrsMat, X, Y);
   }
   else if (crsMat != nullptr) {
     ApplyInverseGS_CrsMatrix (*crsMat, X, Y);
@@ -2241,7 +2237,7 @@ ApplyInverseSGS (const Tpetra::MultiVector<scalar_type,local_ordinal_type,global
   const crs_matrix_type* crsMat =
     dynamic_cast<const crs_matrix_type*> (A_.getRawPtr ());
   if (blockCrsMat != nullptr)  {
-    const_cast<this_type*> (this)->ApplyInverseSGS_BlockCrsMatrix(*blockCrsMat, X, Y);
+    const_cast<this_type&> (*this).ApplyInverseSGS_BlockCrsMatrix(*blockCrsMat, X, Y);
   }
   else if (crsMat != nullptr) {
     ApplyInverseSGS_CrsMatrix (*crsMat, X, Y);

packages/intrepid2/unit-test/Discretization/Basis/HierarchicalBases/AnalyticPolynomialsMatchTests.cpp

@@ -558,7 +558,11 @@ namespace
           OutputScalar derivedValue  =  derivedOutputViewHost(fieldOrdinalDerived, pointOrdinal);
 
           bool valuesMatch = true;
-          TEUCHOS_TEST_FLOATING_EQUALITY(standardValue, derivedValue, tol, out, valuesMatch);
+          bool valuesAreBothSmall = valuesAreSmall(standardValue, derivedValue, tol);
+          if (!valuesAreBothSmall)
+          {
+            TEUCHOS_TEST_FLOATING_EQUALITY(standardValue, derivedValue, tol, out, valuesMatch);
+          }
 
           if (!valuesMatch)
           {
@@ -597,7 +601,11 @@ namespace
             OutputScalar standardValue = standardOutputViewHost(fieldOrdinalStandard,pointOrdinal,d);
             OutputScalar derivedValue  =  derivedOutputViewHost(fieldOrdinalDerived, pointOrdinal,d);
 
-            TEUCHOS_TEST_FLOATING_EQUALITY(standardValue, derivedValue, tol, out, valuesMatch);
+            bool valuesAreBothSmall = valuesAreSmall(standardValue, derivedValue, tol);
+            if (!valuesAreBothSmall)
+            {
+              TEUCHOS_TEST_FLOATING_EQUALITY(standardValue, derivedValue, tol, out, valuesMatch);
+            }
           }
 
           if (!valuesMatch)

packages/intrepid2/unit-test/Shared/Polylib/LegendreJacobiPolynomials/IntegratedJacobiTests.cpp

@@ -138,13 +138,17 @@ namespace
 
           for (int i=0; i<=polyOrder; i++)
           {
-            if (! approximatelyEqual(secondPathIntegratedJacobiView_dt_host(i), integratedJacobiView_dt_host(i), tol) )
+            bool valuesAreBothSmall = valuesAreSmall(secondPathIntegratedJacobiView_dt_host(i), integratedJacobiView_dt_host(i), tol);
+            if (!valuesAreBothSmall)
             {
-              out << "for polyOrder " << i << ", alpha = " << alpha << ", x = " << x << ", t = " << t << ": ";
-              out << secondPathIntegratedJacobiView_dt_host(i) << " != " << integratedJacobiView_dt_host(i);
-              out << " (diff = " << abs(secondPathIntegratedJacobiView_dt_host(i) - integratedJacobiView_dt_host(i));
-              out << "; tol = " << tol << ")\n";
-              success = false;
+              if (! approximatelyEqual(secondPathIntegratedJacobiView_dt_host(i), integratedJacobiView_dt_host(i), tol) )
+              {
+                out << "for polyOrder " << i << ", alpha = " << alpha << ", x = " << x << ", t = " << t << ": ";
+                out << secondPathIntegratedJacobiView_dt_host(i) << " != " << integratedJacobiView_dt_host(i);
+                out << " (diff = " << abs(secondPathIntegratedJacobiView_dt_host(i) - integratedJacobiView_dt_host(i));
+                out << "; tol = " << tol << ")\n";
+                success = false;
+              }
             }
           }
         }

packages/intrepid2/unit-test/Shared/Polylib/LegendreJacobiPolynomials/IntegratedLegendreTests.cpp

@@ -89,13 +89,17 @@ namespace
 
         for (int i=0; i<=polyOrder; i++)
         {
-          if (! approximatelyEqual(integrated_legendre_values_second_path_host(i), integrated_legendre_values_host(i), tol) )
+          bool valuesAreBothSmall = valuesAreSmall(integrated_legendre_values_second_path_host(i), integrated_legendre_values_host(i), tol);
+          if (! valuesAreBothSmall)
           {
-            out << "for polyOrder " << i << ", x = " << x << ", t = " << t << ": ";
-            out << integrated_legendre_values_second_path_host(i) << " != " << integrated_legendre_values_host(i);
-            out << " (diff = " << abs(integrated_legendre_values_second_path_host(i) - integrated_legendre_values_host(i));
-            out << "; tol = " << tol << ")\n";
-            success = false;
+            if (! approximatelyEqual(integrated_legendre_values_second_path_host(i), integrated_legendre_values_host(i), tol) )
+            {
+              out << "for polyOrder " << i << ", x = " << x << ", t = " << t << ": ";
+              out << integrated_legendre_values_second_path_host(i) << " != " << integrated_legendre_values_host(i);
+              out << " (diff = " << abs(integrated_legendre_values_second_path_host(i) - integrated_legendre_values_host(i));
+              out << "; tol = " << tol << ")\n";
+              success = false;
+            }
           }
         }
       }

packages/intrepid2/unit-test/Shared/Polylib/LegendreJacobiPolynomials/LegendreTests.cpp

@@ -107,9 +107,9 @@ namespace
         auto jacobiValueHost = getHostCopy(jacobiValue);
 
         expectedValue = scaleFactor * jacobiValueHost(0);
-//        bool valuesAreBothSmall = valuesAreSmall(expectedValue, legendreDerivativesHost(i), tol);
-//        if (! valuesAreBothSmall)
-//        {
+        bool valuesAreBothSmall = valuesAreSmall(expectedValue, legendreDerivativesHost(i), tol);
+        if (! valuesAreBothSmall)
+        {
           if (! approximatelyEqual(expectedValue, legendreDerivativesHost(i), tol) )
           {
             double diff = abs(expectedValue - legendreDerivativesHost(i));
@@ -118,7 +118,7 @@ namespace
             out << "; actual was " << legendreDerivativesHost(i) << " (diff: " << diff << ") for derivative " << derivativeOrder << " of i = " << i;
             out << " at x = " << x << std::endl;
           }
-//        }
+        }
       }
     }
   }

packages/muelu/src/Transfers/Smoothed-Aggregation/MueLu_SaPFactory_kokkos_def.hpp

@@ -173,13 +173,19 @@ namespace MueLu {
 
       {
         SubFactoryMonitor m2(*this, "Fused (I-omega*D^{-1} A)*Ptent", coarseLevel);
-        RCP<Vector> invDiag = Utilities_kokkos::GetMatrixDiagonalInverse(*A);
-
+        RCP<Vector> invDiag;
+        {
+          SubFactoryMonitor m3(*this, "Diagonal Extraction", coarseLevel);
+          invDiag = Utilities_kokkos::GetMatrixDiagonalInverse(*A);
+        }
         SC omega = dampingFactor / lambdaMax;
         TEUCHOS_TEST_FOR_EXCEPTION(!std::isfinite(Teuchos::ScalarTraits<SC>::magnitude(omega)), Exceptions::RuntimeError, "Prolongator damping factor needs to be finite.");
 
-        // finalP = Ptent + (I - \omega D^{-1}A) Ptent
-        finalP = Xpetra::IteratorOps<Scalar, LocalOrdinal, GlobalOrdinal, Node>::Jacobi(omega, *invDiag, *A, *Ptent, finalP, GetOStream(Statistics2), std::string("MueLu::SaP-") + toString(coarseLevel.GetLevelID()), APparams);
+        {
+          SubFactoryMonitor m3(*this, "Xpetra::IteratorOps::Jacobi", coarseLevel);
+          // finalP = Ptent + (I - \omega D^{-1}A) Ptent
+          finalP = Xpetra::IteratorOps<Scalar, LocalOrdinal, GlobalOrdinal, Node>::Jacobi(omega, *invDiag, *A, *Ptent, finalP, GetOStream(Statistics2), std::string("MueLu::SaP-") + toString(coarseLevel.GetLevelID()), APparams);
+        }
       }
 
     } else {

packages/muelu/src/Utils/MueLu_UtilitiesBase_decl.hpp

@@ -164,6 +164,7 @@ namespace MueLu {
     NOTE -- it's assumed that A has been fillComplete'd.
     */
     static RCP<Vector> GetMatrixDiagonalInverse(const Matrix& A, Magnitude tol = Teuchos::ScalarTraits<Scalar>::eps()*100) {
+      Teuchos::TimeMonitor MM = *Teuchos::TimeMonitor::getNewTimer("UtilitiesBase::GetMatrixDiagonalInverse");
 
       RCP<const Matrix> rcpA = Teuchos::rcpFromRef(A);
 

packages/muelu/src/Utils/MueLu_Utilities_kokkos_def.hpp

@@ -138,6 +138,7 @@ namespace MueLu {
   Teuchos::RCP<Xpetra::Vector<Scalar,LocalOrdinal,GlobalOrdinal,Node> >
   Utilities_kokkos<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
   GetMatrixDiagonalInverse(const Matrix& A, Magnitude tol) {
+    Teuchos::TimeMonitor MM = *Teuchos::TimeMonitor::getNewTimer("Utilities_kokkos::GetMatrixDiagonalInverse");
     // Some useful type definitions
     using local_matrix_type = typename Matrix::local_matrix_type;
     using local_graph_type  = typename local_matrix_type::staticcrsgraph_type;
@@ -154,14 +155,13 @@ namespace MueLu {
 
     // Get/Create distributed objects
     RCP<const Map> rowMap = A.getRowMap();
-    RCP<Vector> diag      = VectorFactory::Build(rowMap);
+    RCP<Vector> diag      = VectorFactory::Build(rowMap,false);
 
     // Now generate local objects
     local_matrix_type localMatrix = A.getLocalMatrix();
-    local_graph_type  localGraph  = localMatrix.graph;
     auto diagVals = diag->template getLocalView<memory_space>();
 
-    ordinal_type numRows = localGraph.numRows();
+    ordinal_type numRows = localMatrix.graph.numRows();
 
     // Note: 2019-11-21, LBV
     // This could be implemented with a TeamPolicy over the rows
@@ -180,6 +180,7 @@ namespace MueLu {
                                } else {
                                  diagVals(rowIdx, 0) = KAT::zero();
                                }
+                               break;
                              }
                            }
 

packages/seacas/libraries/ioss/src/Ioss_Initializer.C

@@ -39,8 +39,6 @@
 
 Ioss::Initializer::Initializer()
 {
-  std::locale::global(std::locale(""));
-
   // List all storage types here with a call to their factory method.
   // This is Used to get the linker to pull in all needed libraries.
   Ioss::Sphere::factory();