Merge 'trilinos/Trilinos:develop' (8011ce2) into 'tcad-charon/Trilino…

…s:develop' (3e6eae2).

* trilinos-develop:
  Ctest: lightsaber update
  NOX: fix some cuda 11 compiler warnings
  PIRO: fix unit test model evaluator for CUDA UVM off builds
  Stokhos:  Only use new cuSPARSE SpMv/SpMM functions on Cuda 11 or greater
  Stokhos:  Fix unused variable warning
  Stokhos:  Remove debugging statements that were accidentally committed
  Stokhos:  Fix usage of cuSPARSE for newer CUDA toolkits.
  Stokhos:  Get 2-D diffusion examples working without UVM
  Stokhos:  Fix Belos-Tpetra PCE adapter with multiple MPI procs
  Stokhos:  Fix more tests when UVM is disabled.
  WIP on fixing Stokhos to run without UVM

cmake/ctest/drivers/lightsaber/cron_driver.sh

@@ -27,6 +27,7 @@ export https_proxy="https://wwwproxy.sandia.gov:80"
 export TDD_FORCE_CMAKE_INSTALL=1
 export TDD_DEBUG_VERBOSE=1
 
+. /etc/profile
 source ~/.bashrc
 
 

packages/nox/src/NOX_Multiphysics_Solver_Generic.H

@@ -127,14 +127,16 @@ public:
     const Teuchos::RCP<NOX::StatusTest::Generic>& tests,
     const Teuchos::RCP<Teuchos::ParameterList>& params) = 0;
 
-  //! reset methods inherited from NOX::Solver::Generic and needed here to avoid hiding this overloaded virtual method
+  //! reset methods inherited from NOX::Solver::Generic are needed here to avoid hiding this overloaded virtual method
+
+  virtual void reset() = 0;
 
   virtual void
   reset(const NOX::Abstract::Vector& initialGuess) = 0;
 
   virtual void
   reset(const NOX::Abstract::Vector& initialGuess,
-    const Teuchos::RCP<NOX::StatusTest::Generic>& tests) = 0;
+        const Teuchos::RCP<NOX::StatusTest::Generic>& tests) = 0;
 
 };
 } // namespace Solver

packages/nox/src/NOX_Multiphysics_Solver_Manager.C

@@ -153,7 +153,7 @@ void NOX::Multiphysics::Solver::Manager::deprecated(const std::string& oldName,
        << std::endl;
 }
 
-NOX::StatusTest::StatusType NOX::Multiphysics::Solver::Manager::getStatus()
+NOX::StatusTest::StatusType NOX::Multiphysics::Solver::Manager::getStatus() const
 {
   checkNullPtr("getStatus");
   return cplPtr->getStatus();

packages/nox/src/NOX_Multiphysics_Solver_Manager.H

@@ -120,7 +120,7 @@ public:
                  const Teuchos::RCP<NOX::StatusTest::Generic>& tests);
   virtual void reset(const NOX::Abstract::Vector& initialGuess);
   virtual void reset();
-  virtual NOX::StatusTest::StatusType getStatus();
+  virtual NOX::StatusTest::StatusType getStatus() const;
   virtual NOX::StatusTest::StatusType step();
   virtual NOX::StatusTest::StatusType solve();
   virtual const NOX::Abstract::Group& getSolutionGroup() const;

packages/nox/src/NOX_Solver_Generic.H

@@ -119,7 +119,7 @@ public:
 
   //! Resets the solver, sets a new status test, and sets a new initial guess.
   virtual void reset(const NOX::Abstract::Vector& initial_guess,
-             const Teuchos::RCP<NOX::StatusTest::Generic>& test) = 0;
+                     const Teuchos::RCP<NOX::StatusTest::Generic>& test) = 0;
 
   //! Do one nonlinear step in the iteration sequence and return status.
   virtual NOX::StatusTest::StatusType step() = 0;

packages/piro/test/MockModelEval_A_Tpetra.cpp

@@ -413,100 +413,108 @@ void MockModelEval_A_Tpetra::evalModelImpl(
         ConverterT::getTpetraMultiVector(g_hess_pp_v) :
         Teuchos::null;
 
-  auto x = x_in->getData();
-  auto p = p_vec->getData();
-
-  double x0;
-  for (int i=0; i<myVecLength; i++) {
-    if( x_in->getMap()->getGlobalElement(i) == 0) {
-      x0 = x[i];
-      TEUCHOS_ASSERT(comm->getRank() == 0);
-    }
-  }
-  comm->broadcast(0, sizeof(double), (char*)(&x0));
+  {
+    auto x_host = x_in->getLocalViewHost(Tpetra::Access::ReadOnly);
+    auto p_host = p_vec->getLocalViewHost(Tpetra::Access::ReadOnly);
+    auto x = Kokkos::subview(x_host,Kokkos::ALL(),0);
+    auto p = Kokkos::subview(p_host,Kokkos::ALL(),0);
 
-  if (f_out != Teuchos::null) {
-    f_out->putScalar(0.0);
-    auto f_out_data = f_out->getDataNonConst();
+    double x0;
     for (int i=0; i<myVecLength; i++) {
-      int gid = x_in->getMap()->getGlobalElement(i);
-
-      if (gid==0) { // f_0 = (x_0)^3 - p_0
-        f_out_data[i] = x[i] * x[i] * x[i] -  p[0];
+      if( x_in->getMap()->getGlobalElement(i) == 0) {
+        x0 = x[i];
+        TEUCHOS_ASSERT(comm->getRank() == 0);
       }
-      else // f_i = x_i * (1 + x_0 - p_0^(1/3)) - (i+p_1) - 0.5*(x_0 - p_0),  (for i != 0)
-        f_out_data[i] = x[i] - (gid + p[1]) - 0.5*(x0 - p[0]) + x[i] * (x0 - std::cbrt(p[0]));
     }
-  }
-  if (W_out != Teuchos::null) {
-    Teuchos::RCP<Tpetra_CrsMatrix> W_out_crs =
-      Teuchos::rcp_dynamic_cast<Tpetra_CrsMatrix>(W_out, true);
-    W_out_crs->resumeFill();
-    W_out_crs->setAllToScalar(0.0);
-
-    double diag=0, extra_diag=0;
-    for (int i=0; i<myVecLength; i++) {
-      auto gid = x_in->getMap()->getGlobalElement(i);
-      if(gid==0) {
-        diag = 3.0 * x0 * x0;
-        W_out_crs->replaceLocalValues(i, 1, &diag, &i);
+    comm->broadcast(0, sizeof(double), (char*)(&x0));
+    
+    if (f_out != Teuchos::null) {
+      f_out->putScalar(0.0);
+      auto f_out_data = f_out->getDataNonConst();
+      for (int i=0; i<myVecLength; i++) {
+        int gid = x_in->getMap()->getGlobalElement(i);
+        
+        if (gid==0) { // f_0 = (x_0)^3 - p_0
+          f_out_data[i] = x[i] * x[i] * x[i] -  p[0];
+        }
+        else // f_i = x_i * (1 + x_0 - p_0^(1/3)) - (i+p_1) - 0.5*(x_0 - p_0),  (for i != 0)
+          f_out_data[i] = x[i] - (gid + p[1]) - 0.5*(x0 - p[0]) + x[i] * (x0 - std::cbrt(p[0]));
       }
-      else {
-        diag = 1.0 + x0 - std::cbrt(p[0]);
-        W_out_crs->replaceLocalValues(i, 1, &diag, &i);
-        decltype(gid) col = 0;
-        extra_diag = -0.5 + x[i];
-        W_out_crs->replaceGlobalValues(gid, 1, &extra_diag, &col);
+    }
+    if (W_out != Teuchos::null) {
+      Teuchos::RCP<Tpetra_CrsMatrix> W_out_crs =
+        Teuchos::rcp_dynamic_cast<Tpetra_CrsMatrix>(W_out, true);
+      W_out_crs->resumeFill();
+      W_out_crs->setAllToScalar(0.0);
+
+      double diag=0, extra_diag=0;
+      for (int i=0; i<myVecLength; i++) {
+        auto gid = x_in->getMap()->getGlobalElement(i);
+        if(gid==0) {
+          diag = 3.0 * x0 * x0;
+          W_out_crs->replaceLocalValues(i, 1, &diag, &i);
+        }
+        else {
+          diag = 1.0 + x0 - std::cbrt(p[0]);
+          W_out_crs->replaceLocalValues(i, 1, &diag, &i);
+          decltype(gid) col = 0;
+          extra_diag = -0.5 + x[i];
+          W_out_crs->replaceGlobalValues(gid, 1, &extra_diag, &col);
+        }
       }
+      if(!Teuchos::nonnull(x_dot_in))
+        W_out_crs->fillComplete();
     }
-    if(!Teuchos::nonnull(x_dot_in))
-      W_out_crs->fillComplete();
-  }
-
-  const Teuchos::RCP<Tpetra_Operator> H_pp_out =
+
+    const Teuchos::RCP<Tpetra_Operator> H_pp_out =
       Teuchos::nonnull(outArgs.get_hess_g_pp(0,0,0)) ?
-          ConverterT::getTpetraOperator(outArgs.get_hess_g_pp(0,0,0)) :
-          Teuchos::null;
-
-  if (H_pp_out != Teuchos::null) {
-    Teuchos::RCP<Tpetra_CrsMatrix> H_pp_out_crs =
-      Teuchos::rcp_dynamic_cast<Tpetra_CrsMatrix>(H_pp_out, true);
-    H_pp_out_crs->resumeFill();
-    H_pp_out_crs->setAllToScalar(0.0);
-
-    if (comm->getRank() == 0) {
-      std::vector<double> vals = {2, 1};
-      std::vector<typename Tpetra_CrsGraph::global_ordinal_type> indices = {0, 1};
-      H_pp_out_crs->replaceGlobalValues(0, 2, &vals[0], &indices[0]);
-      vals[0] = 1;
-      H_pp_out_crs->replaceGlobalValues(1, 2, &vals[0], &indices[0]);
-    }
-    H_pp_out_crs->fillComplete();
-  }
-
-  if (Teuchos::nonnull(dfdp_out)) {
-    dfdp_out->putScalar(0.0);
-    auto dfdp_out_data_0 = dfdp_out->getVectorNonConst(0)->getDataNonConst();
-    auto dfdp_out_data_1 = dfdp_out->getVectorNonConst(1)->getDataNonConst();
-    for (int i=0; i<myVecLength; i++) {
-      const int gid = x_in->getMap()->getGlobalElement(i);
-      if  (gid==0) {
-        dfdp_out_data_0[i] = -1.0;
+      ConverterT::getTpetraOperator(outArgs.get_hess_g_pp(0,0,0)) :
+      Teuchos::null;
+
+    if (H_pp_out != Teuchos::null) {
+      Teuchos::RCP<Tpetra_CrsMatrix> H_pp_out_crs =
+        Teuchos::rcp_dynamic_cast<Tpetra_CrsMatrix>(H_pp_out, true);
+      H_pp_out_crs->resumeFill();
+      H_pp_out_crs->setAllToScalar(0.0);
+
+      if (comm->getRank() == 0) {
+        std::vector<double> vals = {2, 1};
+        std::vector<typename Tpetra_CrsGraph::global_ordinal_type> indices = {0, 1};
+        H_pp_out_crs->replaceGlobalValues(0, 2, &vals[0], &indices[0]);
+        vals[0] = 1;
+        H_pp_out_crs->replaceGlobalValues(1, 2, &vals[0], &indices[0]);
       }
-      else {
-        dfdp_out_data_0[i] = 0.5 - x[i] / (3.0 * std::cbrt(p[0]*p[0]));
-        dfdp_out_data_1[i] = -1.0;
+      H_pp_out_crs->fillComplete();
+    }
+
+    if (Teuchos::nonnull(dfdp_out)) {
+      dfdp_out->putScalar(0.0);
+      auto dfdp_out_data_0 = dfdp_out->getVectorNonConst(0)->getDataNonConst();
+      auto dfdp_out_data_1 = dfdp_out->getVectorNonConst(1)->getDataNonConst();
+      for (int i=0; i<myVecLength; i++) {
+        const int gid = x_in->getMap()->getGlobalElement(i);
+        if  (gid==0) {
+          dfdp_out_data_0[i] = -1.0;
+        }
+        else {
+          dfdp_out_data_0[i] = 0.5 - x[i] / (3.0 * std::cbrt(p[0]*p[0]));
+          dfdp_out_data_1[i] = -1.0;
+        }
       }
     }
   }
-
   // Response: g = 0.5*(Sum(x)-Sum(p)-12)^2 + 0.5*(p0-1)^2
   // min g(x(p), p) s.t. f(x, p) = 0 reached for p_0 = 1, p_1 = 3
 
   double term1, term2;
   term1 = x_in->meanValue();
-  term1 =  vecLength * term1 - (p[0] + p[1]) - 12.0;
-  term2 = p[0] - 1.0;
+
+  {
+    auto p_host = p_vec->getLocalViewHost(Tpetra::Access::ReadOnly);
+    auto p = Kokkos::subview(p_host,Kokkos::ALL(),0);
+    term1 =  vecLength * term1 - (p[0] + p[1]) - 12.0;
+    term2 = p[0] - 1.0;
+  }
 
   if (Teuchos::nonnull(g_out)) {
     g_out->getDataNonConst()[0] = 0.5*term1*term1 + 0.5*term2*term2;
@@ -536,6 +544,8 @@ void MockModelEval_A_Tpetra::evalModelImpl(
     double temp= lag_multiplier_f_in->dot(*x_direction->getVector(0));
     comm->broadcast(0, sizeof(double), (char*)(&x_direction_0));
 
+    auto x_host = x_in->getLocalViewHost(Tpetra::Access::ReadOnly);
+    auto x = Kokkos::subview(x_host,Kokkos::ALL(),0);
     f_hess_xx_v_out->getVectorNonConst(0)->putScalar(0);
     for (int i=0; i<myVecLength; i++) {
       if (x_in->getMap()->getGlobalElement(i)==0){
@@ -549,6 +559,8 @@ void MockModelEval_A_Tpetra::evalModelImpl(
   }
 
   if (Teuchos::nonnull(f_hess_xp_v_out)) {
+    auto p_host = p_vec->getLocalViewHost(Tpetra::Access::ReadOnly);
+    auto p = Kokkos::subview(p_host,Kokkos::ALL(),0);
     TEUCHOS_ASSERT(Teuchos::nonnull(p_direction));
     f_hess_xp_v_out->getVectorNonConst(0)->putScalar(0);
     for (int i=0; i<myVecLength; i++) {
@@ -563,6 +575,8 @@ void MockModelEval_A_Tpetra::evalModelImpl(
     TEUCHOS_ASSERT(Teuchos::nonnull(x_direction));
     f_hess_px_v_out->getVectorNonConst(0)->putScalar(0);
     Tpetra_Vector temp_vec(lag_multiplier_f_in->getMap());
+    auto p_host = p_vec->getLocalViewHost(Tpetra::Access::ReadOnly);
+    auto p = Kokkos::subview(p_host,Kokkos::ALL(),0);
     for (int i=0; i<myVecLength; i++) {
       if (x_in->getMap()->getGlobalElement(i)==0)
         temp_vec.getDataNonConst()[i] = 0;
@@ -577,6 +591,8 @@ void MockModelEval_A_Tpetra::evalModelImpl(
     TEUCHOS_ASSERT(Teuchos::nonnull(p_direction));
     f_hess_pp_v_out->getVectorNonConst(0)->putScalar(0);
     Tpetra_Vector temp_vec(lag_multiplier_f_in->getMap());
+    auto p_host = p_vec->getLocalViewHost(Tpetra::Access::ReadOnly);
+    auto p = Kokkos::subview(p_host,Kokkos::ALL(),0);
     for (int i=0; i<myVecLength; i++) {
       if (x_in->getMap()->getGlobalElement(i)==0)
         temp_vec.getDataNonConst()[i] = 0;

packages/stokhos/example/epetra/twoD_diffusion_ME.cpp

@@ -52,7 +52,7 @@ namespace {
   struct KL_Diffusion_Func {
     double mean;
     mutable Teuchos::Array<double> point;
-    Teuchos::RCP< Stokhos::KL::ExponentialRandomField<double> > rf;
+    Teuchos::RCP< Stokhos::KL::ExponentialRandomField<double, Kokkos::DefaultHostExecutionSpace> > rf;
 
     KL_Diffusion_Func(double xyLeft, double xyRight,
                       double mean_, double std_dev,
@@ -75,7 +75,7 @@ namespace {
       rfParams.set("Correlation Lengths", correlation_length);
 
       rf =
-        Teuchos::rcp(new Stokhos::KL::ExponentialRandomField<double>(rfParams));
+        Teuchos::rcp(new Stokhos::KL::ExponentialRandomField<double, Kokkos::DefaultHostExecutionSpace>(rfParams));
     }
 
     ~KL_Diffusion_Func() {