Merge branch 'develop' of ssh://github.com/trilinos/Trilinos into alc…

…f-workshop

* 'develop' of ssh://github.com/trilinos/Trilinos:
  Ifpack2 Chebyshev: Add options for power method
  PIRO: add accessors to the underlying loca steppers
  Tacho LDL for symmetric indefinite problems. (trilinos#9091)
  Geminga: Run on Tesla K40 only
  MueLu Ifpack2Smoother: Use Ifpack2 methods to set zero starting solution
  Ifpack2: Add methods `supportsZeroStartingSolution` and `setZeroStartingSolution`
  Add space in CMAKE_CXX_FLAGS to be safe (trilinos#8449)
  Set non-cache vars for CMAKE_<LANG>_FLAGS with user override (trilinos#8449)
  Xpetra: regionMG matvec function definition added to EpetraCrsMatrix
  tpetra:  fixing downstream compiler errors shown in trilinos#9084
  MueLu Ifpack2: Add tests for reuse and Jacobi
  MueLu: Reuse Chebyshev smoother
  tpetra:  adding back and deprecating getLocalRowView and getGlobalRowView in Tpetra_BlockMultiVector and Tpetra_BlockVector These functions were inadvertantly removed without deprecation during UVM removal. The functions are, however, deprecated.  Users should use getLocalBlockHost with access flags instead of these functions
  Adding xpetra shortnames header to Xpetra_TpetraCrsMatrix
  Add further documentation for region matvec in xpetra
  added small comments and removed unused lines
  MueLu: regionMG add a region matvec to xpetra for use with the muelu region mg

cmake/ctest/drivers/geminga/cron_driver.sh

@@ -100,6 +100,8 @@ env
 
 export CUDA_LAUNCH_BLOCKING=1
 export CUDA_MANAGED_FORCE_DEVICE_ALLOC=1
+# Only run on the Tesla K40, not the Quadro
+export CUDA_VISIBLE_DEVICES=0
 # Machine independent cron_driver:
 SCRIPT_DIR=`cd "\`dirname \"$0\"\`";pwd`
 $SCRIPT_DIR/../cron_driver.py

cmake/std/atdm/ATDMDevEnvSettings.cmake

@@ -222,16 +222,12 @@ IF (ATDM_FPIC)
   SET(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "Build targets with position independent code")
 ENDIF()
 
-ATDM_SET_CACHE(CMAKE_C_FLAGS
-  "$ENV{ATDM_CONFIG_C_FLAGS} $ENV{EXTRA_C_FLAGS}" CACHE STRING)
-ATDM_SET_CACHE(CMAKE_CXX_FLAGS
-  "$ENV{ATDM_CONFIG_CXX_FLAGS}${EXTRA_EXTRA_CXX_FLAGS} $ENV{EXTRA_CXX_FLAGS}" CACHE STRING)
-ATDM_SET_CACHE(CMAKE_Fortran_FLAGS
-  "$ENV{ATDM_CONFIG_Fortran_FLAGS} $ENV{EXTRA_Fortran_FLAGS}" CACHE STRING)
-# NOTE: That above "EXTRA_<LANG>_FLAGS" env vars are left in for backward
-# compatibility with the EMPIRE configuration and are not directly set in any
-# offical ATDM Trilinos configuration setup.  Only the
-# "ATDM_CONFIG_<LANG>_FLAGS" env vars are set.
+SET(CMAKE_C_FLAGS
+  "$ENV{ATDM_CONFIG_C_FLAGS} ${CMAKE_C_FLAGS}")
+SET(CMAKE_CXX_FLAGS
+  "$ENV{ATDM_CONFIG_CXX_FLAGS} ${EXTRA_EXTRA_CXX_FLAGS} ${CMAKE_CXX_FLAGS}")
+SET(CMAKE_Fortran_FLAGS
+  "$ENV{ATDM_CONFIG_Fortran_FLAGS} ${CMAKE_Fortran_FLAGS}")
 
 IF (NOT "$ENV{ATDM_CONFIG_OPENMP_FORTRAN_FLAGS}" STREQUAL "")
   ATDM_SET_CACHE(OpenMP_Fortran_FLAGS "$ENV{ATDM_CONFIG_OPENMP_FORTRAN_FLAGS}"

cmake/std/atdm/README.md

@@ -14,6 +14,7 @@ build locally as described below.
 
 **Outline:**
 * <a href="#quick-start">Quick-start</a>
+* <a href="#setting-and-overridding-compiler-flags">Setting and overridding compiler flags</a>
 * <a href="#parallel-build-and-test-processes">Parallel build and test processes</a>
 * <a href="#installation-and-usage">Installation and usage</a>
 * <a href="#checkin-test-atdmsh">checkin-test-atdm.sh</a>
@@ -414,6 +415,21 @@ where if the CMake cache variable `ATDM_INSTALLED_ENV_LOAD_SCRIPT_NAME` is not
 specified, then it is given the name `load_matching_env.sh` by default.
 
 
+## Setting and overridding compiler flags
+
+Any selected configuration may set C, C++, and Fortran compiler flags by
+internally setting the variables `CMAKE_C_FLAGS`, `CMAKE_CXX_FLAGS`, and
+`CMAKE_Fortran_FLAGS`.  However, the user can also append (and therefore
+override) any of these compiler flags by configuring with the standard CMake
+cache variables:
+
+```
+  -D CMAKE_C_FLAGS="<extra_c_flags>"
+  -D CMAKE_CXX_FLAGS="<extra_cxx_flags>"
+  -D CMAKE_Fortran_FLAGS="<extra_fortran_flags>"
+```
+
+
 ## Parallel build and test processes
 
 By default, each system's `<system_name>/environment.sh` script automatically

packages/ifpack2/src/Ifpack2_AdditiveSchwarz_decl.hpp

@@ -645,6 +645,10 @@ class AdditiveSchwarz :
   void
   setParameterList (const Teuchos::RCP<Teuchos::ParameterList>& plist);
 
+  bool supportsZeroStartingSolution() { return true; }
+
+  void setZeroStartingSolution (bool zeroStartingSolution) { ZeroStartingSolution_ = zeroStartingSolution; };
+
   /// \brief Get a list of the preconditioner's default parameters.
   ///
   /// See the documentation of setParameters() for a list of the

packages/ifpack2/src/Ifpack2_BlockRelaxation_decl.hpp

@@ -209,6 +209,10 @@ class BlockRelaxation :
   */
   void setParameters(const Teuchos::ParameterList& params);
 
+  bool supportsZeroStartingSolution() { return true; }
+
+  void setZeroStartingSolution (bool zeroStartingSolution) { ZeroStartingSolution_ = zeroStartingSolution; };
+
   //! Return a list of all the parameters that this class accepts.
   Teuchos::RCP<const Teuchos::ParameterList>
   getValidParameters () const;

packages/ifpack2/src/Ifpack2_Chebyshev_decl.hpp

@@ -425,6 +425,10 @@ class Chebyshev :
   /// always provide a row Map or range Map vector for this parameter.
   void setParameters (const Teuchos::ParameterList& params);
 
+  bool supportsZeroStartingSolution() { return true; }
+
+  void setZeroStartingSolution (bool zeroStartingSolution);
+
   /// \brief Initialize the preconditioner.
   ///
   /// The compute() method will call initialize() automatically if it

packages/ifpack2/src/Ifpack2_Chebyshev_def.hpp

@@ -97,6 +97,13 @@ Chebyshev<MatrixType>::setParameters (const Teuchos::ParameterList& List)
 }
 
 
+template<class MatrixType>
+void
+Chebyshev<MatrixType>::setZeroStartingSolution (bool zeroStartingSolution)
+{
+  impl_.setZeroStartingSolution(zeroStartingSolution);
+}
+
 template<class MatrixType>
 Teuchos::RCP<const Teuchos::Comm<int> >
 Chebyshev<MatrixType>::getComm () const

packages/ifpack2/src/Ifpack2_Details_Chebyshev_decl.hpp

@@ -262,6 +262,8 @@ class Chebyshev : public Teuchos::Describable {
   /// from Ifpack.
   void setParameters (Teuchos::ParameterList& plist);
 
+  void setZeroStartingSolution (bool zeroStartingSolution) { zeroStartingSolution_ = zeroStartingSolution; }
+
   /// \brief (Re)compute the left scaling D_inv, and estimate min and
   ///   max eigenvalues of D_inv * A.
   ///
@@ -474,6 +476,16 @@ class Chebyshev : public Teuchos::Describable {
   //! Number of power method iterations for estimating the max eigenvalue.
   int eigMaxIters_;
 
+  //! Relative tolerance for power method iterations for estimating the max eigenvalue.
+  MT eigRelTolerance_;
+
+  //! Whether the iteration vectors of the power method should be saved.
+  bool eigKeepVectors_;
+
+  //! Iteration vectors of the power method. Can be saved for the purpose of multiple setups.
+  Teuchos::RCP<V> eigVector_;
+  Teuchos::RCP<V> eigVector2_;
+
   //! Frequency of normalization in the power method.
   int eigNormalizationFreq_;
 

packages/ifpack2/src/Ifpack2_Details_Chebyshev_def.hpp

@@ -286,6 +286,8 @@ Chebyshev (Teuchos::RCP<const row_matrix_type> A) :
   minDiagVal_ (STS::eps ()),
   numIters_ (1),
   eigMaxIters_ (10),
+  eigRelTolerance_(Teuchos::ScalarTraits<MT>::zero ()),
+  eigKeepVectors_(false),
   eigNormalizationFreq_(1),
   zeroStartingSolution_ (true),
   assumeMatrixUnchanged_ (false),
@@ -314,6 +316,8 @@ Chebyshev (Teuchos::RCP<const row_matrix_type> A,
   minDiagVal_ (STS::eps ()),
   numIters_ (1),
   eigMaxIters_ (10),
+  eigRelTolerance_(Teuchos::ScalarTraits<MT>::zero ()),
+  eigKeepVectors_(false),
   eigNormalizationFreq_(1),
   zeroStartingSolution_ (true),
   assumeMatrixUnchanged_ (false),
@@ -357,6 +361,8 @@ setParameters (Teuchos::ParameterList& plist)
   const ST defaultMinDiagVal = STS::eps ();
   const int defaultNumIters = 1;
   const int defaultEigMaxIters = 10;
+  const MT defaultEigRelTolerance = Teuchos::ScalarTraits<MT>::zero ();
+  const bool defaultEigKeepVectors = false;
   const int defaultEigNormalizationFreq = 1;
   const bool defaultZeroStartingSolution = true; // Ifpack::Chebyshev default
   const bool defaultAssumeMatrixUnchanged = false;
@@ -376,6 +382,8 @@ setParameters (Teuchos::ParameterList& plist)
   ST minDiagVal = defaultMinDiagVal;
   int numIters = defaultNumIters;
   int eigMaxIters = defaultEigMaxIters;
+  MT eigRelTolerance = defaultEigRelTolerance;
+  bool eigKeepVectors = defaultEigKeepVectors;
   int eigNormalizationFreq = defaultEigNormalizationFreq;
   bool zeroStartingSolution = defaultZeroStartingSolution;
   bool assumeMatrixUnchanged = defaultAssumeMatrixUnchanged;
@@ -581,6 +589,15 @@ setParameters (Teuchos::ParameterList& plist)
     "\" parameter (also called \"eigen-analysis: iterations\") must be a "
     "nonnegative integer.  You gave a value of " << eigMaxIters << ".");
 
+  if (plist.isType<double>("chebyshev: eigenvalue relative tolerance"))
+    eigRelTolerance = Teuchos::as<MT>(plist.get<double> ("chebyshev: eigenvalue relative tolerance"));
+  else if (plist.isType<MT>("chebyshev: eigenvalue relative tolerance"))
+    eigRelTolerance = plist.get<MT> ("chebyshev: eigenvalue relative tolerance");
+  else if (plist.isType<ST>("chebyshev: eigenvalue relative tolerance"))
+    eigRelTolerance = Teuchos::ScalarTraits<ST>::magnitude(plist.get<ST> ("chebyshev: eigenvalue relative tolerance"));
+
+  eigKeepVectors = plist.get ("chebyshev: eigenvalue keep vectors", eigKeepVectors);
+
   eigNormalizationFreq = plist.get ("chebyshev: eigenvalue normalization frequency", eigNormalizationFreq);
   TEUCHOS_TEST_FOR_EXCEPTION(
     eigNormalizationFreq < 0, std::invalid_argument,
@@ -650,6 +667,8 @@ setParameters (Teuchos::ParameterList& plist)
   minDiagVal_ = minDiagVal;
   numIters_ = numIters;
   eigMaxIters_ = eigMaxIters;
+  eigRelTolerance_ = eigRelTolerance;
+  eigKeepVectors_ = eigKeepVectors;
   eigNormalizationFreq_ = eigNormalizationFreq;
   zeroStartingSolution_ = zeroStartingSolution;
   assumeMatrixUnchanged_ = assumeMatrixUnchanged;
@@ -696,6 +715,8 @@ Chebyshev<ScalarType, MV>::reset ()
   W_ = Teuchos::null;
   computedLambdaMax_ = STS::nan ();
   computedLambdaMin_ = STS::nan ();
+  eigVector_ = Teuchos::null;
+  eigVector2_ = Teuchos::null;
 }
 
 
@@ -1359,9 +1380,16 @@ powerMethodWithInitGuess (const op_type& A,
   const ST zero = static_cast<ST> (0.0);
   const ST one = static_cast<ST> (1.0);
   ST lambdaMax = zero;
-  ST RQ_top, RQ_bottom = one, norm;
-
-  V y (A.getRangeMap ());
+  ST lambdaMaxOld = one;
+  ST norm;
+
+  Teuchos::RCP<V> y;
+  if (eigVector2_.is_null()) {
+    y = rcp(new V(A.getRangeMap ()));
+    if (eigKeepVectors_)
+      eigVector2_ = y;
+  } else
+    y = eigVector2_;
   norm = x.norm2 ();
   TEUCHOS_TEST_FOR_EXCEPTION
     (norm == zero, std::runtime_error,
@@ -1386,16 +1414,31 @@ powerMethodWithInitGuess (const op_type& A,
     if (debug_) {
       *out_ << "  Iteration " << iter << endl;
     }
-    A.apply (x, y);
-    solve (x, D_inv, y);
+    A.apply (x, *y);
+    solve (x, D_inv, *y);
 
     if (((iter+1) % eigNormalizationFreq_ == 0) && (iter < numIters-2)) {
       norm = x.norm2 ();
       if (norm == zero) { // Return something reasonable.
         if (debug_) {
           *out_ << "   norm is zero; returning zero" << endl;
+          *out_ << "   Power method terminated after "<< iter << " iterations." << endl;
         }
         return zero;
+      } else {
+        lambdaMaxOld = lambdaMax;
+        lambdaMax = pow(norm, Teuchos::ScalarTraits<MT>::one() / eigNormalizationFreq_);
+        if (Teuchos::ScalarTraits<ST>::magnitude(lambdaMax-lambdaMaxOld) < eigRelTolerance_ * Teuchos::ScalarTraits<ST>::magnitude(lambdaMax)) {
+          if (debug_) {
+            *out_ << "  lambdaMax: " << lambdaMax << endl;
+            *out_ << "  Power method terminated after "<< iter << " iterations." << endl;
+          }
+          return lambdaMax;
+        } else if (debug_) {
+          *out_ << "  lambdaMaxOld: " << lambdaMaxOld << endl;
+          *out_ << "  lambdaMax: " << lambdaMax << endl;
+          *out_ << "  |lambdaMax-lambdaMaxOld|/|lambdaMax|: " << Teuchos::ScalarTraits<ST>::magnitude(lambdaMax-lambdaMaxOld)/Teuchos::ScalarTraits<ST>::magnitude(lambdaMax) << endl;
+        }
       }
       x.scale (one / norm);
     }
@@ -1408,19 +1451,19 @@ powerMethodWithInitGuess (const op_type& A,
   if (norm == zero) { // Return something reasonable.
     if (debug_) {
       *out_ << "   norm is zero; returning zero" << endl;
+      *out_ << "   Power method terminated after "<< numIters << " iterations." << endl;
     }
     return zero;
   }
   x.scale (one / norm);
-  A.apply (x, y);
-  solve (y, D_inv, y);
-  RQ_top = y.dot (x);
-  RQ_bottom = one;
+  A.apply (x, *y);
+  solve (*y, D_inv, *y);
+  lambdaMax = y->dot (x);
   if (debug_) {
-    *out_ << "   RQ_top: " << RQ_top
-          << ", RQ_bottom: " << RQ_bottom << endl;
+    *out_ << "  lambdaMax: " << lambdaMax << endl;
+    *out_ << "  Power method terminated after "<< numIters << " iterations." << endl;
   }
-  lambdaMax = RQ_top / RQ_bottom;
+
   return lambdaMax;
 }
 
@@ -1456,13 +1499,19 @@ powerMethod (const op_type& A, const V& D_inv, const int numIters)
   }
 
   const ST zero = static_cast<ST> (0.0);
-  V x (A.getDomainMap ());
-  // For the first pass, just let the pseudorandom number generator
-  // fill x with whatever values it wants; don't try to make its
-  // entries nonnegative.
-  computeInitialGuessForPowerMethod (x, false);
-
-  ST lambdaMax = powerMethodWithInitGuess (A, D_inv, numIters, x);
+  Teuchos::RCP<V> x;
+  if (eigVector_.is_null()) {
+    x = rcp(new V(A.getDomainMap ()));
+    if (eigKeepVectors_)
+      eigVector_ = x;
+    // For the first pass, just let the pseudorandom number generator
+    // fill x with whatever values it wants; don't try to make its
+    // entries nonnegative.
+    computeInitialGuessForPowerMethod (*x, false);
+  } else
+    x = eigVector_;
+
+  ST lambdaMax = powerMethodWithInitGuess (A, D_inv, numIters, *x);
 
   // mfh 07 Jan 2015: Taking the real part here is only a concession
   // to the compiler, so that this class can build with ScalarType =
@@ -1483,8 +1532,8 @@ powerMethod (const op_type& A, const V& D_inv, const int numIters)
 
     // For the second pass, make all the entries of the initial guess
     // vector have nonnegative real parts.
-    computeInitialGuessForPowerMethod (x, true);
-    lambdaMax = powerMethodWithInitGuess (A, D_inv, numIters, x);
+    computeInitialGuessForPowerMethod (*x, true);
+    lambdaMax = powerMethodWithInitGuess (A, D_inv, numIters, *x);
   }
   return lambdaMax;
 }
@@ -1672,6 +1721,7 @@ describe (Teuchos::FancyOStream& out,
           << "userEigRatio_: " << userEigRatio_ << endl
           << "numIters_: " << numIters_ << endl
           << "eigMaxIters_: " << eigMaxIters_ << endl
+          << "eigRelTolerance_: " << eigRelTolerance_ << endl
           << "eigNormalizationFreq_: " << eigNormalizationFreq_ << endl
           << "zeroStartingSolution_: " << zeroStartingSolution_ << endl
           << "assumeMatrixUnchanged_: " << assumeMatrixUnchanged_ << endl

packages/ifpack2/src/Ifpack2_Details_GaussSeidel.hpp

@@ -236,7 +236,7 @@ namespace Details
           row = useApplyRows ? applyRows[numApplyRows - 1 - i] : numApplyRows - 1 - i;
         for(LO v = 0; v < numVecs; v++)
         {
-          auto bRow = b.getLocalBlock (row, v, Tpetra::Access::ReadOnly);
+          auto bRow = b.getLocalBlockHost (row, v, Tpetra::Access::ReadOnly);
           for(LO k = 0; k < blockSize; k++)
           {
             accum(k, v) = KAT::zero();
@@ -250,7 +250,7 @@ namespace Details
           IST* blk = &Avalues(j * bs2);
           for(LO v = 0; v < numVecs; v++)
           {
-            auto xCol = x.getLocalBlock (col, v, Tpetra::Access::ReadOnly);
+            auto xCol = x.getLocalBlockHost (col, v, Tpetra::Access::ReadOnly);
             for(LO br = 0; br < blockSize; br++)
             {
               for(LO bc = 0; bc < blockSize; bc++)
@@ -266,7 +266,7 @@ namespace Details
         Kokkos::deep_copy(dinv_accum, KAT::zero());
         for(LO v = 0; v < numVecs; v++)
         {
-          auto bRow = b.getLocalBlock (row, v, Tpetra::Access::ReadOnly);
+          auto bRow = b.getLocalBlockHost (row, v, Tpetra::Access::ReadOnly);
           for(LO br = 0; br < blockSize; br++)
           {
             accum(br, v) = bRow(br) - accum(br, v);
@@ -285,7 +285,7 @@ namespace Details
         //Update x
         for(LO v = 0; v < numVecs; v++)
         {
-          auto xRow = x.getLocalBlock (row, v, Tpetra::Access::ReadWrite);
+          auto xRow = x.getLocalBlockHost (row, v, Tpetra::Access::ReadWrite);
           for(LO k = 0; k < blockSize; k++)
           {
             xRow(k) += omega * dinv_accum(k, v);