Merge 'trilinos/Trilinos:develop' (c3bf529) into 'tcad-charon/Trilino…

…s:develop' (f3f47c8).

* trilinos-develop:
  Zoltan2: use ZOLTAN2_HAVE_EPETRA. (trilinos#5752)
  Testing: geminga: update valgrind version
  ShuLU: fix for trilinos#5538 (trilinos#5748)
  ROL: fix for Tpetra_DEPRECATED_CODE=OFF (trilinos#5739)
  IOSS: Serialize IO test only with parallel-aware exodus
  Amesos2: Count entries in matrix before allocating (trilinos#5745)
  Tpetra: Fix trilinos#5762
  Ctest: Fixing missing space
  Ctest: geminga test updates
  Testing: change valgrind test days
  Automatic snapshot commit from seacas at e6496d0cc0
  Fix smart-jenkins-driver.sh (trilinos#4933)
  KokkosKernels: CUDA SPGEMM symbolic workaround
  MueLu RefMaxwell: Allow setting importer parameters
  MueLu: Allow setting distributor parameters at end of setup
  Xpetra: Allow setting parameters on Distributor of Import/Export
  Xpetra::ImportFactory: Add optional parameter list
  MueLu: add some braces to region MG smoothers
  MueLu: define list of valid region smoothers
  MueLu: unify setup of region relaxation smoothers

cmake/ctest/drivers/atdm/smart-jenkins-driver.sh

@@ -21,7 +21,8 @@ if [ "${WORKSPACE}" == ""  ] ; then
 fi
 
 export ATDM_CONFIG_BUILD_NAME=$JOB_NAME
-source $WORKSPACE/Trilinos/cmake/std/atdm/utils/get_system_info.sh
+export ATDM_CONFIG_SCRIPT_DIR=$WORKSPACE/Trilinos/cmake/std/atdm
+source ${ATDM_CONFIG_SCRIPT_DIR}/utils/get_system_info.sh
 
 echo
 echo "Running: $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/$ATDM_CONFIG_SYSTEM_NAME/drivers/$JOB_NAME.sh ..."

cmake/ctest/drivers/geminga/CMakeLists.txt

@@ -7,18 +7,15 @@ execute_process(COMMAND date +%A OUTPUT_VARIABLE DAYOFWEEK)
 # parallel valgrind tests run every night except Sunday
 # coverage tests run on Saturday night
 
-if(${DAYOFWEEK} MATCHES "Saturday")
+if((${DAYOFWEEK} MATCHES "Sunday") OR (${DAYOFWEEK} MATCHES "Tuesday") OR (${DAYOFWEEK} MATCHES "Thursday")) 
   SET(RUN_HEAVY_MEMORY_TESTS ON)
   SET(RUN_COVERAGE_TESTS OFF)
-  SET(RUN_CUDA_OLD_TESTS ON)
-elseif(${DAYOFWEEK} MATCHES "Sunday")
+elseif((${DAYOFWEEK} MATCHES "Monday") OR (${DAYOFWEEK} MATCHES "Wednesday"))
   SET(RUN_HEAVY_MEMORY_TESTS OFF)
   SET(RUN_COVERAGE_TESTS ON)
-  SET(RUN_CUDA_OLD_TESTS OFF)
 else()
   SET(RUN_HEAVY_MEMORY_TESTS OFF)
   SET(RUN_COVERAGE_TESTS OFF)
-  SET(RUN_CUDA_OLD_TESTS OFF)
 endif()
 
 SET(RUN_CUDA_OLD_TESTS ON)

cmake/ctest/drivers/geminga/TrilinosCTestDriverCore.geminga.gcc.cmake

@@ -102,6 +102,11 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
 
   SET_DEFAULT(COMPILER_VERSION "$ENV{SEMS_COMPILER_NAME}-$ENV{SEMS_COMPILER_VERSION}")
 
+  # Options for valgrind, if needed
+  SET(CTEST_MEMORYCHECK_COMMAND_OPTIONS
+      "--leak-check=full --gen-suppressions=all --error-limit=no --log-file=nightly_suppressions.txt" ${CTEST_MEMORYCHECK_COMMAND_OPTIONS} )
+
+
   # Ensure that MPI is on for all parallel builds that might be run.
   IF(COMM_TYPE STREQUAL MPI)
 
@@ -112,8 +117,6 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
             "-DMPI_EXEC_POST_NUMPROCS_FLAGS:STRING=--bind-to\\\;socket\\\;--map-by\\\;socket"
        )
 
-    SET(CTEST_MEMORYCHECK_COMMAND_OPTIONS
-        "--gen-suppressions=all --error-limit=no --log-file=nightly_suppressions.txt" ${CTEST_MEMORYCHECK_COMMAND_OPTIONS} )
 
   ELSE()
 

cmake/ctest/drivers/geminga/ctest_linux_nightly_mpi_debug_muelu_geminga.cmake

@@ -70,7 +70,7 @@ SET(BUILD_NAME_DETAILS DEFAULT)
 SET(CTEST_PARALLEL_LEVEL 8)
 SET(CTEST_TEST_TYPE Nightly)
 SET(CTEST_TEST_TIMEOUT 14400) # twice the default value, for valgrind
-SET(CTEST_MEMORYCHECK_COMMAND /home/aprokop/local/opt/valgrind-3.10.1/bin/valgrind)
+SET(CTEST_MEMORYCHECK_COMMAND /usr/local/bin/valgrind)
 SET(CTEST_DO_MEMORY_TESTING FALSE)
 
 SET(Trilinos_PACKAGES MueLu Xpetra Amesos2)

cmake/ctest/drivers/geminga/ctest_linux_nightly_mpi_debug_valgrind_muelu_geminga.cmake

@@ -71,7 +71,7 @@ SET(CTEST_PARALLEL_LEVEL 8)
 SET(CTEST_TEST_TYPE Nightly)
 SET(Trilinos_TRACK  Specialized)     # Set CDash track.
 SET(CTEST_TEST_TIMEOUT 14400) # twice the default value, for valgrind
-SET(CTEST_MEMORYCHECK_COMMAND /home/aprokop/local/opt/valgrind-3.10.1/bin/valgrind)
+SET(CTEST_MEMORYCHECK_COMMAND /usr/local/bin/valgrind)
 SET(CTEST_DO_MEMORY_TESTING TRUE)
 
 SET(Trilinos_PACKAGES MueLu Xpetra Amesos2)

cmake/ctest/drivers/geminga/ctest_linux_nightly_serial_debug_muelu_geminga.cmake

@@ -71,7 +71,7 @@ SET(CTEST_PARALLEL_LEVEL 8)
 SET(CTEST_TEST_TYPE Nightly)
 SET(Trilinos_TRACK  Nightly)  # Set the CDash track to Nightly
 SET(CTEST_TEST_TIMEOUT 14400) # twice the default value, for valgrind
-SET(CTEST_MEMORYCHECK_COMMAND /home/aprokop/local/opt/valgrind-3.10.1/bin/valgrind)
+SET(CTEST_MEMORYCHECK_COMMAND /usr/local/bin/valgrind)
 SET(CTEST_DO_MEMORY_TESTING FALSE)
 
 SET(Trilinos_PACKAGES MueLu Xpetra Amesos2)

cmake/ctest/drivers/geminga/ctest_linux_nightly_serial_debug_valgrind_muelu_geminga.cmake

@@ -69,7 +69,10 @@ SET(BUILD_NAME_DETAILS VALGRIND)
 
 SET(CTEST_PARALLEL_LEVEL 8)
 SET(CTEST_TEST_TYPE Nightly)
+SET(Trilinos_TRACK  Specialized)     # Set CDash track.
 SET(CTEST_TEST_TIMEOUT 14400) # twice the default value, for valgrind
+SET(CTEST_MEMORYCHECK_COMMAND /usr/local/bin/valgrind)
+SET(CTEST_DO_MEMORY_TESTING TRUE)
 
 SET(Trilinos_PACKAGES MueLu Xpetra Amesos2)
 
@@ -80,9 +83,6 @@ SET(EXTRA_CONFIGURE_OPTIONS
   "-DTeuchos_GLOBALLY_REDUCE_UNITTEST_RESULTS=ON"
 )
 
-SET(CTEST_DO_MEMORY_TESTING TRUE)
-SET(CTEST_MEMORYCHECK_COMMAND /home/aprokop/local/opt/valgrind-3.10.1/bin/valgrind)
-
 #
 # Set the rest of the system-specific options and run the dashboard build/test
 #

packages/amesos2/example/GappedMtxGIDs-1proc.cpp

@@ -395,8 +395,7 @@ readCrsMatrixFromFile (const std::string& matrixFilename,
      "Failed to open file \"" << matrixFilename << "\" on Process 0.");
 
   using Teuchos::RCP;
-  //maxNumElementsPerRow as defined by the .mm files that go with this example
-  RCP<MAT> A (new MAT (rowMap, 2));
+  RCP<MAT> A;
 
   if (myRank == 0) 
   {
@@ -409,6 +408,10 @@ readCrsMatrixFromFile (const std::string& matrixFilename,
       std::getline (inFile, line);
     } 
 
+    std::map<GO, size_t> counts;
+    Teuchos::Array<Scalar> vals;
+    Teuchos::Array<GO> gblRowInds;
+    Teuchos::Array<GO> gblColInds;
     while (inFile) {
       std::getline (inFile, line);
       GO gblRowInd {};
@@ -421,9 +424,25 @@ readCrsMatrixFromFile (const std::string& matrixFilename,
           gblRowInd -= 1 ;
           gblColInd -= 1 ;
         }
-        A->insertGlobalValues (gblRowInd, LO (1), &val, &gblColInd);
+        counts[gblRowInd]++;
+        vals.push_back(val);
+        gblRowInds.push_back(gblRowInd);
+        gblColInds.push_back(gblColInd);
       }
     }
+
+    // Max number of entries in any row
+    using pair_type = decltype(counts)::value_type;
+    auto pr = std::max_element(
+        std::begin(counts),
+        std::end(counts),
+        [] (pair_type const& p1, pair_type const& p2){ return p1.second < p2.second; }
+    );
+    size_t maxCount = (counts.empty()) ? size_t(0) : pr->second;
+    A = Teuchos::rcp(new MAT(rowMap, maxCount));
+    for (typename Teuchos::Array<GO>::size_type i=0; i<gblRowInds.size(); i++) {
+      A->insertGlobalValues (gblRowInds[i], gblColInds(i,1), vals(i,1));
+    }
   }
 
   A->fillComplete (domainMap, rangeMap);

packages/kokkos-kernels/src/sparse/impl/KokkosSparse_spgemm_impl_def.hpp

@@ -158,7 +158,14 @@ void KokkosSPGEMM
     //first get the max flops for a row, which will be used for max row size.
     //If we did compression in single step, row_mapB[i] points the begining of row i,
     //and new_row_mapB[i] points to the end of row i.
-    if (compression_applied){
+
+#ifdef KOKKOS_ENABLE_CUDA
+    //BMK 8-19-19: emergency bug workaround on CUDA (#402): disable compression
+    if (compression_applied && !std::is_same<MyExecSpace, Kokkos::Cuda>::value) {
+#else
+    if (compression_applied) {
+#endif
+
 		nnz_lno_t maxNumRoughZeros = this->handle->get_spgemm_handle()->compressed_max_row_flops;
 
     	if (compress_in_single_step){

packages/muelu/adapters/xpetra/MueLu_RefMaxwell_def.hpp

@@ -927,6 +927,15 @@ namespace MueLu {
       D0x_      = MultiVectorFactory::Build(D0_Matrix_->getDomainMap(), 1);
     residual_  = MultiVectorFactory::Build(SM_Matrix_->getDomainMap(), 1);
 
+    if (!ImporterH_.is_null() && parameterList_.isSublist("refmaxwell: ImporterH params")){
+      RCP<ParameterList> importerParams = rcpFromRef(parameterList_.sublist("refmaxwell: ImporterH params"));
+      ImporterH_->setDistributorParameters(importerParams);
+    }
+    if (!Importer22_.is_null() && parameterList_.isSublist("refmaxwell: Importer22 params")){
+      RCP<ParameterList> importerParams = rcpFromRef(parameterList_.sublist("refmaxwell: Importer22 params"));
+      Importer22_->setDistributorParameters(importerParams);
+    }
+
 #ifdef HAVE_MUELU_CUDA
     if (parameterList_.get<bool>("refmaxwell: cuda profile setup", false)) cudaProfilerStop();
 #endif
@@ -976,6 +985,57 @@ namespace MueLu {
         Xpetra::IO<SC, LO, GlobalOrdinal, Node>::Write(std::string("A22.mat"), *A22_);
     }
 
+    if (parameterList_.isSublist("matvec params"))
+    {
+      RCP<ParameterList> matvecParams = rcpFromRef(parameterList_.sublist("matvec params"));
+
+      {
+        RCP<const Import> xpImporter = SM_Matrix_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = SM_Matrix_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      {
+        RCP<const Import> xpImporter = D0_Matrix_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = D0_Matrix_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      {
+        RCP<const Import> xpImporter = D0_T_Matrix_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = D0_T_Matrix_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      {
+        RCP<const Import> xpImporter = R11_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = R11_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      {
+        RCP<const Import> xpImporter = P11_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = P11_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      if (!ImporterH_.is_null())
+        ImporterH_->setDistributorParameters(matvecParams);
+      if (!Importer22_.is_null())
+        Importer22_->setDistributorParameters(matvecParams);
+    }
+
+
     VerboseObject::SetDefaultVerbLevel(oldVerbLevel);
   }
 

packages/muelu/doc/UsersGuide/masterList.xml

@@ -117,6 +117,14 @@
        <visible>true</visible>
      </parameter>
 
+     <parameter>
+      <name>matvec params</name>
+      <type>\parameterlist</type>
+      <description>Pass parameters to the underlying linear algebra</description>
+      <comment-ML>not supported by ML</comment-ML>
+      <visible>false</visible>
+    </parameter>
+
   </general>
 
   <smoothing_and_coarse>

packages/muelu/doc/UsersGuide/paramlist_hidden.tex

@@ -24,6 +24,8 @@
 
 \cbb{hierarchy label}{string}{""}{Label for the hierarchy. Is applied to timer labels.}
 
+\cba{matvec params}{\parameterlist}{Pass parameters to the underlying linear algebra}
+
 \cbb{smoother: pre or post}{string}{"both"}{Pre- and post-smoother combination. Possible values: "pre" (only pre-smoother), "post" (only post-smoother), "both" (both pre-and post-smoothers), "none" (no smoothing).}
 
 \cbb{smoother: type}{string}{"RELAXATION"}{Smoother type. Possible values: see Table~\ref{tab:smoothers}.}