From 0be2ed4a256af54d9c2dec5e45fdd2f617616e62 Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Fri, 2 Mar 2018 07:54:44 -0700 Subject: [PATCH 1/5] Disable Panzer examples for CUDA builds on hansen/shiller (#2318, TRIL-171) EMPIRE does not build or test these examples and they don't currently build (see #2318). This just removes the clutter on CDash while these things can be fixed offline so that we can focus on the remaining failures. --- cmake/std/atdm/shiller/tweaks/CUDA-DEBUG-CUDA.cmake | 1 + cmake/std/atdm/shiller/tweaks/CUDA-RELEASE-CUDA.cmake | 1 + cmake/std/atdm/shiller/tweaks/CUDA_COMMON_TWEAKS.cmake | 9 +++++++++ 3 files changed, 11 insertions(+) create mode 100644 cmake/std/atdm/shiller/tweaks/CUDA-DEBUG-CUDA.cmake create mode 100644 cmake/std/atdm/shiller/tweaks/CUDA-RELEASE-CUDA.cmake create mode 100644 cmake/std/atdm/shiller/tweaks/CUDA_COMMON_TWEAKS.cmake diff --git a/cmake/std/atdm/shiller/tweaks/CUDA-DEBUG-CUDA.cmake b/cmake/std/atdm/shiller/tweaks/CUDA-DEBUG-CUDA.cmake new file mode 100644 index 000000000000..04fe70e8f92d --- /dev/null +++ b/cmake/std/atdm/shiller/tweaks/CUDA-DEBUG-CUDA.cmake @@ -0,0 +1 @@ +INCLUDE("${CMAKE_CURRENT_LIST_DIR}/CUDA_COMMON_TWEAKS.cmake") diff --git a/cmake/std/atdm/shiller/tweaks/CUDA-RELEASE-CUDA.cmake b/cmake/std/atdm/shiller/tweaks/CUDA-RELEASE-CUDA.cmake new file mode 100644 index 000000000000..04fe70e8f92d --- /dev/null +++ b/cmake/std/atdm/shiller/tweaks/CUDA-RELEASE-CUDA.cmake @@ -0,0 +1 @@ +INCLUDE("${CMAKE_CURRENT_LIST_DIR}/CUDA_COMMON_TWEAKS.cmake") diff --git a/cmake/std/atdm/shiller/tweaks/CUDA_COMMON_TWEAKS.cmake b/cmake/std/atdm/shiller/tweaks/CUDA_COMMON_TWEAKS.cmake new file mode 100644 index 000000000000..30f7ce6676fc --- /dev/null +++ b/cmake/std/atdm/shiller/tweaks/CUDA_COMMON_TWEAKS.cmake @@ -0,0 +1,9 @@ +ATDM_SET_ENABLE(Panzer_ENABLE_EXAMPLES OFF) +ATDM_SET_ENABLE(PanzerCore_ENABLE_EXAMPLES OFF) +ATDM_SET_ENABLE(PanzerDofMgr_ENABLE_EXAMPLES OFF) +ATDM_SET_ENABLE(PanzerDiscFE_ENABLE_EXAMPLES OFF) +ATDM_SET_ENABLE(PanzerAdaptersSTK_ENABLE_EXAMPLES OFF) +ATDM_SET_ENABLE(PanzerAdaptersIOSS_ENABLE_EXAMPLES OFF) +ATDM_SET_ENABLE(PanzerMiniEM_ENABLE_EXAMPLES OFF) +# ToDo: Change TriBITS so that, by default, Panzer_ENABLE_EXAMPLES=OFF will +# disable examples in the Panzer subpackages!q \ No newline at end of file From 82ff8b5c8225b98a24d39f382437c5e31df90206 Mon Sep 17 00:00:00 2001 From: Dan Ibanez Date: Fri, 2 Mar 2018 11:31:49 -0700 Subject: [PATCH 2/5] Teuchos: Fix uninitialized variable (#2321) Fix #2316. PR #2321. --- packages/teuchos/core/src/Teuchos_BigUInt.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/teuchos/core/src/Teuchos_BigUInt.hpp b/packages/teuchos/core/src/Teuchos_BigUInt.hpp index f2b236b0f0e6..5b6d3c5e8392 100644 --- a/packages/teuchos/core/src/Teuchos_BigUInt.hpp +++ b/packages/teuchos/core/src/Teuchos_BigUInt.hpp @@ -176,7 +176,7 @@ BigUInt& BigUInt::operator>>=(std::uint32_t b) { auto ndigits = b / 32; auto nbits = b - (ndigits * 32); for (int i = 0; i < n; ++i) { - std::uint32_t xi; + std::uint32_t xi = 0; if (i + ndigits < n) xi = x[i + ndigits] >> nbits; if (nbits && i + ndigits + 1 < n) xi |= x[i + ndigits + 1] << (32 - nbits); x[i] = xi; From 561ba2adff66d9ffc5b5f305128393781980dff6 Mon Sep 17 00:00:00 2001 From: "K. Devine" Date: Fri, 2 Mar 2018 11:50:43 -0700 Subject: [PATCH 3/5] ID printing and rudimentary correctness testing added --- packages/zoltan2/example/block/block.cpp | 45 ++++++++++++++++++------ 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/packages/zoltan2/example/block/block.cpp b/packages/zoltan2/example/block/block.cpp index 32ce235b4e47..6936ff0a97b0 100644 --- a/packages/zoltan2/example/block/block.cpp +++ b/packages/zoltan2/example/block/block.cpp @@ -51,8 +51,6 @@ #include #include -using namespace std; - /*! \example block.cpp An example of the use of the Block algorithm to partition data. \todo error handling @@ -80,18 +78,20 @@ int main(int argc, char *argv[]) /////////////////////////////////////////////////////////////////////// // Generate some input data. - size_t localCount = 40*(rank+1); - globalId_t *globalIds = new globalId_t [localCount]; + int localCount = 40*(rank+1); + int totalCount = 20*nprocs*(nprocs+1); + int targetCount = totalCount / nprocs; + globalId_t *globalIds = new globalId_t[localCount]; if (rank==0) - for (int i=0, num=40; i <= nprocs ; i++, num+=40) - cout << "Rank " << i << " has " << num << " ids." << endl; + for (int i=0, num=40; i < nprocs ; i++, num+=40) + std::cout << "Rank " << i << " generates " << num << " ids." << std::endl; globalId_t offset = 0; for (int i=1; i <= rank; i++) offset += 40*i; - for (size_t i=0; i < localCount; i++) + for (int i=0; i < localCount; i++) globalIds[i] = offset++; /////////////////////////////////////////////////////////////////////// @@ -132,10 +132,35 @@ int main(int argc, char *argv[]) problem->solve(); /////////////////////////////////////////////////////////////////////// - // Check the solution. + // Check and print the solution. + // Count number of IDs assigned to each part; compare to targetCount + + const globalId_t *ids = NULL; + ia.getIDsView(ids); + std::vector partCounts(nprocs, 0), globalPartCounts(nprocs, 0); + + for (size_t i = 0; i < ia.getLocalNumIDs(); i++) { + int pp = problem->getSolution().getPartListView()[i]; + std::cout << rank << " LID " << i << " GID " << ids[i] + << " PART " << pp << std::endl; + partCounts[pp]++; + } + +#ifdef HAVE_ZOLTAN2_MPI + MPI_Allreduce(&(partCounts[0]), &(globalPartCounts[0]), nprocs, + MPI_INT, MPI_SUM, MPI_COMM_WORLD); +#endif - if (rank == 0) - cout << "PASS" << endl; + if (rank == 0) { + int ierr = 0; + for (int i = 0; i < nprocs; i++) + if (globalPartCounts[i] != targetCount) + std::cout << "FAIL: part " << i << " has " << partCounts[i] + << " != " << targetCount << "; " << ++ierr << " errors" + << std::endl; + if (ierr == 0) + std::cout << "PASS" << std::endl; + } delete [] globalIds; delete problem; From fd96bc67972ef93e256c2559df981153f19b87fc Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Fri, 2 Mar 2018 12:06:07 -0700 Subject: [PATCH 4/5] Change KOKKOS_ARCH from BDW to HSW (#2320, TRIL-171) After talking wit Si H., he said that hansen and shiller are Haswell not Boradwell. --- cmake/std/atdm/shiller/environment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/std/atdm/shiller/environment.sh b/cmake/std/atdm/shiller/environment.sh index 75117bea4c90..b05c7b3f54fd 100755 --- a/cmake/std/atdm/shiller/environment.sh +++ b/cmake/std/atdm/shiller/environment.sh @@ -15,7 +15,7 @@ echo "Using hansen/shiller compiler stack $ATDM_CONFIG_COMPILER to build $ATDM_C module load ninja/1.7.2 -export ATDM_CONFIG_KOKKOS_ARCH=BDW +export ATDM_CONFIG_KOKKOS_ARCH=HSW if [ "$ATDM_CONFIG_COMPILER" == "GNU" ]; then module load devpack/openmpi/2.1.1/gcc/4.9.3/cuda/8.0.61 export OMPI_CXX=`which g++` From 62255d849db8afb10a82bc9e14439e7288350c6d Mon Sep 17 00:00:00 2001 From: Matthias Mayr Date: Fri, 2 Mar 2018 14:20:57 -0800 Subject: [PATCH 5/5] Revert "MueLu/HHG: experiments with Import() and duplicated GIDs" This reverts commit 506af3b17d4c8a5de1d2bccdf1bd27d859aef1c0. Build/Test Cases Summary Enabled Packages: MueLu Enabled all Forward Packages 0) MPI_RELEASE_DEBUG_SHARED_PT => passed: passed=495,notpassed=0 (91.93 min) --- .../luc/region_algorithms/CMakeLists.txt | 6 - .../research/luc/region_algorithms/Import.cpp | 187 ------------------ 2 files changed, 193 deletions(-) delete mode 100644 packages/muelu/research/luc/region_algorithms/Import.cpp diff --git a/packages/muelu/research/luc/region_algorithms/CMakeLists.txt b/packages/muelu/research/luc/region_algorithms/CMakeLists.txt index b87b48454a31..9943f5e88ae6 100644 --- a/packages/muelu/research/luc/region_algorithms/CMakeLists.txt +++ b/packages/muelu/research/luc/region_algorithms/CMakeLists.txt @@ -13,12 +13,6 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../../test/unit_tests) COMM serial mpi ) - TRIBITS_ADD_EXECUTABLE( - ImportTest - SOURCES Import.cpp - COMM serial mpi - ) - # TRIBITS_COPY_FILES_TO_BINARY_DIR(tawiesn_cp # SOURCE_FILES driver.xml # ) diff --git a/packages/muelu/research/luc/region_algorithms/Import.cpp b/packages/muelu/research/luc/region_algorithms/Import.cpp deleted file mode 100644 index 0e295b1b1554..000000000000 --- a/packages/muelu/research/luc/region_algorithms/Import.cpp +++ /dev/null @@ -1,187 +0,0 @@ -// @HEADER -// -// *********************************************************************** -// -// MueLu: A package for multigrid based preconditioning -// Copyright 2012 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact -// Jonathan Hu (jhu@sandia.gov) -// Andrey Prokopenko (aprokop@sandia.gov) -// Ray Tuminaro (rstumin@sandia.gov) -// Tobias Wiesner (tawiesn@sandia.gov) -// -// *********************************************************************** -// -// @HEADER - -// Teuchos -#include -#include - -#include -#include // TODO: move into MueLu.hpp -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -#ifdef HAVE_MUELU_BELOS -#include -#include -#include -#include -#include // => This header defines Belos::XpetraOp -#include // => This header defines Belos::MueLuOp -#endif - -template -int main_(Teuchos::CommandLineProcessor &clp, Xpetra::UnderlyingLib lib, int argc, char *argv[]) { -#include - - using Teuchos::Array; using Teuchos::ArrayView; - using Teuchos::RCP; using Teuchos::rcp; - using Teuchos::tuple; - using Teuchos::TimeMonitor; - - typedef Tpetra::Map map_type; - typedef Tpetra::CrsMatrix crs_matrix_type; - - typedef typename crs_matrix_type::scalar_type scalar_type; - typedef typename crs_matrix_type::local_ordinal_type local_ordinal_type; - typedef typename crs_matrix_type::global_ordinal_type global_ordinal_type; - typedef typename crs_matrix_type::node_type node_type; - typedef typename crs_matrix_type::mag_type magnitude_type; - - bool success = false; - bool verbose = true; - - try { - RCP< const Teuchos::Comm > comm = Teuchos::DefaultComm::getComm(); - const local_ordinal_type myRank = comm->getRank(); - - // Manage the way output stream works - RCP fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); - fancy->setShowAllFrontMatter(false).setShowProcRank(true); - Teuchos::FancyOStream& out = *fancy; - // out.setOutputToRootOnly(0); - // // Do not print anything - // Teuchos::oblackholestream blackhole; - // std::ostream& out = blackhole; - - // ========================================================================= - // Parameters initialization - // ========================================================================= - - //GO nx = 100, ny = 100, nz = 100; - //Galeri::Xpetra::Parameters matrixParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case - Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra - - std::string xmlFileName = "driver.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file. Otherwise, this example uses by default 'scalingTest.xml'"); - - switch (clp.parse(argc,argv)) { - case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break; - case Teuchos::CommandLineProcessor::PARSE_ERROR: - case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break; - case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break; - } - - global_ordinal_type gNumCompFineGIDs = 10, lCompFineGIDOffset = 0; - global_ordinal_type gNumRegFineGIDs = 12, lRegFineGIDOffset = 0; - local_ordinal_type lNumCompFineGIDs = 0, lNumRegFineGIDs = 0; - local_ordinal_type lNumCompCoarseGIDs = 0, lNumRegCoarseGIDs = 0; - Array quasiGIDs, compGIDs; - if(myRank == 0) { - compGIDs.resize(3); - quasiGIDs.resize(3); - for(int i = 0; i < 3; ++i) { - quasiGIDs[i] = i; - compGIDs[i] = i; - } - } else if(myRank == 1) { - quasiGIDs.resize(5); - quasiGIDs[0] = 2; - quasiGIDs[1] = 3; - quasiGIDs[2] = 4; - quasiGIDs[3] = 4; - quasiGIDs[4] = 5; - - compGIDs.resize(3); - for(int i = 0; i < 3; ++i) { - compGIDs[i] = i + 3; - } - } - out << "quasiGIDs: " << quasiGIDs << std::endl; - out << "compGIDs: " << compGIDs << std::endl; - RCP compMap = rcp(new map_type(6, compGIDs, 0, comm)); - RCP quasiMap = rcp(new map_type(8, quasiGIDs, 0, comm)); - - Tpetra::MultiVector quasiVector(quasiMap, 1); - Teuchos::ArrayRCP inputVector = quasiVector.getDataNonConst(0); - if(myRank == 0) { - for(int i = 0; i < 3; ++i) { - inputVector[i] = i; - } - } else if(myRank == 1) { - for(int i = 0; i < 3; ++i) { - inputVector[i] = i + 3; - } - } - Tpetra::MultiVector compVector(compMap, 1); - - Tpetra::Export myExporter(quasiMap, compMap); - compVector.doExport(quasiVector, myExporter, Tpetra::CombineMode::ADD); - Teuchos::ArrayRCP myData = compVector.getData(0); - std::cout << "p=" << comm->getRank() << " | compVector: " << myData() << std::endl; - - success = true; - } - TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success); - - return ( success ? EXIT_SUCCESS : EXIT_FAILURE ); -} //main - -//- -- -------------------------------------------------------- -#define MUELU_AUTOMATIC_TEST_ETI_NAME main_ -#include "MueLu_Test_ETI.hpp" - -int main(int argc, char *argv[]) { - return Automatic_Test_ETI(argc,argv); -}