Skip to content

Commit

Permalink
Merge 'trilinos/Trilinos:develop' (f89b605) into 'EM-Plasma/Trilinos:…
Browse files Browse the repository at this point in the history 
…develop' (2d91d5f).

* trilinos/develop:
  Add space in CMAKE_CXX_FLAGS to be safe (trilinos#8449)
  Set non-cache vars for CMAKE_<LANG>_FLAGS with user override (trilinos#8449)
  tpetra:  fixing downstream compiler errors shown in trilinos#9084
  MueLu Ifpack2: Add tests for reuse and Jacobi
  MueLu: Reuse Chebyshev smoother
  tpetra:  adding back and deprecating getLocalRowView and getGlobalRowView in Tpetra_BlockMultiVector and Tpetra_BlockVector These functions were inadvertantly removed without deprecation during UVM removal. The functions are, however, deprecated.  Users should use getLocalBlockHost with access flags instead of these functions
  • Loading branch information
EMPIRE Jenkins committed May 5, 2021
2 parents 2d91d5f + f89b605 commit 7d08c7d
Show file tree
Hide file tree
Showing 17 changed files with 370 additions and 126 deletions.
16 changes: 6 additions & 10 deletions cmake/std/atdm/ATDMDevEnvSettings.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,16 +222,12 @@ IF (ATDM_FPIC)
SET(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "Build targets with position independent code")
ENDIF()

ATDM_SET_CACHE(CMAKE_C_FLAGS
"$ENV{ATDM_CONFIG_C_FLAGS} $ENV{EXTRA_C_FLAGS}" CACHE STRING)
ATDM_SET_CACHE(CMAKE_CXX_FLAGS
"$ENV{ATDM_CONFIG_CXX_FLAGS}${EXTRA_EXTRA_CXX_FLAGS} $ENV{EXTRA_CXX_FLAGS}" CACHE STRING)
ATDM_SET_CACHE(CMAKE_Fortran_FLAGS
"$ENV{ATDM_CONFIG_Fortran_FLAGS} $ENV{EXTRA_Fortran_FLAGS}" CACHE STRING)
# NOTE: That above "EXTRA_<LANG>_FLAGS" env vars are left in for backward
# compatibility with the EMPIRE configuration and are not directly set in any
# offical ATDM Trilinos configuration setup. Only the
# "ATDM_CONFIG_<LANG>_FLAGS" env vars are set.
SET(CMAKE_C_FLAGS
"$ENV{ATDM_CONFIG_C_FLAGS} ${CMAKE_C_FLAGS}")
SET(CMAKE_CXX_FLAGS
"$ENV{ATDM_CONFIG_CXX_FLAGS} ${EXTRA_EXTRA_CXX_FLAGS} ${CMAKE_CXX_FLAGS}")
SET(CMAKE_Fortran_FLAGS
"$ENV{ATDM_CONFIG_Fortran_FLAGS} ${CMAKE_Fortran_FLAGS}")

IF (NOT "$ENV{ATDM_CONFIG_OPENMP_FORTRAN_FLAGS}" STREQUAL "")
ATDM_SET_CACHE(OpenMP_Fortran_FLAGS "$ENV{ATDM_CONFIG_OPENMP_FORTRAN_FLAGS}"
Expand Down
16 changes: 16 additions & 0 deletions cmake/std/atdm/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ build locally as described below.

**Outline:**
* <a href="#quick-start">Quick-start</a>
* <a href="#setting-and-overridding-compiler-flags">Setting and overridding compiler flags</a>
* <a href="#parallel-build-and-test-processes">Parallel build and test processes</a>
* <a href="#installation-and-usage">Installation and usage</a>
* <a href="#checkin-test-atdmsh">checkin-test-atdm.sh</a>
Expand Down Expand Up @@ -414,6 +415,21 @@ where if the CMake cache variable `ATDM_INSTALLED_ENV_LOAD_SCRIPT_NAME` is not
specified, then it is given the name `load_matching_env.sh` by default.


## Setting and overridding compiler flags

Any selected configuration may set C, C++, and Fortran compiler flags by
internally setting the variables `CMAKE_C_FLAGS`, `CMAKE_CXX_FLAGS`, and
`CMAKE_Fortran_FLAGS`. However, the user can also append (and therefore
override) any of these compiler flags by configuring with the standard CMake
cache variables:

```
-D CMAKE_C_FLAGS="<extra_c_flags>"
-D CMAKE_CXX_FLAGS="<extra_cxx_flags>"
-D CMAKE_Fortran_FLAGS="<extra_fortran_flags>"
```


## Parallel build and test processes

By default, each system's `<system_name>/environment.sh` script automatically
Expand Down
8 changes: 4 additions & 4 deletions packages/ifpack2/src/Ifpack2_Details_GaussSeidel.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ namespace Details
row = useApplyRows ? applyRows[numApplyRows - 1 - i] : numApplyRows - 1 - i;
for(LO v = 0; v < numVecs; v++)
{
auto bRow = b.getLocalBlock (row, v, Tpetra::Access::ReadOnly);
auto bRow = b.getLocalBlockHost (row, v, Tpetra::Access::ReadOnly);
for(LO k = 0; k < blockSize; k++)
{
accum(k, v) = KAT::zero();
Expand All @@ -250,7 +250,7 @@ namespace Details
IST* blk = &Avalues(j * bs2);
for(LO v = 0; v < numVecs; v++)
{
auto xCol = x.getLocalBlock (col, v, Tpetra::Access::ReadOnly);
auto xCol = x.getLocalBlockHost (col, v, Tpetra::Access::ReadOnly);
for(LO br = 0; br < blockSize; br++)
{
for(LO bc = 0; bc < blockSize; bc++)
Expand All @@ -266,7 +266,7 @@ namespace Details
Kokkos::deep_copy(dinv_accum, KAT::zero());
for(LO v = 0; v < numVecs; v++)
{
auto bRow = b.getLocalBlock (row, v, Tpetra::Access::ReadOnly);
auto bRow = b.getLocalBlockHost (row, v, Tpetra::Access::ReadOnly);
for(LO br = 0; br < blockSize; br++)
{
accum(br, v) = bRow(br) - accum(br, v);
Expand All @@ -285,7 +285,7 @@ namespace Details
//Update x
for(LO v = 0; v < numVecs; v++)
{
auto xRow = x.getLocalBlock (row, v, Tpetra::Access::ReadWrite);
auto xRow = x.getLocalBlockHost (row, v, Tpetra::Access::ReadWrite);
for(LO k = 0; k < blockSize; k++)
{
xRow(k) += omega * dinv_accum(k, v);
Expand Down
18 changes: 12 additions & 6 deletions packages/ifpack2/src/Ifpack2_Experimental_RBILUK_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -846,8 +846,10 @@ apply (const Tpetra::MultiVector<scalar_type,local_ordinal_type,global_ordinal_t
for (size_t i = 0; i < D_block_->getNodeNumRows(); ++i)
{
local_ordinal_type local_row = i;
const_host_little_vec_type xval = xBlock.getLocalBlock(local_row, imv, Tpetra::Access::ReadOnly);
little_host_vec_type cval = cBlock.getLocalBlock(local_row, imv, Tpetra::Access::OverwriteAll);
const_host_little_vec_type xval =
xBlock.getLocalBlockHost(local_row, imv, Tpetra::Access::ReadOnly);
little_host_vec_type cval =
cBlock.getLocalBlockHost(local_row, imv, Tpetra::Access::OverwriteAll);
//cval.assign(xval);
Tpetra::COPY (xval, cval);

Expand All @@ -860,7 +862,8 @@ apply (const Tpetra::MultiVector<scalar_type,local_ordinal_type,global_ordinal_t
for (local_ordinal_type j = 0; j < NumL; ++j)
{
local_ordinal_type col = colValsL[j];
const_host_little_vec_type prevVal = cBlock.getLocalBlock(col, imv, Tpetra::Access::ReadOnly);
const_host_little_vec_type prevVal =
cBlock.getLocalBlockHost(col, imv, Tpetra::Access::ReadOnly);

const local_ordinal_type matOffset = blockMatSize*j;
little_block_type lij((typename little_block_type::value_type*) &valsL[matOffset],blockSize_,rowStride);
Expand All @@ -881,8 +884,10 @@ apply (const Tpetra::MultiVector<scalar_type,local_ordinal_type,global_ordinal_t
for (local_ordinal_type i = 0; i < numRows; ++i)
{
local_ordinal_type local_row = (numRows-1)-i;
const_host_little_vec_type rval = rBlock.getLocalBlock(local_row, imv, Tpetra::Access::ReadOnly);
little_host_vec_type yval = yBlock.getLocalBlock(local_row, imv, Tpetra::Access::OverwriteAll);
const_host_little_vec_type rval =
rBlock.getLocalBlockHost(local_row, imv, Tpetra::Access::ReadOnly);
little_host_vec_type yval =
yBlock.getLocalBlockHost(local_row, imv, Tpetra::Access::OverwriteAll);
//yval.assign(rval);
Tpetra::COPY (rval, yval);

Expand All @@ -895,7 +900,8 @@ apply (const Tpetra::MultiVector<scalar_type,local_ordinal_type,global_ordinal_t
for (local_ordinal_type j = 0; j < NumU; ++j)
{
local_ordinal_type col = colValsU[NumU-1-j];
const_host_little_vec_type prevVal = yBlock.getLocalBlock(col, imv, Tpetra::Access::ReadOnly);
const_host_little_vec_type prevVal =
yBlock.getLocalBlockHost(col, imv, Tpetra::Access::ReadOnly);

const local_ordinal_type matOffset = blockMatSize*(NumU-1-j);
little_block_type uij((typename little_block_type::value_type*) &valsU[matOffset], blockSize_, rowStride);
Expand Down
6 changes: 3 additions & 3 deletions packages/ifpack2/test/unit_tests/Ifpack2_UnitTestBlockRelaxation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -708,7 +708,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(Ifpack2BlockRelaxation, TestDiagonalBlockCrsMa
const Scalar exactSol = 0.2;

for (int k = 0; k < num_rows_per_proc; ++k) {
auto ylcl = yBlock.getLocalBlock(k, 0, Tpetra::Access::ReadOnly);
auto ylcl = yBlock.getLocalBlockHost(k, 0, Tpetra::Access::ReadOnly);
for (int j = 0; j < blockSize; ++j) {
TEST_FLOATING_EQUALITY(ylcl(j), exactSol, 1e-14);
}
Expand Down Expand Up @@ -1269,7 +1269,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(Ifpack2BlockRelaxation, TestLowerTriangularBlo

for (size_t k = 0; k < num_rows_per_proc; ++k) {
LO lcl_row = k;
auto ylcl = yBlock.getLocalBlock(lcl_row, 0, Tpetra::Access::ReadOnly);
auto ylcl = yBlock.getLocalBlockHost(lcl_row, 0, Tpetra::Access::ReadOnly);
for (int j = 0; j < blockSize; ++j) {
TEST_FLOATING_EQUALITY(ylcl(j), exactSol[k], 1e-14);
}
Expand Down Expand Up @@ -1329,7 +1329,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(Ifpack2BlockRelaxation, TestUpperTriangularBlo
exactSol[2] = 0.5;

for (int k = 0; k < num_rows_per_proc; ++k) {
auto ylcl = yBlock.getLocalBlock(k, 0, Tpetra::Access::ReadOnly);
auto ylcl = yBlock.getLocalBlockHost(k, 0, Tpetra::Access::ReadOnly);
for (int j = 0; j < blockSize; ++j) {
TEST_FLOATING_EQUALITY(ylcl(j), exactSol[k], 1e-14);
}
Expand Down
8 changes: 4 additions & 4 deletions packages/ifpack2/test/unit_tests/Ifpack2_UnitTestRBILUK.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(RBILUK, LowerTriangularBlockCrsMatrix, Scalar,

for (size_t k = 0; k < num_rows_per_proc; ++k) {
LO lcl_row = k;
auto ylcl = yBlock.getLocalBlock(lcl_row, 0, Tpetra::Access::ReadOnly);
auto ylcl = yBlock.getLocalBlockHost(lcl_row, 0, Tpetra::Access::ReadOnly);
for (int j = 0; j < blockSize; ++j) {
TEST_FLOATING_EQUALITY(ylcl(j),exactSol[k],1e-14);
}
Expand Down Expand Up @@ -260,7 +260,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(RBILUK, UpperTriangularBlockCrsMatrix, Scalar,
exactSol[2] = 0.5;

for (int k = 0; k < num_rows_per_proc; ++k) {
auto ylcl = yBlock.getLocalBlock(k, 0, Tpetra::Access::ReadOnly);
auto ylcl = yBlock.getLocalBlockHost(k, 0, Tpetra::Access::ReadOnly);
for (int j = 0; j < blockSize; ++j) {
TEST_FLOATING_EQUALITY(ylcl(j), exactSol[k], 1e-14);
}
Expand Down Expand Up @@ -313,7 +313,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(RBILUK, FullLocalBlockCrsMatrix, Scalar, Local
exactSol[2] = 2.0/7.0;

for (int k = 0; k < num_rows_per_proc; ++k) {
auto ylcl = yBlock.getLocalBlock(k, 0, Tpetra::Access::ReadOnly);
auto ylcl = yBlock.getLocalBlockHost(k, 0, Tpetra::Access::ReadOnly);
for (int j = 0; j < blockSize; ++j) {
TEST_FLOATING_EQUALITY(ylcl(j), exactSol[k], 1e-14);
}
Expand Down Expand Up @@ -728,7 +728,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(RBILUK, DiagonalBlockCrsMatrix, Scalar, LocalO
const Scalar exactSol = 0.2;

for (int k = 0; k < num_rows_per_proc; ++k) {
auto ylcl = yBlock.getLocalBlock(k, 0, Tpetra::Access::ReadOnly);
auto ylcl = yBlock.getLocalBlockHost(k, 0, Tpetra::Access::ReadOnly);
for (int j = 0; j < blockSize; ++j) {
TEST_FLOATING_EQUALITY(ylcl(j), exactSol, 1e-14);
}
Expand Down
6 changes: 3 additions & 3 deletions packages/ifpack2/test/unit_tests/Ifpack2_UnitTestRelaxation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -915,7 +915,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(Ifpack2Relaxation, TestDiagonalBlockCrsMatrix,
const auto tol = mag_type(100.0) * STS::eps();

for (int k = 0; k < num_rows_per_proc; ++k) {
auto ylcl = yBlock.getLocalBlock(k, 0, Tpetra::Access::ReadOnly);
auto ylcl = yBlock.getLocalBlockHost(k, 0, Tpetra::Access::ReadOnly);
for (int j = 0; j < blockSize; ++j) {
TEST_FLOATING_EQUALITY(ylcl(j), exactSol, tol);
}
Expand Down Expand Up @@ -1031,7 +1031,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(Ifpack2Relaxation, TestLowerTriangularBlockCrs

for (size_t k = 0; k < num_rows_per_proc; ++k) {
LO lcl_row = k;
auto ylcl = yBlock.getLocalBlock(lcl_row, 0, Tpetra::Access::ReadOnly);
auto ylcl = yBlock.getLocalBlockHost(lcl_row, 0, Tpetra::Access::ReadOnly);
for (int j = 0; j < blockSize; ++j) {
TEST_FLOATING_EQUALITY(ylcl(j), exactSol[k], 1e-14);
}
Expand Down Expand Up @@ -1081,7 +1081,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(Ifpack2Relaxation, TestUpperTriangularBlockCrs
exactSol[2] = 0.5;

for (int k = 0; k < num_rows_per_proc; ++k) {
auto ylcl = yBlock.getLocalBlock(k, 0, Tpetra::Access::ReadOnly);
auto ylcl = yBlock.getLocalBlockHost(k, 0, Tpetra::Access::ReadOnly);
for (int j = 0; j < blockSize; ++j) {
TEST_FLOATING_EQUALITY(ylcl(j), exactSol[k], 1e-14);
}
Expand Down
38 changes: 29 additions & 9 deletions packages/muelu/src/Smoothers/MueLu_Ifpack2Smoother_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -599,9 +599,27 @@ namespace MueLu {

template <class Scalar,class LocalOrdinal, class GlobalOrdinal, class Node>
void Ifpack2Smoother<Scalar, LocalOrdinal, GlobalOrdinal, Node>::SetupChebyshev(Level& currentLevel) {
typedef Tpetra::RowMatrix<SC,LO,GO,NO> tRowMatrix;
RCP<BlockedCrsMatrix> bA = rcp_dynamic_cast<BlockedCrsMatrix>(A_);
if (!bA.is_null())
A_ = bA->Merge();

RCP<const tRowMatrix> tA = Utilities::Op2NonConstTpetraRow(A_);

bool reusePreconditioner = false;

if (this->IsSetup() == true) {
this->GetOStream(Warnings0) << "MueLu::Ifpack2Smoother::SetupChebyshev(): SetupChebyshev() has already been called" << std::endl;
this->GetOStream(Warnings0) << "MueLu::Ifpack2Smoother::SetupChebyshev(): reuse of this type is not available, reverting to full construction" << std::endl;
// Reuse the constructed preconditioner
this->GetOStream(Runtime1) << "MueLu::Ifpack2Smoother::SetupChebyshev(): Setup() has already been called, assuming reuse" << std::endl;

RCP<Ifpack2::Details::CanChangeMatrix<tRowMatrix> > prec = rcp_dynamic_cast<Ifpack2::Details::CanChangeMatrix<tRowMatrix> >(prec_);
if (!prec.is_null()) {
prec->setMatrix(tA);
reusePreconditioner = true;
} else {
this->GetOStream(Warnings0) << "MueLu::Ifpack2Smoother::SetupChebyshev(): reuse of this type is not available (failed cast to CanChangeMatrix), "
"reverting to full construction" << std::endl;
}
}

typedef Teuchos::ScalarTraits<SC> STS;
Expand Down Expand Up @@ -680,14 +698,16 @@ namespace MueLu {
}
}

RCP<const Tpetra::RowMatrix<SC, LO, GO, NO> > tA = Utilities::Op2NonConstTpetraRow(A_);
if (!reusePreconditioner) {
prec_ = Ifpack2::Factory::create(type_, tA, overlap_);
SetPrecParameters();
{
SubFactoryMonitor(*this, "Preconditioner init", currentLevel);
prec_->initialize();
}
} else
SetPrecParameters();

prec_ = Ifpack2::Factory::create(type_, tA, overlap_);
SetPrecParameters();
{
SubFactoryMonitor(*this, "Preconditioner init", currentLevel);
prec_->initialize();
}
{
SubFactoryMonitor(*this, "Preconditioner compute", currentLevel);
prec_->compute();
Expand Down
Loading

0 comments on commit 7d08c7d

Please sign in to comment.