Skip to content

Commit

Permalink
Merge Pull Request trilinos#9485 from trilinos/Trilinos/master_merge_…
Browse files Browse the repository at this point in the history 
…20210726_000548

Automatically Merged using Trilinos Master Merge AutoTester
PR Title: Trilinos Master Merge PR Generator: Auto PR created to promote from master_merge_20210726_000548 branch to master
PR Author: trilinos-autotester
  • Loading branch information
@trilinos-autotester @JacobDomagala
trilinos-autotester authored and JacobDomagala committed Jul 30, 2021
2 parents 4c26aae + 42c4b38 commit f6517be
Show file tree
Hide file tree
Showing 63 changed files with 1,910 additions and 689 deletions.
1 change: 0 additions & 1 deletion cmake/std/PullRequestLinuxCuda10.1.105uvmOffTestingSettings.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,6 @@ set (Sacado_ENABLE_TESTS OFF CACHE BOOL "Turn off tests for non-UVM build")
set (SEACAS_ENABLE_TESTS OFF CACHE BOOL "Turn off tests for non-UVM build")
set (ShyLU_DD_ENABLE_TESTS OFF CACHE BOOL "Turn off tests for non-UVM build")
set (STK_ENABLE_TESTS OFF CACHE BOOL "Turn off tests for non-UVM build")
set (Zoltan2_ENABLE_TESTS OFF CACHE BOOL "Turn off tests for non-UVM build")


# ShyLU_DD UVM = OFF tests
Expand Down
69 changes: 60 additions & 9 deletions packages/ifpack2/adapters/thyra/Thyra_Ifpack2PreconditionerFactory_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
#include "Ifpack2_Parameters.hpp"

#include "Tpetra_RowMatrix.hpp"
#include "Tpetra_MixedScalarMultiplyOp.hpp"

#include "Teuchos_TestForException.hpp"
#include "Teuchos_Assert.hpp"
Expand Down Expand Up @@ -160,6 +161,10 @@ void Ifpack2PreconditionerFactory<MatrixType>::initializePrec(

// Process parameter list

bool useHalfPrecision = false;
if (paramList_->isParameter("half precision"))
useHalfPrecision = paramList_->get<bool>("half precision");

Teuchos::RCP<const Teuchos::ParameterList> constParamList = paramList_;
if (constParamList.is_null ()) {
constParamList = getValidParameters ();
Expand Down Expand Up @@ -195,11 +200,41 @@ void Ifpack2PreconditionerFactory<MatrixType>::initializePrec(
}
timer.start(true);

Teuchos::RCP<LinearOpBase<scalar_type> > thyraPrecOp;

typedef Ifpack2::Preconditioner<scalar_type, local_ordinal_type, global_ordinal_type, node_type> Ifpack2Prec;
typedef Tpetra::RowMatrix<scalar_type, local_ordinal_type,
global_ordinal_type, node_type> row_matrix_type;
const Teuchos::RCP<Ifpack2Prec> concretePrecOp =
Ifpack2::Factory::create<row_matrix_type> (preconditionerType, tpetraFwdMatrix);
Teuchos::RCP<Ifpack2Prec> concretePrecOp;

#if defined(HAVE_TPETRA_INST_DOUBLE) && defined(HAVE_TPETRA_INST_FLOAT)
// CAG: There is nothing special about the combination double-float,
// except that I feel somewhat confident that Trilinos builds
// with both scalar types.
typedef typename Teuchos::ScalarTraits<scalar_type>::halfPrecision half_scalar_type;
typedef Ifpack2::Preconditioner<half_scalar_type, local_ordinal_type, global_ordinal_type, node_type> HalfIfpack2Prec;
Teuchos::RCP<HalfIfpack2Prec> concretePrecOpHalf;
#endif

if (useHalfPrecision) {
#if defined(HAVE_TPETRA_INST_DOUBLE) && defined(HAVE_TPETRA_INST_FLOAT)
if (Teuchos::nonnull(out) && Teuchos::includesVerbLevel(verbLevel, Teuchos::VERB_LOW)) {
Teuchos::OSTab(out).o() << "> Creating half precision preconditioner\n";
}


typedef Tpetra::RowMatrix<half_scalar_type, local_ordinal_type,
global_ordinal_type, node_type> row_matrix_type;
auto tpetraFwdMatrixHalf = tpetraFwdMatrix->template convert<half_scalar_type>();
concretePrecOpHalf =
Ifpack2::Factory::create<row_matrix_type> (preconditionerType, tpetraFwdMatrixHalf);
#else
TEUCHOS_TEST_FOR_EXCEPT(true);
#endif
} else {
typedef Tpetra::RowMatrix<scalar_type, local_ordinal_type,
global_ordinal_type, node_type> row_matrix_type;
concretePrecOp =
Ifpack2::Factory::create<row_matrix_type> (preconditionerType, tpetraFwdMatrix);
}

timer.stop();
if (Teuchos::nonnull(out) && Teuchos::includesVerbLevel(verbLevel, Teuchos::VERB_LOW)) {
Expand All @@ -208,13 +243,25 @@ void Ifpack2PreconditionerFactory<MatrixType>::initializePrec(

// Initialize and compute the initial preconditioner

concretePrecOp->setParameters(*packageParamList);
concretePrecOp->initialize();
concretePrecOp->compute();
if (useHalfPrecision) {
#if defined(HAVE_TPETRA_INST_DOUBLE) && defined(HAVE_TPETRA_INST_FLOAT)
concretePrecOpHalf->setParameters(*packageParamList);
concretePrecOpHalf->initialize();
concretePrecOpHalf->compute();

// Wrap concrete preconditioner
RCP<TpetraLinOp> wrappedOp = rcp(new Tpetra::MixedScalarMultiplyOp<scalar_type,half_scalar_type,local_ordinal_type,global_ordinal_type,node_type>(concretePrecOpHalf));

thyraPrecOp = Thyra::createLinearOp(wrappedOp);
#endif
} else {
concretePrecOp->setParameters(*packageParamList);
concretePrecOp->initialize();
concretePrecOp->compute();

// Wrap concrete preconditioner
thyraPrecOp = Thyra::createLinearOp(Teuchos::RCP<TpetraLinOp>(concretePrecOp));
}

const Teuchos::RCP<LinearOpBase<scalar_type> > thyraPrecOp = Thyra::createLinearOp(Teuchos::RCP<TpetraLinOp>(concretePrecOp));
defaultPrec->initializeUnspecified(thyraPrecOp);

totalTimer.stop();
Expand Down Expand Up @@ -321,6 +368,10 @@ Ifpack2PreconditionerFactory<MatrixType>::getValidParameters() const
"Number of rows/columns overlapped between subdomains in different"
"\nprocesses in the additive Schwarz-type domain-decomposition preconditioners."
);
validParamList->set(
"half precision", false,
"Whether a half-precision preconditioner should be built."
);
Teuchos::ParameterList &packageParamList = validParamList->sublist(
"Ifpack2 Settings", false,
"Preconditioner settings that are passed onto the Ifpack preconditioners themselves."
Expand Down
4 changes: 4 additions & 0 deletions packages/muelu/adapters/stratimikos/Thyra_MueLuPreconditionerFactory_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,10 @@ namespace Thyra {
if (useHalfPrecision) {
#if defined(HAVE_MUELU_TPETRA) && defined(HAVE_TPETRA_INST_DOUBLE) && defined(HAVE_TPETRA_INST_FLOAT)

// CAG: There is nothing special about the combination double-float,
// except that I feel somewhat confident that Trilinos builds
// with both scalar types.

// convert to half precision
RCP<XphMat> halfA = Xpetra::convertToHalfPrecision(A);
const std::string userName = "user data";
Expand Down
13 changes: 9 additions & 4 deletions packages/muelu/src/Utils/MueLu_Utilities_kokkos_decl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ namespace MueLu {
NOTE -- it's assumed that A has been fillComplete'd.
*/
static Teuchos::ArrayRCP<SC> GetMatrixDiagonal(const Matrix& A); // FIXME
static RCP<Vector> GetMatrixDiagonal(const Matrix& A); // FIXME

/*! @brief Extract Matrix Diagonal
Expand Down Expand Up @@ -271,7 +271,7 @@ namespace MueLu {
static void ZeroDirichletRows(RCP<MultiVector>& X, const Kokkos::View<const bool*, typename NO::device_type>& dirichletRows, SC replaceWith=Teuchos::ScalarTraits<SC>::zero());

static void ZeroDirichletCols(RCP<Matrix>& A, const Kokkos::View<const bool*, typename NO::device_type>& dirichletCols, SC replaceWith=Teuchos::ScalarTraits<SC>::zero());

static void ApplyRowSumCriterion(const Matrix& A,
const typename Teuchos::ScalarTraits<Scalar>::magnitudeType rowSumTol,
Kokkos::View<bool*, typename NO::device_type> & dirichletRows);
Expand Down Expand Up @@ -382,8 +382,13 @@ namespace MueLu {
#endif
static RCP<Xpetra::Matrix<SC,LO,GO,NO> > Crs2Op(RCP<CrsMatrix> Op) { return Utilities::Crs2Op(Op); }

static ArrayRCP<SC> GetMatrixDiagonal(const Matrix& A) {
return UtilitiesBase::GetMatrixDiagonal(A);
static RCP<Vector> GetMatrixDiagonal(const Matrix& A) {
const auto rowMap = A.getRowMap();
auto diag = Xpetra::VectorFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node>::Build(rowMap,true);

A.getLocalDiagCopy(*diag);

return diag;
}
static RCP<Vector> GetMatrixDiagonalInverse(const Matrix& A, Magnitude tol = Teuchos::ScalarTraits<SC>::eps()*100, const bool doLumped=false) {
return UtilitiesBase::GetMatrixDiagonalInverse(A, tol, doLumped);
Expand Down
57 changes: 24 additions & 33 deletions packages/muelu/src/Utils/MueLu_Utilities_kokkos_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,30 +112,13 @@ namespace MueLu {


template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
Teuchos::ArrayRCP<Scalar> Utilities_kokkos<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
Teuchos::RCP<Xpetra::Vector<Scalar,LocalOrdinal,GlobalOrdinal,Node> >
Utilities_kokkos<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
GetMatrixDiagonal(const Matrix& A) {
// FIXME_KOKKOS
const auto rowMap = A.getRowMap();
auto diag = Xpetra::VectorFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node>::Build(rowMap, true);

size_t numRows = A.getRowMap()->getNodeNumElements();
Teuchos::ArrayRCP<SC> diag(numRows);

Teuchos::ArrayView<const LO> cols;
Teuchos::ArrayView<const SC> vals;
for (size_t i = 0; i < numRows; ++i) {
A.getLocalRowView(i, cols, vals);

LO j = 0;
for (; j < cols.size(); ++j) {
if (Teuchos::as<size_t>(cols[j]) == i) {
diag[i] = vals[j];
break;
}
}
if (j == cols.size()) {
// Diagonal entry is absent
diag[i] = Teuchos::ScalarTraits<SC>::zero();
}
}
A.getLocalDiagCopy(*diag);

return diag;
} //GetMatrixDiagonal
Expand Down Expand Up @@ -226,11 +209,19 @@ namespace MueLu {
crsOp->getLocalDiagCopy(*localDiag,offsets());
}
else {
ArrayRCP<SC> localDiagVals = localDiag->getDataNonConst(0);
Teuchos::ArrayRCP<SC> diagVals = GetMatrixDiagonal(A);
for (LO i = 0; i < localDiagVals.size(); i++)
localDiagVals[i] = diagVals[i];
localDiagVals = diagVals = null;
using local_matrix_type = typename Matrix::local_matrix_type;
using ordinal_type = typename local_matrix_type::ordinal_type;
using execution_space = typename local_matrix_type::execution_space;

auto localDiagVals = localDiag->getDeviceLocalView(Xpetra::Access::ReadWrite);
const auto diagVals = GetMatrixDiagonal(A)->getDeviceLocalView(Xpetra::Access::ReadOnly);
Kokkos::deep_copy(localDiagVals, diagVals);

// Kokkos::parallel_for("Utilities_kokkos::GetMatrixOverlappedDiagonal",
// Kokkos::RangePolicy<ordinal_type, execution_space>(0, diagVals.extent_int(1)),
// KOKKOS_LAMBDA(const ordinal_type idx) {
// localDiagVals(idx, 0) = diagVals(idx, 0);
// });
}

RCP<Vector> diagonal = VectorFactory::Build(colMap);
Expand Down Expand Up @@ -612,7 +603,7 @@ namespace MueLu {
return MueLu::ZeroDirichletCols<double,int,int,Node>(A, dirichletCols, replaceWith);
}

// Applies rowsum criterion
// Applies rowsum criterion
template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
void ApplyRowSumCriterion(const Xpetra::Matrix<Scalar,LocalOrdinal,GlobalOrdinal,Node>& A,
const typename Teuchos::ScalarTraits<Scalar>::magnitudeType rowSumTol,
Expand Down Expand Up @@ -711,7 +702,7 @@ namespace MueLu {
<device, typename local_graph_type::row_map_type, typename local_graph_type::entries_type, lno_nnz_view_t>
(localGraph.row_map, localGraph.entries);

RCP<Xpetra::Vector<LocalOrdinal,LocalOrdinal,GlobalOrdinal,Node> > retval =
RCP<Xpetra::Vector<LocalOrdinal,LocalOrdinal,GlobalOrdinal,Node> > retval =
Xpetra::VectorFactory<LocalOrdinal,LocalOrdinal,GlobalOrdinal,Node>::Build(Op.getRowMap());

// Copy out and reorder data
Expand All @@ -723,7 +714,7 @@ namespace MueLu {
});
return retval;
}

template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
Teuchos::RCP<Xpetra::Vector<LocalOrdinal,LocalOrdinal,GlobalOrdinal,Node> > CuthillMcKee(const Xpetra::Matrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> &Op) {
using local_matrix_type = typename Xpetra::Matrix<Scalar,LocalOrdinal,GlobalOrdinal,Node>::local_matrix_type;
Expand All @@ -740,7 +731,7 @@ namespace MueLu {
<device, typename local_graph_type::row_map_type, typename local_graph_type::entries_type, lno_nnz_view_t>
(localGraph.row_map, localGraph.entries);

RCP<Xpetra::Vector<LocalOrdinal,LocalOrdinal,GlobalOrdinal,Node> > retval =
RCP<Xpetra::Vector<LocalOrdinal,LocalOrdinal,GlobalOrdinal,Node> > retval =
Xpetra::VectorFactory<LocalOrdinal,LocalOrdinal,GlobalOrdinal,Node>::Build(Op.getRowMap());

// Copy out data
Expand All @@ -761,7 +752,7 @@ namespace MueLu {
}

template <class Node>
Teuchos::RCP<Xpetra::Vector<int,int,int,Node> >
Teuchos::RCP<Xpetra::Vector<int,int,int,Node> >
Utilities_kokkos<double,int,int,Node>::ReverseCuthillMcKee(const Matrix &Op) {
return MueLu::ReverseCuthillMcKee<double,int,int,Node>(Op);
}
Expand All @@ -773,7 +764,7 @@ namespace MueLu {
}

template <class Node>
Teuchos::RCP<Xpetra::Vector<int,int,int,Node> >
Teuchos::RCP<Xpetra::Vector<int,int,int,Node> >
Utilities_kokkos<double,int,int,Node>::CuthillMcKee(const Matrix &Op) {
return MueLu::CuthillMcKee<double,int,int,Node>(Op);
}
Expand Down
2 changes: 1 addition & 1 deletion packages/piro/src/Piro_TempusIntegrator_Def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ Piro::TempusIntegrator<Scalar>::TempusIntegrator(Teuchos::RCP< Teuchos::Paramete
else if (sens_method == FORWARD) {
//forward sensitivities
basicIntegrator_ = Teuchos::null;
fwdSensIntegrator_ = Tempus::integratorForwardSensitivity<Scalar>(pList, model);
fwdSensIntegrator_ = Tempus::createIntegratorForwardSensitivity<Scalar>(pList, model);
adjSensIntegrator_ = Teuchos::null;
}
else if (sens_method == ADJOINT) {
Expand Down
10 changes: 5 additions & 5 deletions packages/rol/example/tempus/ROL_TempusReducedObjective.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
#include "Teuchos_RCP.hpp"
#include "Teuchos_ParameterList.hpp"

#include "Tempus_IntegratorBasicOld.hpp"
#include "Tempus_IntegratorBasic.hpp"
#include "Tempus_IntegratorForwardSensitivity.hpp"
#include "Tempus_IntegratorAdjointSensitivity.hpp"
#include "Tempus_IntegratorPseudoTransientForwardSensitivity.hpp"
Expand Down Expand Up @@ -361,7 +361,7 @@ run_tempus(const Thyra::ModelEvaluatorBase::InArgs<Real>& inArgs,
// Create and run integrator
if (dgdp != Teuchos::null && sensitivity_method_ == "Forward") {
RCP<Tempus::IntegratorForwardSensitivity<Real> > integrator =
Tempus::integratorForwardSensitivity<Real>(tempus_params_, wrapped_model);
Tempus::createIntegratorForwardSensitivity<Real>(tempus_params_, wrapped_model);
const bool integratorStatus = integrator->advanceTime();
TEUCHOS_TEST_FOR_EXCEPTION(
!integratorStatus, std::logic_error, "Integrator failed!");
Expand Down Expand Up @@ -389,7 +389,7 @@ run_tempus(const Thyra::ModelEvaluatorBase::InArgs<Real>& inArgs,
else if (dgdp != Teuchos::null &&
sensitivity_method_ == "Pseudotransient Forward") {
RCP<Tempus::IntegratorPseudoTransientForwardSensitivity<Real> > integrator =
Tempus::integratorPseudoTransientForwardSensitivity<Real>(tempus_params_,
Tempus::createIntegratorPseudoTransientForwardSensitivity<Real>(tempus_params_,
wrapped_model);
const bool integratorStatus = integrator->advanceTime();
TEUCHOS_TEST_FOR_EXCEPTION(
Expand Down Expand Up @@ -425,8 +425,8 @@ run_tempus(const Thyra::ModelEvaluatorBase::InArgs<Real>& inArgs,
<< " and Pseudotransient Adjoint.");
}
else {
RCP<Tempus::IntegratorBasicOld<Real> > integrator =
Tempus::integratorBasic<Real>(tempus_params_, wrapped_model);
RCP<Tempus::IntegratorBasic<Real> > integrator =
Tempus::createIntegratorBasic<Real>(tempus_params_, wrapped_model);
const bool integratorStatus = integrator->advanceTime();
TEUCHOS_TEST_FOR_EXCEPTION(
!integratorStatus, std::logic_error, "Integrator failed!");
Expand Down
4 changes: 2 additions & 2 deletions packages/rol/example/tempus/example_01.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
#include "Teuchos_GlobalMPISession.hpp"
#include "Teuchos_XMLParameterListHelpers.hpp"

#include "Tempus_IntegratorBasicOld.hpp"
#include "Tempus_IntegratorBasic.hpp"

#include <fstream>

Expand Down Expand Up @@ -96,7 +96,7 @@ int main(int argc, char *argv[]) {
RCP<Teuchos::ParameterList> tempusParList = sublist(parList, "Tempus", true);
RCP<SinCosModelEvaluator<RealT>> model = Teuchos::rcp(new SinCosModelEvaluator<RealT>());

RCP<Tempus::IntegratorBasicOld<RealT> > integrator = Tempus::integratorBasic<RealT>(tempusParList, model);
RCP<Tempus::IntegratorBasic<RealT> > integrator = Tempus::createIntegratorBasic<RealT>(tempusParList, model);

integrator->advanceTime();

Expand Down
6 changes: 1 addition & 5 deletions packages/rol/example/tempus/example_parabolic.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@
<Parameter name="Initial Condition Consistency Check" type="bool" value="false"/>
<Parameter name="Zero Initial Guess" type="bool" value="0"/>
<Parameter name="Solver Name" type="string" value="Default Solver"/>
<Parameter name="Predictor Name" type="string" value="Default Predictor"/>
<Parameter name="Predictor Stepper Type" type="string" value="Forward Euler"/>

<ParameterList name="Default Solver">
<ParameterList name="NOX">
Expand Down Expand Up @@ -122,10 +122,6 @@
</ParameterList>
</ParameterList>

<ParameterList name="Default Predictor">
<Parameter name="Stepper Type" type="string" value="Forward Euler"/>
</ParameterList>

</ParameterList>

</ParameterList>
Expand Down
4 changes: 2 additions & 2 deletions packages/rol/example/tempus/example_parabolic_modeleval.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@

#include "Teuchos_GlobalMPISession.hpp"

#include "Tempus_IntegratorBasicOld.hpp"
#include "Tempus_IntegratorBasic.hpp"

#include "ROL_Stream.hpp"
#include "ROL_ParameterList.hpp"
Expand Down Expand Up @@ -114,7 +114,7 @@ int main(int argc, char *argv[]) {

// Create Tempus Integrator from ModelEvaluator.
ROL::Ptr<Tempus::Integrator<RealT>> integrator =
ROL::makePtr<Tempus::IntegratorBasicOld<RealT>>(ROL::makePtrFromRef(pl_tempus), meval);
Tempus::createIntegratorBasic<RealT>(ROL::makePtrFromRef(pl_tempus), meval);

// Initialize objective function.
ROL::Ptr<ROL::DynamicObjective<RealT>> dyn_obj =
Expand Down
Loading

0 comments on commit f6517be

Please sign in to comment.