Belos & Amesos2: changes following Tpetra_UVM_Removal branch

packages/amesos2/src/Amesos2_EpetraRowMatrix_AbstractMatrixAdapter_decl.hpp

@@ -127,6 +127,12 @@ namespace Amesos2 {
 			       const Teuchos::ArrayView<scalar_t>& vals,
 			       size_t& nnz) const;
 
+    template<typename KV_GO, typename KV_S>
+    void getGlobalRowCopy_kokkos_view_impl(global_ordinal_t row,
+                                           KV_GO & indices,
+                                           KV_S & vals,
+                                           size_t& nnz) const;
+
     global_size_t getGlobalNNZ_impl() const;
 
     size_t getLocalNNZ_impl() const;
@@ -175,29 +181,39 @@ namespace Amesos2 {
     // hands off implementation to the adapter for the subclass
     RCP<const super_t> get_impl(const Teuchos::Ptr<const Tpetra::Map<local_ordinal_t,global_ordinal_t,node_t> > map, EDistribution distribution = ROOTED) const;
 
-    typename super_t::spmtx_ptr_t  getSparseRowPtr() const;
+    #ifdef TPETRA_ENABLE_DEPRECATED_CODE
+    using spmtx_ptr_t = typename super_t::spmtx_ptr_t;
+    using spmtx_idx_t = typename super_t::spmtx_idx_t;
+    using spmtx_val_t = typename super_t::spmtx_vals_t;
+    #else
+    using spmtx_ptr_t = typename MatrixTraits<DerivedMat>::sparse_ptr_type;
+    using spmtx_idx_t = typename MatrixTraits<DerivedMat>::sparse_idx_type;
+    using spmtx_val_t = typename MatrixTraits<DerivedMat>::sparse_values_type;
+    #endif
+
+    spmtx_ptr_t  getSparseRowPtr() const;
 
-    typename super_t::spmtx_idx_t  getSparseColInd() const;
+    spmtx_idx_t  getSparseColInd() const;
 
-    typename super_t::spmtx_vals_t getSparseValues() const;
+    spmtx_val_t getSparseValues() const;
 
     template<class KV>
     void getSparseRowPtr_kokkos_view(KV & view) const {
-      Kokkos::View<typename super_t::spmtx_ptr_t, Kokkos::HostSpace> src(
+      Kokkos::View<spmtx_ptr_t, Kokkos::HostSpace> src(
         getSparseRowPtr(), getGlobalNumRows_impl()+1);
       deep_copy_or_assign_view(view, src);
     }
 
     template<class KV>
     void getSparseColInd_kokkos_view(KV & view) const {
-      Kokkos::View<typename super_t::spmtx_idx_t, Kokkos::HostSpace> src(
+      Kokkos::View<spmtx_idx_t, Kokkos::HostSpace> src(
         getSparseColInd(), getGlobalNNZ_impl());
       deep_copy_or_assign_view(view, src);
     }
 
     template<class KV>
     void getSparseValues_kokkos_view(KV & view) const {
-      Kokkos::View<typename super_t::spmtx_vals_t, Kokkos::HostSpace> src(
+      Kokkos::View<spmtx_val_t, Kokkos::HostSpace> src(
         getSparseValues(), getGlobalNNZ_impl());
       deep_copy_or_assign_view(view, src);
     }

packages/amesos2/src/Amesos2_EpetraRowMatrix_AbstractMatrixAdapter_def.hpp

@@ -161,6 +161,26 @@ namespace Amesos2 {
   }
 
 
+  template <class DerivedMat>
+  template<typename KV_GO, typename KV_S>
+  void
+  AbstractConcreteMatrixAdapter<
+    Epetra_RowMatrix,
+    DerivedMat>::getGlobalRowCopy_kokkos_view_impl(global_ordinal_t row,
+                                                   KV_GO & indices,
+                                                   KV_S & vals,
+                                                   size_t& nnz) const
+  {
+    using index_t = typename KV_GO::value_type;
+    using value_t = typename KV_S::value_type;
+    ArrayView<value_t>  vals_array    (vals.data(),    vals.extent(0));
+    ArrayView<index_t>  indices_array (indices.data(), indices.extent(0));
+
+    this->getGlobalRowCopy_impl(row, indices_array, vals_array, nnz);
+  }
+
+
+
   template <class DerivedMat>
   typename AbstractConcreteMatrixAdapter<
     Epetra_RowMatrix,
@@ -355,12 +375,12 @@ namespace Amesos2 {
 
   template <class DerivedMat>
   typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>
-  ::super_t::spmtx_ptr_t
+  ::spmtx_ptr_t
   AbstractConcreteMatrixAdapter<Epetra_RowMatrix, DerivedMat>::getSparseRowPtr() const
   {
-    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::super_t::spmtx_ptr_t  sp_rowptr = nullptr;
-    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::super_t::spmtx_idx_t  sp_colind = nullptr;
-    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::super_t::spmtx_vals_t sp_values = nullptr;
+    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::spmtx_ptr_t  sp_rowptr = nullptr;
+    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::spmtx_idx_t  sp_colind = nullptr;
+    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::spmtx_val_t  sp_values = nullptr;
 
     this->mat_->ExtractCrsDataPointers(sp_rowptr, sp_colind, sp_values);
 
@@ -369,12 +389,12 @@ namespace Amesos2 {
 
   template <class DerivedMat>
   typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>
-  ::super_t::spmtx_idx_t
+  ::spmtx_idx_t
   AbstractConcreteMatrixAdapter<Epetra_RowMatrix, DerivedMat>::getSparseColInd() const
   {
-    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::super_t::spmtx_ptr_t  sp_rowptr = nullptr;
-    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::super_t::spmtx_idx_t  sp_colind = nullptr;
-    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::super_t::spmtx_vals_t sp_values = nullptr;
+    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::spmtx_ptr_t  sp_rowptr = nullptr;
+    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::spmtx_idx_t  sp_colind = nullptr;
+    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::spmtx_val_t  sp_values = nullptr;
 
     this->mat_->ExtractCrsDataPointers(sp_rowptr, sp_colind, sp_values);
 
@@ -383,12 +403,12 @@ namespace Amesos2 {
 
   template <class DerivedMat>
   typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>
-  ::super_t::spmtx_vals_t
+  ::spmtx_val_t
   AbstractConcreteMatrixAdapter<Epetra_RowMatrix, DerivedMat>::getSparseValues() const
   {
-    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::super_t::spmtx_ptr_t  sp_rowptr = nullptr;
-    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::super_t::spmtx_idx_t  sp_colind = nullptr;
-    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::super_t::spmtx_vals_t sp_values = nullptr;
+    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::spmtx_ptr_t  sp_rowptr = nullptr;
+    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::spmtx_idx_t  sp_colind = nullptr;
+    typename AbstractConcreteMatrixAdapter<Epetra_RowMatrix,DerivedMat>::spmtx_val_t  sp_values = nullptr;
 
     this->mat_->ExtractCrsDataPointers(sp_rowptr, sp_colind, sp_values);
 

packages/amesos2/src/Amesos2_KLU2_def.hpp

@@ -499,7 +499,7 @@ KLU2<Matrix,Vector>::loadA_impl(EPhase current_phase)
 #endif
 
   // Only the root image needs storage allocated
-  if( this->root_ ){
+  if( this->root_ ) {
     host_nzvals_view_ = host_value_type_array(
       Kokkos::ViewAllocateWithoutInitializing("host_nzvals_view_"), this->globalNumNonZeros_);
     host_rows_view_ = host_ordinal_type_array(
@@ -528,8 +528,7 @@ KLU2<Matrix,Vector>::loadA_impl(EPhase current_phase)
     }
   }
 
-
-  if( this->root_ ){
+  if( this->root_ ) {
     TEUCHOS_TEST_FOR_EXCEPTION( nnz_ret != as<local_ordinal_type>(this->globalNumNonZeros_),
                         std::runtime_error,
                         "Did not get the expected number of non-zero vals");

packages/amesos2/src/Amesos2_KokkosCrsMatrix_MatrixAdapter_decl.hpp

@@ -85,9 +85,11 @@ namespace Amesos2 {
     typedef typename MatrixTraits<matrix_t>::global_ordinal_t global_ordinal_t;
     typedef typename MatrixTraits<matrix_t>::node_t                     node_t;
     typedef typename MatrixTraits<matrix_t>::global_size_t       global_size_t;
+    #ifdef TPETRA_ENABLE_DEPRECATED_CODE
     typedef typename MatrixTraits<matrix_t>::sparse_ptr_type       spmtx_ptr_t;
     typedef typename MatrixTraits<matrix_t>::sparse_idx_type       spmtx_idx_t;
     typedef typename MatrixTraits<matrix_t>::sparse_values_type   spmtx_vals_t;
+    #endif
     typedef no_special_impl                                       get_crs_spec;
     typedef no_special_impl                                       get_ccs_spec;
     typedef ConcreteMatrixAdapter<matrix_t>                               type;
@@ -101,9 +103,11 @@ namespace Amesos2 {
     global_size_t getGlobalNumRows_impl() const;
     global_size_t getGlobalNumCols_impl() const;
     global_size_t getGlobalNNZ_impl() const;
+    #ifdef TPETRA_ENABLE_DEPRECATED_CODE
     spmtx_ptr_t  getSparseRowPtr() const;
     spmtx_idx_t  getSparseColInd() const;
     spmtx_vals_t getSparseValues() const;
+    #endif
 
     template<class KV>
     void getSparseRowPtr_kokkos_view(KV & view) const {
@@ -154,6 +158,13 @@ namespace Amesos2 {
              const Teuchos::ArrayView<scalar_t>& vals,
              size_t& nnz) const;
 
+
+    template <typename KV_GO, typename KV_S>
+    void getGlobalRowCopy_kokkos_view_impl(global_ordinal_t row,
+                                           KV_GO & indices,
+                                           KV_S & vals,
+                                           size_t& nnz) const;
+
   };
 
 } // end namespace Amesos2

packages/amesos2/src/Amesos2_KokkosCrsMatrix_MatrixAdapter_def.hpp

@@ -189,6 +189,7 @@ namespace Amesos2 {
   }
 
   // implementation functions
+  #ifdef TPETRA_ENABLE_DEPRECATED_CODE
   template <typename Scalar, typename LocalOrdinal, typename ExecutionSpace>
   void
   ConcreteMatrixAdapter<
@@ -202,7 +203,25 @@ namespace Amesos2 {
                         "getGlobalRowCopy_impl not implemented for Kokkos CrsMatrix yet.  "
                         "Please contact the Amesos2 developers." );
     }
+  #endif
 
+  template <typename Scalar, typename LocalOrdinal, typename ExecutionSpace>
+  template <typename KV_GO, typename KV_S>
+  void
+  ConcreteMatrixAdapter<
+    KokkosSparse::CrsMatrix<Scalar,LocalOrdinal,ExecutionSpace>>::getGlobalRowCopy_kokkos_view_impl(global_ordinal_t row,
+                                                   KV_GO & indices,
+                                                   KV_S & vals,
+                                                   size_t& nnz) const
+    {
+      TEUCHOS_TEST_FOR_EXCEPTION( true,
+                        std::runtime_error,
+                        "getGlobalRowCopy_kokkos_view_impl not implemented for Kokkos CrsMatrix yet.  "
+                        "Please contact the Amesos2 developers." );
+    }
+
+
+  #ifdef TPETRA_ENABLE_DEPRECATED_CODE
   template <typename Scalar, typename LocalOrdinal, typename ExecutionSpace>
   void
   ConcreteMatrixAdapter<
@@ -237,6 +256,7 @@ namespace Amesos2 {
   {
     return this->mat_->values.data();
   }
+  #endif
 
 } // end namespace Amesos2
 

packages/amesos2/src/Amesos2_Kokkos_View_Copy_Assign.hpp

@@ -88,6 +88,30 @@ implement_copy_or_assign_same_mem_check_types(bool bInitialize, dst_t & dst, con
   bAssigned = true;
 }
 
+
+// deep-copy version (no checking)
+// bInitialize:
+//   If bInitialize is false, then the data needs to be allocated but not initialized.
+//   If we are about to solve into x we don't care about setting the original values.
+//   In this case, we are allocating so we first make the memory via update_dst_size.
+//   Then we only copy from the source if bInitialize is true.
+// bAssigned:
+//   bAssigned tells the caller if the data was simply assigned, so it is set false in this case.
+template<class dst_t, class src_t> // actual implementation
+void deep_copy_only(bool bInitialize, dst_t & dst, const src_t & src, bool & bAssigned) {
+  update_dst_size(dst, src); // allocates if necessary
+  if(bInitialize) { // bInitialize false would be for solver getting x, where the actual values are not needed
+    Kokkos::deep_copy(dst, src); // full copy
+  }
+  bAssigned = false;
+}
+
+template<class dst_t, class src_t> // actual implementation
+void deep_copy_only(dst_t & dst, const src_t & src) {
+  bool bAssigned;
+  deep_copy_only(true, dst, src, bAssigned);
+}
+
 // now handle type mismatch for same memory space - now types are different
 // bInitialize:
 //   If bInitialize is false, then the data needs to be allocated but not initialized.