Skip to content

Commit

Permalink
Merge pull request #10161 from iyamazaki/shylu-basker2-clean
Browse files Browse the repository at this point in the history 
ShyLU,Basker : fix threaded factorization
  • Loading branch information
@hkthorn
hkthorn authored Feb 17, 2022
2 parents 55d7185 + 546201f commit 9be5241
Show file tree
Hide file tree
Showing 18 changed files with 999 additions and 597 deletions.
39 changes: 32 additions & 7 deletions packages/amesos2/src/Amesos2_ShyLUBasker_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,11 @@ ShyLUBasker<Matrix,Vector>::ShyLUBasker(
ShyLUbasker->Options.btf_matching = 2; // use cardinary matching from Trilinos, globally
ShyLUbasker->Options.blk_matching = 1; // use max-weight matching from Basker on each diagonal block
ShyLUbasker->Options.min_block_size = 0; // no merging small blocks
ShyLUbasker->Options.amd_dom = BASKER_TRUE; // use block-wise AMD
ShyLUbasker->Options.use_metis = BASKER_TRUE; // use scotch/metis for ND (TODO: should METIS optional?)
ShyLUbasker->Options.run_nd_on_leaves = BASKER_FALSE; // run ND on the final leaf-nodes
ShyLUbasker->Options.amd_dom = BASKER_TRUE; // use block-wise AMD
ShyLUbasker->Options.use_metis = BASKER_TRUE; // use scotch/metis for ND (TODO: should METIS optional?)
ShyLUbasker->Options.use_nodeNDP = BASKER_TRUE; // use nodeNDP to compute ND partition
ShyLUbasker->Options.run_nd_on_leaves = BASKER_TRUE; // run ND on the final leaf-nodes
ShyLUbasker->Options.run_amd_on_leaves = BASKER_FALSE; // run AMD on the final leaf-nodes
ShyLUbasker->Options.transpose = BASKER_FALSE;
ShyLUbasker->Options.replace_tiny_pivot = BASKER_TRUE;
ShyLUbasker->Options.verbose_matrix_out = BASKER_FALSE;
Expand All @@ -122,9 +124,11 @@ ShyLUBasker<Matrix,Vector>::ShyLUBasker(
ShyLUbaskerTr->Options.btf_matching = 2; // use cardinary matching from Trilinos, globally
ShyLUbaskerTr->Options.blk_matching = 1; // use max-weight matching from Basker on each diagonal block
ShyLUbaskerTr->Options.min_block_size = 0; // no merging small blocks
ShyLUbaskerTr->Options.amd_dom = BASKER_TRUE; // use block-wise AMD
ShyLUbaskerTr->Options.use_metis = BASKER_TRUE; // use scotch/metis for ND (TODO: should METIS optional?)
ShyLUbaskerTr->Options.run_nd_on_leaves = BASKER_FALSE; // run ND on the final leaf-nodes
ShyLUbaskerTr->Options.amd_dom = BASKER_TRUE; // use block-wise AMD
ShyLUbaskerTr->Options.use_metis = BASKER_TRUE; // use scotch/metis for ND (TODO: should METIS optional?)
ShyLUbaskerTr->Options.use_nodeNDP = BASKER_TRUE; // use nodeNDP to compute ND partition
ShyLUbaskerTr->Options.run_nd_on_leaves = BASKER_TRUE; // run ND on the final leaf-nodes
ShyLUbaskerTr->Options.run_amd_on_leaves = BASKER_FALSE; // run ND on the final leaf-nodes
ShyLUbaskerTr->Options.transpose = BASKER_TRUE;
ShyLUbaskerTr->Options.replace_tiny_pivot = BASKER_TRUE;
ShyLUbaskerTr->Options.verbose_matrix_out = BASKER_FALSE;
Expand Down Expand Up @@ -612,11 +616,26 @@ ShyLUBasker<Matrix,Vector>::setParameters_impl(const Teuchos::RCP<Teuchos::Param
ShyLUbasker->Options.use_metis = parameterList->get<bool>("use_metis");
ShyLUbaskerTr->Options.use_metis = parameterList->get<bool>("use_metis");
}
if(parameterList->isParameter("use_nodeNDP"))
{
ShyLUbasker->Options.use_nodeNDP = parameterList->get<bool>("use_nodeNDP");
ShyLUbaskerTr->Options.use_nodeNDP = parameterList->get<bool>("use_nodeNDP");
}
if(parameterList->isParameter("run_nd_on_leaves"))
{
ShyLUbasker->Options.run_nd_on_leaves = parameterList->get<bool>("run_nd_on_leaves");
ShyLUbaskerTr->Options.run_nd_on_leaves = parameterList->get<bool>("run_nd_on_leaves");
}
if(parameterList->isParameter("run_amd_on_leaves"))
{
ShyLUbasker->Options.run_amd_on_leaves = parameterList->get<bool>("run_amd_on_leaves");
ShyLUbaskerTr->Options.run_amd_on_leaves = parameterList->get<bool>("run_amd_on_leaves");
}
if(parameterList->isParameter("amd_on_blocks"))
{
ShyLUbasker->Options.amd_dom = parameterList->get<bool>("amd_on_blocks");
ShyLUbaskerTr->Options.amd_dom = parameterList->get<bool>("amd_on_blocks");
}
if(parameterList->isParameter("transpose"))
{
// NDE: set transpose vs non-transpose mode as bool; track separate shylubasker objects
Expand Down Expand Up @@ -713,8 +732,14 @@ ShyLUBasker<Matrix,Vector>::getValidParameters_impl() const
"Replace tiny pivots during the numerical factorization");
pl->set("use_metis", true,
"Use METIS for ND");
pl->set("use_nodeNDP", true,
"Use nodeND to compute ND partition");
pl->set("run_nd_on_leaves", false,
"Run ND on the final leaf-nodes");
"Run ND on the final leaf-nodes for ND factorization");
pl->set("run_amd_on_leaves", false,
"Run AMD on the final leaf-nodes for ND factorization");
pl->set("amd_on_blocks", true,
"Run AMD on each diagonal blocks");
pl->set("transpose", false,
"Solve the transpose A");
pl->set("use_sequential_diag_facto", false,
Expand Down
7 changes: 5 additions & 2 deletions packages/shylu/shylu_node/basker/src/shylubasker_decl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ namespace BaskerNS

BASKER_INLINE
void t_reset_BTF_factor(Int kid);
void t_reset_BTF_factor_top(Int start, Int end);

void t_reset_ND_factor(Int kid);

Expand Down Expand Up @@ -563,14 +564,16 @@ namespace BaskerNS
(
BASKER_MATRIX &M,
BASKER_SYMBOLIC_TREE &ST,
double fill_factor,
Int option
);

BASKER_INLINE
void L_assign_nnz
(
BASKER_MATRIX &M,
BASKER_SYMBOLIC_TREE &ST,
BASKER_SYMBOLIC_TREE &ST,
double fill_factor,
Int option
);

Expand Down Expand Up @@ -1049,7 +1052,7 @@ namespace BaskerNS
void t_local_reach_short_btf(const Int, const Int, Int &);

BASKER_INLINE
int t_local_reach_btf(Int, BASKER_MATRIX&,Int,Int&);
int t_local_reach_btf(Int, BASKER_MATRIX&,Int,Int&, Int,Int);

void t_prune_btf(const Int, const BASKER_MATRIX &, const BASKER_MATRIX&, const Int, const Int);

Expand Down
3 changes: 2 additions & 1 deletion packages/shylu/shylu_node/basker/src/shylubasker_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1079,7 +1079,8 @@ namespace BaskerNS
return BASKER_ERROR;
}
if(Options.verbose == BASKER_TRUE) {
std::cout << "Basker Factor: Time to permute and copy from input vals to new vals and blocks: " << copyperm_timer.seconds();
std::cout << "Basker Factor: Time to permute and copy from input vals to new vals and blocks: "
<< copyperm_timer.seconds() << std::endl;
}
//end sfactor_copy2 replacement stuff

Expand Down
5 changes: 4 additions & 1 deletion packages/shylu/shylu_node/basker/src/shylubasker_matrix_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -573,7 +573,7 @@ namespace BaskerNS
for(Int i = col_ptr(k-scol); i < M.col_ptr(k+1); i++)
{
Int j = M.row_idx(i);
/*if (kid == 3)
/*if (nrow == 289 && ncol == 100)
{
printf( " > row_idx[%d] = %d, val[%d] = %e (%d + %d) with k = %d+%d\n",i,j, i,M.val(i), srow,nrow, scol,k );
}*/
Expand Down Expand Up @@ -615,6 +615,9 @@ namespace BaskerNS
{
row_idx(temp_count) = j-srow;
val(temp_count) = M.val(i);
/*if (nrow == 289 && ncol == 100) {
printf( " (%d %e) k=%d nnz = %d\n",j-srow,M.val(i),k-scol,temp_count );
}*/

anorm_k += abs(M.val(i));
temp_count++;
Expand Down
11 changes: 11 additions & 0 deletions packages/shylu/shylu_node/basker/src/shylubasker_nfactor.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,10 @@ namespace BaskerNS
printf("Time DOMAIN: %lf \n", timer.seconds());
timer.reset();
}
#ifdef BASKER_TIMER
printf("Time DOMAIN: %lf \n", timer.seconds());
timer.reset();
#endif

#else// else basker_kokkos
#pragma omp parallel
Expand Down Expand Up @@ -272,6 +276,10 @@ namespace BaskerNS
printf("Time SEP: %lf \n", timer.seconds());
timer.reset();
}
#ifdef BASKER_TIMER
printf("Time SEP: %lf \n", timer.seconds());
timer.reset();
#endif
}

// ---------------------------------------------------------------------------------------- //
Expand Down Expand Up @@ -348,6 +356,9 @@ namespace BaskerNS
{
printf("Time BTF: %lf \n", timer.seconds());
}
#ifdef BASKER_TIMER
printf("Time BTF: %lf \n", timer.seconds());
#endif
}//end btf call

Kokkos::Timer tzback;
Expand Down
Loading

0 comments on commit 9be5241

Please sign in to comment.