Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Zoltan2: Distributed Multi-GPU coloring algorithms for D1, D2 and PD2 #8957

Merged
merged 25 commits into from
May 18, 2021
Merged

Zoltan2: Distributed Multi-GPU coloring algorithms for D1, D2 and PD2 #8957

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
184949e
Code from AiMOS integrated with a recent clone of develop. All tests …
ibogle Apr 2, 2021
0da3f80
Fixed type warnings and removed some unnecessary KokkosKernels changes.
ibogle Apr 3, 2021
3410063
Added 2GL base class for D1-2GL, D2, PD2 to reduce code duplication.
ibogle Apr 14, 2021
70ae7f0
Made member functions containing GPU kernels public to resolve build …
ibogle Apr 15, 2021
54d5945
Removed commented-out code in KokkosKernels changes
ibogle Apr 16, 2021
b5bff11
Applied Karen's initial feedback:
ibogle Apr 19, 2021
d531239
Started revising the coloring algorithms
ibogle Apr 20, 2021
c3d88fc
Updated D1 code:
ibogle Apr 21, 2021
1a2ae85
Updated 2GL subclasses:
ibogle Apr 22, 2021
338ca39
Added comments to AlgHybrid2GL constructSecondGhostLayer and doOwnedT…
ibogle Apr 22, 2021
3218a39
Addressed some of Erik's more minor
ibogle Apr 22, 2021
93e7ef1
Revised usage of host and device views for 2GL
ibogle Apr 22, 2021
e02f78c
Added comments, made sure to address Karen's feedback.
ibogle Apr 22, 2021
4d28ab3
Commented/cleaned up D1.
ibogle Apr 23, 2021
794c60a
Made a change to class member access to solve build errors
ibogle Apr 23, 2021
c05259b
Modified member access to D1 to fix GPU build errors
ibogle Apr 24, 2021
b368cf0
Switched execution and memory spaces used in serial function calls.
ibogle Apr 26, 2021
92bb965
Further modified the serial exec/memory spaces to solve build problems
ibogle Apr 27, 2021
f827765
Change to Kokkos size views to fix build errors
ibogle Apr 27, 2021
ee123da
Switched D2 and PD2 to always use NB_BIT, as our previous implementat…
ibogle May 5, 2021
459db9e
Changed procs_to_send argument of communication function to pass by r…
ibogle May 10, 2021
d182c96
Fixed boundary_size, and atomics.
ibogle May 11, 2021
48c06da
Nearly done with Karen's feedback.
ibogle May 13, 2021
6d58faf
Addressed all of Karen's feedback, and made equivalent changes to D1 …
ibogle May 14, 2021
a61bfec
Removed template keyword to fix build errors
ibogle May 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions packages/kokkos-kernels/src/graph/KokkosGraph_Distance1Color.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,12 @@ void graph_color_symbolic(

gch->set_tictoc(handle->get_verbose());

color_view_type colors_out = color_view_type("Graph Colors", num_rows);

color_view_type colors_out; //= color_view_type("Graph Colors", num_rows);
if(gch->get_vertex_colors().use_count() > 0){
colors_out = gch->get_vertex_colors();
} else {
colors_out = color_view_type("Graph Colors", num_rows);
}

typedef typename Impl::GraphColor
<typename KernelHandle::GraphColoringHandleType, lno_row_view_t_, lno_nnz_view_t_> BaseGraphColoring;
Expand Down
14 changes: 13 additions & 1 deletion packages/kokkos-kernels/src/graph/KokkosGraph_Distance1ColorHandle.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,10 @@ class GraphColoringHandle
nnz_lno_persistent_work_view_t lower_triangle_src;
nnz_lno_persistent_work_view_t lower_triangle_dst;

bool use_vtx_list;
nnz_lno_temp_work_view_t vertex_list;
size_type vertex_list_size;

color_view_t vertex_colors;
bool is_coloring_called_before;
nnz_lno_t num_colors;
Expand Down Expand Up @@ -188,7 +192,7 @@ class GraphColoringHandle
overall_coloring_time_phase4(0),
overall_coloring_time_phase5(0),
coloring_time(0),
num_phases(0), size_of_edge_list(0), lower_triangle_src(), lower_triangle_dst(),
num_phases(0), size_of_edge_list(0), lower_triangle_src(), lower_triangle_dst(), use_vtx_list(false),
vertex_colors(), is_coloring_called_before(false), num_colors(0)
{
this->choose_default_algorithm();
Expand Down Expand Up @@ -640,7 +644,15 @@ class GraphColoringHandle
int get_num_phases() const { return this->num_phases;}
color_view_t get_vertex_colors() const {return this->vertex_colors;}
bool is_coloring_called() const {return this->is_coloring_called_before;}
bool get_use_vtx_list() const{return this->use_vtx_list;}
nnz_lno_temp_work_view_t get_vertex_list() const {return this->vertex_list;}
size_type get_vertex_list_size() const{return this->vertex_list_size;}
//setters
void set_vertex_list(nnz_lno_temp_work_view_t vertex_list_, size_type vertex_list_size_){
this->vertex_list = vertex_list_;
this->vertex_list_size = vertex_list_size_;
this->use_vtx_list = true;
}
void set_coloring_algo_type(const ColoringAlgorithm &col_algo){this->coloring_algorithm_type = col_algo;}
void set_conflict_list_type(const ConflictList &cl){this->conflict_list_type = cl;}
void set_min_reduction_for_conflictlist(const double &min_reduction){this->min_reduction_for_conflictlist = min_reduction;}
Expand Down
14 changes: 14 additions & 0 deletions packages/kokkos-kernels/src/graph/KokkosGraph_Distance2ColorHandle.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,10 @@ class GraphColorDistance2Handle
double overall_coloring_time_phase5; //
double coloring_time; // the time that it took to color the graph

bool use_vtx_list;
nnz_lno_temp_work_view_type vertex_list;
size_type vertex_list_size;

int num_phases; // Number of phases used by the coloring algorithm

color_view_type vertex_colors;
Expand All @@ -144,6 +148,7 @@ class GraphColorDistance2Handle
, overall_coloring_time_phase4(0)
, overall_coloring_time_phase5(0)
, coloring_time(0)
, use_vtx_list(false)
, num_phases(0)
, vertex_colors()
, is_coloring_called_before(false)
Expand Down Expand Up @@ -282,7 +287,16 @@ class GraphColorDistance2Handle

bool is_coloring_called() const { return this->is_coloring_called_before; }

bool get_use_vtx_list() const {return this->use_vtx_list;}
nnz_lno_temp_work_view_type get_vertex_list() const {return this->vertex_list;}
size_type get_vertex_list_size() const {return this->vertex_list_size;}

// setters
void set_vertex_list(nnz_lno_temp_work_view_type vertex_list_, size_type vertex_list_size_){
this->vertex_list = vertex_list_;
this->vertex_list_size = vertex_list_size_;
this->use_vtx_list = true;
}
void set_coloring_called() { this->is_coloring_called_before = true; }

void set_coloring_algo_type(const GraphColoringAlgorithmDistance2& col_algo) { this->coloring_algorithm_type = col_algo; }
Expand Down
29 changes: 19 additions & 10 deletions packages/kokkos-kernels/src/graph/impl/KokkosGraph_Distance1Color_impl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,11 +367,16 @@ class GraphColor_VB:public GraphColor <HandleType,lno_row_view_t_,lno_nnz_view_t
//the conflictlist
nnz_lno_temp_work_view_t current_vertexList =
nnz_lno_temp_work_view_t(Kokkos::ViewAllocateWithoutInitializing("vertexList"), this->nv);
nnz_lno_t current_vertexListLength = this->nv;

//init vertexList sequentially.
Kokkos::parallel_for("KokkosGraph::GraphColoring::InitList",
my_exec_space(0, this->nv), functorInitList<nnz_lno_temp_work_view_t> (current_vertexList));

if(this->cp->get_use_vtx_list()){
current_vertexList = this->cp->get_vertex_list();
current_vertexListLength = this->cp->get_vertex_list_size();
} else {
Kokkos::parallel_for("KokkosGraph::GraphColoring::InitList",
my_exec_space(0, this->nv), functorInitList<nnz_lno_temp_work_view_t> (current_vertexList));
}

// the next iteration's conflict list
nnz_lno_temp_work_view_t next_iteration_recolorList;
Expand All @@ -388,7 +393,6 @@ class GraphColor_VB:public GraphColor <HandleType,lno_row_view_t_,lno_nnz_view_t
}

nnz_lno_t numUncolored = this->nv;
nnz_lno_t current_vertexListLength = this->nv;


double t, total=0.0;
Expand Down Expand Up @@ -2304,7 +2308,7 @@ class GraphColor_EB:public GraphColor <HandleType,in_row_index_view_type_,in_non
nnz_lno_temp_work_view_t color_set ("color_set", this->nv); //initialized with zero.
//initialize colors, color bans
Kokkos::parallel_for ("KokkosGraph::GraphColoring::initColors",
my_exec_space (0, this->nv) , init_colors (kok_colors, color_ban, numInitialColors));
my_exec_space (0, this->nv) , init_colors (kok_colors, color_ban, numInitialColors, color_set));
//std::cout << "nv:" << this->nv << " init_colors" << std::endl;

//worklist
Expand Down Expand Up @@ -2515,23 +2519,28 @@ class GraphColor_EB:public GraphColor <HandleType,in_row_index_view_type_,in_non
color_view_type kokcolors;
color_temp_work_view_type color_ban; //colors
color_t hash; //the number of colors to be assigned initially.

nnz_lno_temp_work_view_t color_set;
//the value to initialize the color_ban_. We avoid using the first bit representing the sign.
//Therefore if idx is int, it can represent 32-1 colors. Use color_set to represent more.
color_t color_ban_init_val;


init_colors (color_view_type colors,color_temp_work_view_type color_ban_,color_t hash_):
kokcolors(colors), color_ban(color_ban_), hash(hash_){
init_colors (color_view_type colors,color_temp_work_view_type color_ban_,color_t hash_, nnz_lno_temp_work_view_t color_set_):
kokcolors(colors), color_ban(color_ban_), hash(hash_), color_set(color_set_){
color_t tmp = 1;
color_ban_init_val = tmp <<( sizeof(color_t) * 8 -1);
}

KOKKOS_INLINE_FUNCTION
void operator()(const size_type &ii) const {
//set colors based on their indices.
color_t tmp1 = 1;
kokcolors(ii) = tmp1 << (ii % hash);
//color_t tmp1 = 1;
if(kokcolors(ii) > 0){
color_t colorsize = sizeof(color_t) * 8 - 1;
color_set(ii) = (kokcolors(ii) - 1) / colorsize;
kokcolors(ii) = 1 << ((kokcolors(ii) - 1) % colorsize);
}
//kokcolors(ii) = tmp1 << (ii % hash);
color_ban(ii) = color_ban_init_val;
}
};
Expand Down
76 changes: 68 additions & 8 deletions packages/kokkos-kernels/src/graph/impl/KokkosGraph_Distance2Color_impl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,13 @@ class GraphColorDistance2
{
//Delegate to different coloring functions, depending on algorithm
using_edge_filtering = false;
color_view_type colors_out("Graph Colors", this->nr);
//color_view_type colors_out("Graph Colors", this->nr);
color_view_type colors_out;
if(gc_handle->get_vertex_colors().use_count() > 0){
colors_out = gc_handle->get_vertex_colors();
} else {
colors_out = color_view_type("Graph Colors", this->nr);
}
switch(this->gc_handle->get_coloring_algo_type())
{
case COLORING_D2_VB_BIT_EF:
Expand Down Expand Up @@ -244,8 +250,15 @@ class GraphColorDistance2
lno_view_t current_vertexList(
Kokkos::ViewAllocateWithoutInitializing("vertexList"), this->nr);

lno_t current_vertexListLength = this->nr;

// init conflictlist sequentially.
Kokkos::parallel_for("InitList", range_policy_type(0, this->nr), functorInitList<lno_view_t>(current_vertexList));
if(this->gc_handle->get_use_vtx_list()){
current_vertexList = this->gc_handle->get_vertex_list();
current_vertexListLength = this->gc_handle->get_vertex_list_size();
} else {
Kokkos::parallel_for("InitList", range_policy_type(0, this->nr), functorInitList<lno_view_t>(current_vertexList));
}

// Next iteratons's conflictList
lno_view_t next_iteration_recolorList(Kokkos::ViewAllocateWithoutInitializing("recolorList"), this->nr);
Expand All @@ -255,7 +268,6 @@ class GraphColorDistance2

lno_t numUncolored = this->nr;
lno_t numUncoloredPreviousIter = this->nr + 1;
lno_t current_vertexListLength = this->nr;

double time;
double total_time = 0.0;
Expand Down Expand Up @@ -445,7 +457,8 @@ class GraphColorDistance2
break;
}
}
if(color)
//make sure vertices with a valid color do not get recolored
if(color && (colors(v) == 0 || colors(v) == CONFLICTED || colors(v) == UNCOLORABLE))
{
//Color v
colors(v) = color;
Expand All @@ -466,7 +479,7 @@ class GraphColorDistance2
}
}
}
else
else if (colors(v) == 0 || colors(v) == CONFLICTED || colors(v) == UNCOLORABLE)
{
colors(v) = UNCOLORABLE;
}
Expand Down Expand Up @@ -719,8 +732,32 @@ class GraphColorDistance2
}
const lno_t numVerts = this->nr;
const lno_t numCols = this->nc;
//note: relying on forbidden and colors_out being initialized to 0
//color_type max_color = 0;
//for(int i = 0; i < numCols; i++) if(colors_out(i) > max_color) max_color = colors_out(i);
//note: initializing forbidden to account for previously-colored vertices
forbidden_view forbidden("Forbidden", batch * numCols);
/*for(color_type colorBase = 1; colorBase < max_color; colorBase += 32 * batch){
switch(batch)
{
case 1:
Kokkos::parallel_for("NB D2 Forbidden", range_policy_type(0, numCols),
NB_RefreshForbidden<1>(colorBase, forbidden, colors_out, this->t_xadj, this->t_adj, numVerts));
break;
case 2:
Kokkos::parallel_for("NB D2 Forbidden", range_policy_type(0, numCols),
NB_RefreshForbidden<2>(colorBase, forbidden, colors_out, this->t_xadj, this->t_adj, numVerts));
break;
case 4:
Kokkos::parallel_for("NB D2 Forbidden", range_policy_type(0, numCols),
NB_RefreshForbidden<4>(colorBase, forbidden, colors_out, this->t_xadj, this->t_adj, numVerts));
break;
case 8:
Kokkos::parallel_for("NB D2 Forbidden", range_policy_type(0, numCols),
NB_RefreshForbidden<8>(colorBase, forbidden, colors_out, this->t_xadj, this->t_adj, numVerts));
break;
default:;
}
}*/
int iter = 0;
Kokkos::Impl::Timer timer;
lno_t currentWork = this->nr;
Expand All @@ -737,6 +774,29 @@ class GraphColorDistance2
lno_t vertsPerThread = 1;
lno_t workBatches = (currentWork + vertsPerThread - 1) / vertsPerThread;
timer.reset();
//refresh forbidden before coloring, to ensure previously-colored vertices do not get recolored unnecessarily.
switch(batch)
{
case 1:
Kokkos::parallel_for("NB D2 Forbidden", range_policy_type(0, numCols),
NB_RefreshForbidden<1>(colorBase, forbidden, colors_out, this->t_xadj, this->t_adj, numVerts));
break;
case 2:
Kokkos::parallel_for("NB D2 Forbidden", range_policy_type(0, numCols),
NB_RefreshForbidden<2>(colorBase, forbidden, colors_out, this->t_xadj, this->t_adj, numVerts));
break;
case 4:
Kokkos::parallel_for("NB D2 Forbidden", range_policy_type(0, numCols),
NB_RefreshForbidden<4>(colorBase, forbidden, colors_out, this->t_xadj, this->t_adj, numVerts));
break;
case 8:
Kokkos::parallel_for("NB D2 Forbidden", range_policy_type(0, numCols),
NB_RefreshForbidden<8>(colorBase, forbidden, colors_out, this->t_xadj, this->t_adj, numVerts));
break;
default:;
}
forbiddenTime += timer.seconds();
timer.reset();
switch(batch)
{
case 1:
Expand Down Expand Up @@ -790,7 +850,7 @@ class GraphColorDistance2
timer.reset();
//if still using this color set, refresh forbidden.
//This avoids using too many colors, by relying on forbidden from before conflict resolution (which is now stale).
if(currentWork)
/*if(currentWork)
{
switch(batch)
{
Expand All @@ -814,7 +874,7 @@ class GraphColorDistance2
}
forbiddenTime += timer.seconds();
timer.reset();
}
}*/
iter++;
}
//Will need to run with a different color base, so rebuild the work list
Expand Down
Loading