Skip to content

Commit

Permalink
Merge Pull Request #6704 from gsjaardema/Trilinos/seacas-cgns-periodi…
Browse files Browse the repository at this point in the history 
…c-fix

Automatically Merged using Trilinos Pull Request AutoTester
PR Title: IOSS: cgns - fix handling of periodic single-block models in parallel
PR Author: gsjaardema
  • Loading branch information
@trilinos-autotester
trilinos-autotester authored Feb 1, 2020
2 parents 0f44d6b + 4968cdd commit eedb71e
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 53 deletions.
94 changes: 64 additions & 30 deletions packages/seacas/libraries/ioss/src/cgns/Iocgns_DatabaseIO.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,29 @@ extern char hdf5_access[64];

namespace {

bool has_decomp_descriptor(int cgns_file_ptr, int base, int zone, int zgc_idx)
{
bool has_decomp_flag = false;
if (cg_goto(cgns_file_ptr, base, "Zone_t", zone, "ZoneGridConnectivity", 0,
"GridConnectivity1to1_t", zgc_idx, "end") == CG_OK) {
int ndescriptor = 0;
cg_ndescriptors(&ndescriptor);
if (ndescriptor > 0) {
for (int i = 0; i < ndescriptor; i++) {
char name[33];
char *text;
cg_descriptor_read(i + 1, name, &text);
if (strcmp(name, "Decomp") == 0) {
has_decomp_flag = true;
break;
}
cg_free(text);
}
}
}
return has_decomp_flag;
}

template <typename T> void pack(int &idx, std::vector<int> &pack, T *from, int count)
{
for (int i = 0; i < count; i++) {
Expand Down Expand Up @@ -134,29 +157,6 @@ namespace {
#endif
};

std::pair<std::string, int> decompose_name(const std::string &name, bool is_parallel)
{
int proc = 0;
std::string zname{name};

if (is_parallel) {
// Name should/might be of the form `basename_proc-#`. Strip
// off the `_proc-#` portion and return just the basename.
auto tokens = Ioss::tokenize(zname, "_");
zname = tokens[0];
if (tokens.size() >= 2) {
size_t idx = tokens.size() - 1;
if (tokens[idx].substr(0, 5) == "proc-") {
auto ptoken = Ioss::tokenize(tokens[idx], "-");
proc = std::stoi(ptoken[1]);
idx--;
zname = tokens[idx];
}
}
}
return std::make_pair(zname, proc);
}

#ifdef SEACAS_HAVE_MPI
void add_zgc_fpp(int cgns_file_ptr, Ioss::StructuredBlock *block,
const std::map<std::string, int> &zone_name_map, int myProcessor,
Expand All @@ -178,7 +178,12 @@ namespace {
CGCHECK(cg_1to1_read(cgns_file_ptr, base, db_zone, ii + 1, connectname, donorname,
range.data(), donor_range.data(), transform.data()));

auto donorname_proc = decompose_name(donorname, isParallel);
bool is_decomp = false;
if (isParallel) {
is_decomp = has_decomp_descriptor(cgns_file_ptr, base, db_zone, ii + 1);
}

auto donorname_proc = Iocgns::Utils::decompose_name(donorname, isParallel);
std::string donor_name = donorname_proc.first;

// Get number of nodes shared with other "previous" zones...
Expand All @@ -204,12 +209,13 @@ namespace {
range_end[1] += offset[1];
range_end[2] += offset[2];

auto con_name = decompose_name(connectname, isParallel).first;
auto con_name = Iocgns::Utils::decompose_name(connectname, isParallel).first;
block->m_zoneConnectivity.emplace_back(con_name, zone, donor_name, donor_zone, transform,
range_beg, range_end, donor_beg, donor_end, offset);

block->m_zoneConnectivity.back().m_ownerProcessor = myProcessor;
block->m_zoneConnectivity.back().m_donorProcessor = donorname_proc.second;
block->m_zoneConnectivity.back().m_fromDecomp = is_decomp;
}
}

Expand Down Expand Up @@ -314,6 +320,23 @@ namespace {
bool zone_added = false;
int nconn = 0;
CGCHECK(cg_n1to1(cgns_file_ptr, base, zone, &nconn));

// See if any of the zgc have a "Decomp" descriptor node. If so, then
// We can unambiguously determine whether a ZGC is from decomp or is
// normal inter-zone ZGC. If the descriptor does not exist, then have
// to rely on hueristics...
bool has_decomp_flag = false;
for (int i = 0; i < nconn; i++) {
if (has_decomp_descriptor(cgns_file_ptr, base, zone, i + 1)) {
has_decomp_flag = true;
break;
}
}

#if IOSS_DEBUG_OUTPUT
fmt::print("CGNS DatabaseIO has decomp flag? {}\n", has_decomp_flag);
#endif

for (int i = 0; i < nconn; i++) {
char connectname[CGNS_MAX_NAME_LENGTH + 1];
char donorname[CGNS_MAX_NAME_LENGTH + 1];
Expand All @@ -324,10 +347,21 @@ namespace {
CGCHECK(cg_1to1_read(cgns_file_ptr, base, zone, i + 1, connectname, donorname, range.data(),
donor_range.data(), transform.data()));

auto donorname_proc = decompose_name(donorname, true);
auto donorname_proc = Iocgns::Utils::decompose_name(donorname, true);
std::string donor_name = donorname_proc.first;
auto donor_proc = donorname_proc.second;

bool is_from_decomp = false;
if (has_decomp_flag) {
is_from_decomp = has_decomp_descriptor(cgns_file_ptr, base, zone, i + 1);
}
else {
is_from_decomp = donor_name == zone_name && donor_proc >= 0 && donor_proc != myProcessor;
}

if (donor_name == zone_name) {
if (is_from_decomp) {
// See if the descriptor named "Decomp" exists as a child of this ZGC.
// If so, then
// Determine which face of the zone on this processor is
// shared with the other processor...
int face = find_face(range);
Expand Down Expand Up @@ -685,7 +719,7 @@ namespace Iocgns {
assert(size[7] == 0);
assert(size[8] == 0);

auto name_proc = decompose_name(zname, isParallel);
auto name_proc = Iocgns::Utils::decompose_name(zname, isParallel);
std::string zone_name = name_proc.first;
int proc = name_proc.second;
assert(proc == myProcessor);
Expand Down Expand Up @@ -979,7 +1013,7 @@ namespace Iocgns {
char zone_name[CGNS_MAX_NAME_LENGTH + 1];
CGCHECKM(cg_zone_read(get_file_pointer(), base, zone, zone_name, size));

auto name_proc = decompose_name(zone_name, isParallel);
auto name_proc = Iocgns::Utils::decompose_name(zone_name, isParallel);
std::string zname = name_proc.first;
int proc = name_proc.second;
if (proc != myProcessor) {
Expand Down Expand Up @@ -1029,7 +1063,7 @@ namespace Iocgns {
CGCHECKM(cg_1to1_read(get_file_pointer(), base, zone, i + 1, connectname, donorname,
range.data(), donor_range.data(), transform.data()));

auto donorname_proc = decompose_name(donorname, isParallel);
auto donorname_proc = Iocgns::Utils::decompose_name(donorname, isParallel);
std::string donor_name = donorname_proc.first;

// Get number of nodes shared with other "previous" zones...
Expand Down
52 changes: 29 additions & 23 deletions packages/seacas/libraries/ioss/src/cgns/Iocgns_Utils.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,29 +80,6 @@
}

namespace {
std::pair<std::string, int> decompose_name(const std::string &name, bool is_parallel)
{
int proc = 0;
std::string zname{name};

if (is_parallel) {
// Name should/might be of the form `basename_proc-#`. Strip
// off the `_proc-#` portion and return just the basename.
auto tokens = Ioss::tokenize(zname, "_");
zname = tokens[0];
if (tokens.size() >= 2) {
size_t idx = tokens.size() - 1;
if (tokens[idx].substr(0, 5) == "proc-") {
auto ptoken = Ioss::tokenize(tokens[idx], "-");
proc = std::stoi(ptoken[1]);
idx--;
zname = tokens[idx];
}
}
}
return std::make_pair(zname, proc);
}

int power_2(int count)
{
// Return the maximum power of two which is less than or equal to 'count'
Expand Down Expand Up @@ -384,6 +361,29 @@ namespace {
}
} // namespace

std::pair<std::string, int> Iocgns::Utils::decompose_name(const std::string &name, bool is_parallel)
{
int proc = is_parallel ? -1 : 0;
std::string zname{name};

if (is_parallel) {
// Name should/might be of the form `basename_proc-#`. Strip
// off the `_proc-#` portion and return just the basename.
auto tokens = Ioss::tokenize(zname, "_");
zname = tokens[0];
if (tokens.size() >= 2) {
size_t idx = tokens.size() - 1;
if (tokens[idx].substr(0, 5) == "proc-") {
auto ptoken = Ioss::tokenize(tokens[idx], "-");
proc = std::stoi(ptoken[1]);
idx--;
zname = tokens[idx];
}
}
}
return std::make_pair(zname, proc);
}

template <typename INT>
void Iocgns::Utils::map_cgns_connectivity(const Ioss::ElementTopology *topo, size_t element_count,
INT *idata)
Expand Down Expand Up @@ -1229,6 +1229,12 @@ size_t Iocgns::Utils::common_write_meta_data(int file_ptr, const Ioss::Region &r
CGERR(cg_1to1_write(file_ptr, base, db_zone, connect_name.c_str(), donor_name.c_str(),
owner_range.data(), donor_range.data(), zgc.m_transform.data(),
&zgc_idx));

if (zgc.is_from_decomp()) {
CGERR(cg_goto(file_ptr, base, "Zone_t", db_zone, "ZoneGridConnectivity", 0,
"GridConnectivity1to1_t", zgc_idx, "end"));
CGERR(cg_descriptor_write("Decomp", "is_decomp"));
}
}
}
}
Expand Down
2 changes: 2 additions & 0 deletions packages/seacas/libraries/ioss/src/cgns/Iocgns_Utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ namespace Iocgns {
static const size_t CG_CELL_CENTER_FIELD_ID = 1ul << 33;
static const size_t CG_VERTEX_FIELD_ID = 1ul << 34;

static std::pair<std::string, int> decompose_name(const std::string &name, bool is_parallel);

static size_t index(const Ioss::Field &field);

static void cgns_error(int cgnsid, const char *file, const char *function, int lineno,
Expand Down
15 changes: 15 additions & 0 deletions packages/seacas/libraries/ioss/src/main/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,21 @@ TRIBITS_ADD_ADVANCED_TEST(structured_cgns_roundtrip_zgc
COMM mpi serial
)

TRIBITS_ADD_ADVANCED_TEST(structured_cgns_periodic_roundtrip_fpp
TEST_0 EXEC io_shell ARGS --compose external ${CMAKE_CURRENT_SOURCE_DIR}/test/self-looping-grid.cgns self-looping-fpp.cgns
NUM_MPI_PROCS 1-4
NOEXEPREFIX NOEXESUFFIX
TEST_1 EXEC io_shell ARGS --external --compose external self-looping-fpp.cgns self-looping2.cgns
NUM_MPI_PROCS 1-4
NOEXEPREFIX NOEXESUFFIX
TEST_2 EXEC io_shell ARGS --external self-looping2.cgns self-looping3.cgns
NUM_MPI_PROCS 1-4
NOEXEPREFIX NOEXESUFFIX
TEST_3 CMND ${CGNS_CGNSDIFF_BINARY} ARGS -d ${CMAKE_CURRENT_SOURCE_DIR}/test/self-looping-grid.cgns self-looping3.cgns
NUM_MPI_PROCS 1
COMM mpi
)

TRIBITS_ADD_ADVANCED_TEST(structured_cgns_roundtrip_fpp
TEST_0 EXEC io_shell ARGS --compose external ${CMAKE_CURRENT_SOURCE_DIR}/test/5blocks-struc.cgns 5blocks.cgns
NUM_MPI_PROCS 1-4
Expand Down

0 comments on commit eedb71e

Please sign in to comment.