From 358897d69cb81f4c6d7ac86a961b0ca8de1d791a Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Sat, 14 Aug 2021 15:40:10 -0600 Subject: [PATCH 01/70] Tpetra: Remove Distributor from pack/unpack interfaces --- .../core/inout/Tpetra_Details_CooMatrix.hpp | 4 +- .../core/src/Tpetra_BlockCrsMatrix_decl.hpp | 4 +- .../core/src/Tpetra_BlockCrsMatrix_def.hpp | 4 +- .../core/src/Tpetra_BlockMultiVector_decl.hpp | 4 +- .../core/src/Tpetra_BlockMultiVector_def.hpp | 4 +- .../tpetra/core/src/Tpetra_CrsGraph_decl.hpp | 13 ++----- .../tpetra/core/src/Tpetra_CrsGraph_def.hpp | 31 +++++++-------- .../tpetra/core/src/Tpetra_CrsMatrix_decl.hpp | 14 ++----- .../tpetra/core/src/Tpetra_CrsMatrix_def.hpp | 38 +++++++------------ .../src/Tpetra_Details_packCrsGraph_decl.hpp | 20 ++-------- .../src/Tpetra_Details_packCrsGraph_def.hpp | 30 +++++---------- .../src/Tpetra_Details_packCrsMatrix_decl.hpp | 20 ++-------- .../src/Tpetra_Details_packCrsMatrix_def.hpp | 33 +++++----------- ..._Details_unpackCrsGraphAndCombine_decl.hpp | 9 ----- ...a_Details_unpackCrsGraphAndCombine_def.hpp | 11 ------ ...Details_unpackCrsMatrixAndCombine_decl.hpp | 9 ----- ..._Details_unpackCrsMatrixAndCombine_def.hpp | 13 ------- .../core/src/Tpetra_DistObject_decl.hpp | 10 +---- .../tpetra/core/src/Tpetra_DistObject_def.hpp | 12 +++--- .../core/src/Tpetra_MultiVector_decl.hpp | 4 +- .../core/src/Tpetra_MultiVector_def.hpp | 4 +- packages/tpetra/core/src/Tpetra_Packable.hpp | 10 +---- .../tpetra/core/src/Tpetra_RowGraph_decl.hpp | 3 +- .../tpetra/core/src/Tpetra_RowGraph_def.hpp | 6 +-- .../tpetra/core/src/Tpetra_RowMatrix_decl.hpp | 6 +-- .../tpetra/core/src/Tpetra_RowMatrix_def.hpp | 10 ++--- .../test/CrsGraph/CrsGraph_PackUnpack.cpp | 10 ++--- .../CrsGraph_UnpackIntoStaticGraph.cpp | 12 +----- .../test/CrsMatrix/CrsMatrix_PackUnpack.cpp | 19 ++++------ .../CrsMatrix/CrsMatrix_createDeepCopy.cpp | 10 ++--- .../ImportExport2/ImportExport2_UnitTests.cpp | 5 +-- 31 files changed, 99 insertions(+), 283 deletions(-) diff --git a/packages/tpetra/core/inout/Tpetra_Details_CooMatrix.hpp b/packages/tpetra/core/inout/Tpetra_Details_CooMatrix.hpp index 1a32d7e7f611..6d8a159274e1 100644 --- a/packages/tpetra/core/inout/Tpetra_Details_CooMatrix.hpp +++ b/packages/tpetra/core/inout/Tpetra_Details_CooMatrix.hpp @@ -1209,8 +1209,7 @@ class CooMatrix : public ::Tpetra::DistObject { buffer_device_type>& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - ::Tpetra::Distributor& /* distor */) + size_t& constantNumPackets) { using Teuchos::Comm; using Teuchos::RCP; @@ -1406,7 +1405,6 @@ class CooMatrix : public ::Tpetra::DistObject { Kokkos::DualView numPacketsPerLID, const size_t /* constantNumPackets */, - ::Tpetra::Distributor& /* distor */, const ::Tpetra::CombineMode /* combineMode */) { using Teuchos::Comm; diff --git a/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_decl.hpp b/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_decl.hpp index 06b10eef9890..ce7b1f7cd89e 100644 --- a/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_decl.hpp @@ -740,8 +740,7 @@ class BlockCrsMatrix : buffer_device_type>& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& /* distor */) override; + size_t& constantNumPackets) override; virtual void unpackAndCombine @@ -752,7 +751,6 @@ class BlockCrsMatrix : Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& /* distor */, const CombineMode combineMode) override; //@} diff --git a/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_def.hpp b/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_def.hpp index 6e35e3cf2b72..1da618cd6d1e 100644 --- a/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_def.hpp +++ b/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_def.hpp @@ -2377,8 +2377,7 @@ class GetLocalDiagCopy { buffer_device_type>& exports, // output Kokkos::DualView numPacketsPerLID, // output - size_t& constantNumPackets, - Distributor& /* distor */) + size_t& constantNumPackets) { using ::Tpetra::Details::Behavior; using ::Tpetra::Details::dualViewStatusToString; @@ -2634,7 +2633,6 @@ class GetLocalDiagCopy { Kokkos::DualView numPacketsPerLID, const size_t /* constantNumPackets */, - Distributor& /* distor */, const CombineMode combineMode) { using ::Tpetra::Details::Behavior; diff --git a/packages/tpetra/core/src/Tpetra_BlockMultiVector_decl.hpp b/packages/tpetra/core/src/Tpetra_BlockMultiVector_decl.hpp index d7f9b0a82141..0250ff716c43 100644 --- a/packages/tpetra/core/src/Tpetra_BlockMultiVector_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_BlockMultiVector_decl.hpp @@ -665,8 +665,7 @@ class BlockMultiVector : buffer_device_type>& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); virtual void unpackAndCombine @@ -677,7 +676,6 @@ class BlockMultiVector : Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode); //@} diff --git a/packages/tpetra/core/src/Tpetra_BlockMultiVector_def.hpp b/packages/tpetra/core/src/Tpetra_BlockMultiVector_def.hpp index 96603bd020b5..b9a44c49a199 100644 --- a/packages/tpetra/core/src/Tpetra_BlockMultiVector_def.hpp +++ b/packages/tpetra/core/src/Tpetra_BlockMultiVector_def.hpp @@ -571,8 +571,7 @@ packAndPrepare buffer_device_type>& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { TEUCHOS_TEST_FOR_EXCEPTION (true, std::logic_error, @@ -590,7 +589,6 @@ unpackAndCombine Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode) { TEUCHOS_TEST_FOR_EXCEPTION diff --git a/packages/tpetra/core/src/Tpetra_CrsGraph_decl.hpp b/packages/tpetra/core/src/Tpetra_CrsGraph_decl.hpp index a7bc7f167021..ed4f882a85ab 100644 --- a/packages/tpetra/core/src/Tpetra_CrsGraph_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_CrsGraph_decl.hpp @@ -1302,22 +1302,19 @@ namespace Tpetra { const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) override; + size_t& constantNumPackets) override; virtual void pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const override; + size_t& constantNumPackets) const override; void packFillActive (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const; + size_t& constantNumPackets) const; void packFillActiveNew (const Kokkos::DualView& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const; + size_t& constantNumPackets) const; virtual void unpackAndCombine @@ -1338,7 +1334,6 @@ namespace Tpetra { Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode) override; //@} diff --git a/packages/tpetra/core/src/Tpetra_CrsGraph_def.hpp b/packages/tpetra/core/src/Tpetra_CrsGraph_def.hpp index 06c7ceb0edbc..8e609554dde0 100644 --- a/packages/tpetra/core/src/Tpetra_CrsGraph_def.hpp +++ b/packages/tpetra/core/src/Tpetra_CrsGraph_def.hpp @@ -5690,8 +5690,7 @@ namespace Tpetra { buffer_device_type>& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { using Tpetra::Details::ProfilingRegion; using GO = global_ordinal_type; @@ -5759,7 +5758,7 @@ namespace Tpetra { ArrayView numPacketsPerLID_av (numPacketsPerLID_h.data (), numPacketsPerLID_h.extent (0)); srcRowGraphPtr->pack (exportLIDs_av, exports_a, numPacketsPerLID_av, - constantNumPackets, distor); + constantNumPackets); const size_t newSize = static_cast (exports_a.size ()); if (static_cast (exports.extent (0)) != newSize) { using exports_dv_type = Kokkos::DualView; @@ -5788,11 +5787,11 @@ namespace Tpetra { using Tpetra::Details::packCrsGraphNew; packCrsGraphNew (*srcCrsGraphPtr, exportLIDs, exportPIDs, exports, numPacketsPerLID, - constantNumPackets, false, distor); + constantNumPackets, false); } else { srcCrsGraphPtr->packFillActiveNew (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); } if (verbose) { @@ -5808,19 +5807,18 @@ namespace Tpetra { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const + size_t& constantNumPackets) const { auto col_map = this->getColMap(); // packCrsGraph requires k_rowPtrsPacked to be set if( !col_map.is_null() && (rowPtrsPacked_dev_.extent(0) != 0 || getRowMap()->getNodeNumElements() ==0)) { using Tpetra::Details::packCrsGraph; packCrsGraph(*this, exports, numPacketsPerLID, - exportLIDs, constantNumPackets, distor); + exportLIDs, constantNumPackets); } else { this->packFillActive(exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); } } @@ -5830,8 +5828,7 @@ namespace Tpetra { packFillActive (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& /* distor */) const + size_t& constantNumPackets) const { using std::endl; using LO = LocalOrdinal; @@ -6021,8 +6018,7 @@ namespace Tpetra { buffer_device_type>& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const + size_t& constantNumPackets) const { using std::endl; using LO = local_ordinal_type; @@ -6251,7 +6247,6 @@ namespace Tpetra { Kokkos::DualView numPacketsPerLID, const size_t /* constantNumPackets */, - Distributor& /* distor */, const CombineMode /* combineMode */ ) { using Details::ProfilingRegion; @@ -7067,7 +7062,7 @@ namespace Tpetra { // The basic algorithm here is: // - // 1. Call the moral equivalent of "distor.do" to handle the import. + // 1. Call the moral equivalent of "Distor.do" to handle the import. // 2. Copy all the Imported and Copy/Permuted data into the raw // CrsGraph pointers, still using GIDs. // 3. Call an optimized version of MakeColMap that avoids the @@ -7274,7 +7269,7 @@ namespace Tpetra { // Pack & Prepare w/ owning PIDs packCrsGraphWithOwningPIDs(*this, destGraph->exports_, numExportPacketsPerLID, ExportLIDs, - SourcePids, constantNumPackets, Distor); + SourcePids, constantNumPackets); } // Do the exchange of remote data. @@ -7401,7 +7396,7 @@ namespace Tpetra { size_t mynnz = unpackAndCombineWithOwningPIDsCount(*this, RemoteLIDs, hostImports, numImportPacketsPerLID, - constantNumPackets, Distor, INSERT, + constantNumPackets, INSERT, NumSameIDs, PermuteToLIDs, PermuteFromLIDs); size_t N = BaseRowMap->getNodeNumElements(); @@ -7427,7 +7422,7 @@ namespace Tpetra { // takes five methods. unpackAndCombineIntoCrsArrays(*this, RemoteLIDs, hostImports, numImportPacketsPerLID, constantNumPackets, - Distor, INSERT, NumSameIDs, PermuteToLIDs, + INSERT, NumSameIDs, PermuteToLIDs, PermuteFromLIDs, N, mynnz, MyPID, CSR_rowptr(), CSR_colind_GID(), SourcePids(), TargetPids); diff --git a/packages/tpetra/core/src/Tpetra_CrsMatrix_decl.hpp b/packages/tpetra/core/src/Tpetra_CrsMatrix_decl.hpp index 7bc82914263f..555e10b451d9 100644 --- a/packages/tpetra/core/src/Tpetra_CrsMatrix_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_CrsMatrix_decl.hpp @@ -3074,8 +3074,7 @@ namespace Tpetra { buffer_device_type>& exportLIDs, Kokkos::DualView& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) override; + size_t& constantNumPackets) override; private: /// \brief Unpack the imported column indices and values, and @@ -3087,7 +3086,6 @@ namespace Tpetra { Kokkos::DualView imports, Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode, const bool verbose); @@ -3100,7 +3098,6 @@ namespace Tpetra { Kokkos::DualView imports, Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode); public: @@ -3119,7 +3116,6 @@ namespace Tpetra { Kokkos::DualView imports, Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode CM) override; /// \brief Pack this object's data for an Import or Export. @@ -3138,8 +3134,6 @@ namespace Tpetra { /// output, then that number gives the constant number of /// entries for all packed rows on all processes in the /// matrix's communicator. - /// \param distor [in/out] The Distributor object which implements - /// the Import or Export operation that is calling this method. /// /// \subsection Tpetra_CrsMatrix_packNew_summary Packing scheme /// @@ -3233,8 +3227,7 @@ namespace Tpetra { packNew (const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, const Kokkos::DualView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& dist) const; + size_t& constantNumPackets) const; private: /// \brief Pack this matrix (part of implementation of packAndPrepare). @@ -3247,8 +3240,7 @@ namespace Tpetra { packNonStaticNew (const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, const Kokkos::DualView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const; + size_t& constantNumPackets) const; /// \brief Pack data for the current row to send. /// diff --git a/packages/tpetra/core/src/Tpetra_CrsMatrix_def.hpp b/packages/tpetra/core/src/Tpetra_CrsMatrix_def.hpp index 6b89763912da..0147280f6659 100644 --- a/packages/tpetra/core/src/Tpetra_CrsMatrix_def.hpp +++ b/packages/tpetra/core/src/Tpetra_CrsMatrix_def.hpp @@ -6521,8 +6521,7 @@ CrsMatrix:: const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { using Details::Behavior; using Details::dualViewStatusToString; @@ -6599,7 +6598,7 @@ CrsMatrix:: } try { srcCrsMat->packNew (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); } catch (std::exception& e) { lclBad = 1; @@ -6659,7 +6658,7 @@ CrsMatrix:: // Teuchos::Array* objects. try { srcRowMat->pack (exportLIDs_av, exports_a, numPacketsPerLID_av, - constantNumPackets, distor); + constantNumPackets); } catch (std::exception& e) { lclBad = 1; @@ -6996,19 +6995,18 @@ CrsMatrix:: packNew (const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, const Kokkos::DualView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& dist) const + size_t& constantNumPackets) const { // The call to packNew in packAndPrepare catches and handles any exceptions. Details::ProfilingRegion region_pack_new("Tpetra::CrsMatrix::packNew", "Import/Export"); if (this->isStaticGraph ()) { using ::Tpetra::Details::packCrsMatrixNew; packCrsMatrixNew (*this, exports, numPacketsPerLID, exportLIDs, - constantNumPackets, dist); + constantNumPackets); } else { this->packNonStaticNew (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, dist); + constantNumPackets); } } @@ -7018,8 +7016,7 @@ CrsMatrix:: packNonStaticNew (const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, const Kokkos::DualView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& /* distor */) const + size_t& constantNumPackets) const { using Details::Behavior; using Details::dualViewStatusToString; @@ -7302,7 +7299,6 @@ CrsMatrix:: Kokkos::DualView imports, Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode) { using Details::Behavior; @@ -7374,7 +7370,7 @@ CrsMatrix:: int lclBad = 0; try { unpackAndCombineImpl(importLIDs, imports, numPacketsPerLID, - constantNumPackets, distor, combineMode, + constantNumPackets, combineMode, verbose); } catch (std::exception& e) { lclBad = 1; @@ -7401,7 +7397,7 @@ CrsMatrix:: } else { unpackAndCombineImpl(importLIDs, imports, numPacketsPerLID, - constantNumPackets, distor, combineMode, + constantNumPackets, combineMode, verbose); } @@ -7430,7 +7426,6 @@ CrsMatrix:: Kokkos::DualView imports, Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor & distor, const CombineMode combineMode, const bool verbose) { @@ -7460,7 +7455,7 @@ CrsMatrix:: using Details::unpackCrsMatrixAndCombineNew; unpackCrsMatrixAndCombineNew(*this, imports, numPacketsPerLID, importLIDs, constantNumPackets, - distor, combineMode); + combineMode); } else { { @@ -7495,7 +7490,7 @@ CrsMatrix:: unpackAndCombineImplNonStatic(importLIDs, imports, numPacketsPerLID, constantNumPackets, - distor, combineMode); + combineMode); } if (verbose) { @@ -7514,7 +7509,6 @@ CrsMatrix:: Kokkos::DualView imports, Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode) { using Kokkos::View; @@ -8348,7 +8342,7 @@ CrsMatrix:: // The basic algorithm here is: // - // 1. Call the moral equivalent of "distor.do" to handle the import. + // 1. Call the moral equivalent of "Distor.do" to handle the import. // 2. Copy all the Imported and Copy/Permuted data into the raw // CrsMatrix / CrsGraphData pointers, still using GIDs. // 3. Call an optimized version of MakeColMap that avoids the @@ -8603,8 +8597,7 @@ CrsMatrix:: numExportPacketsPerLID, ExportLIDs, SourcePids, - constantNumPackets, - Distor); + constantNumPackets); } catch (std::exception& e) { errStrm << "Proc " << myRank << ": packCrsMatrixWithOwningPIDs threw: " @@ -8650,8 +8643,7 @@ CrsMatrix:: numExportPacketsPerLID, ExportLIDs, SourcePids, - constantNumPackets, - Distor); + constantNumPackets); if (verbose) { std::ostringstream os; os << *verbosePrefix << "Done with packCrsMatrixWithOwningPIDs" @@ -8892,7 +8884,6 @@ CrsMatrix:: hostImports, numImportPacketsPerLID, constantNumPackets, - Distor, INSERT, NumSameIDs, PermuteToLIDs, @@ -8938,7 +8929,6 @@ CrsMatrix:: hostImports, numImportPacketsPerLID, constantNumPackets, - Distor, INSERT, NumSameIDs, PermuteToLIDs, diff --git a/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_decl.hpp b/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_decl.hpp index e7c2c395fca6..08f569731930 100644 --- a/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_decl.hpp @@ -79,11 +79,6 @@ template class ArrayView; namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -112,8 +107,6 @@ namespace Details { /// to expect a possibly /// different ("nonconstant") number of packets per local index /// (i.e., a possibly different number of entries per row). /// -/// \param distor [in] The distributor (not used) -/// /// This is the public interface to the pack machinery /// converts passed Teuchos::ArrayView objects to Kokkos::View objects (and /// copies back in to the Teuchos::ArrayView objects, if needed). When @@ -125,8 +118,7 @@ packCrsGraph (const CrsGraph& sourceGraph, Teuchos::Array::packet_type>& exports, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); /// \brief Pack specified entries of the given local sparse graph for /// communication, for "new" DistObject interface. @@ -152,8 +144,6 @@ packCrsGraph (const CrsGraph& sourceGraph, /// output argument of Tpetra::DistObject::packAndPrepare (which /// see). /// -/// \param distor [in] (Not used.) -/// /// This method implements CrsGraph::packNew, and thus /// CrsGraph::packAndPrepare, for the case where the graph to /// pack has a valid KokkosSparse::CrsGraph. @@ -177,8 +167,7 @@ packCrsGraphNew (const CrsGraph& sourceGraph, typename CrsGraph::buffer_device_type > numPacketsPerLID, size_t& constantNumPackets, - const bool pack_pids, - Distributor& distor); + const bool pack_pids); /// \brief Pack specified entries of the given local sparse graph for /// communication. @@ -203,8 +192,6 @@ packCrsGraphNew (const CrsGraph& sourceGraph, /// to expect a possibly /// different ("nonconstant") number of packets per local index /// (i.e., a possibly different number of entries per row). /// -/// \param distor [in] The distributor (not used) -/// /// This is the public interface to the pack machinery /// converts passed Teuchos::ArrayView objects to Kokkos::View objects (and /// copies back in to the Teuchos::ArrayView objects, if needed). When @@ -219,8 +206,7 @@ packCrsGraphWithOwningPIDs (const CrsGraph& sourceGraph, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, const Teuchos::ArrayView& sourcePIDs, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); } // namespace Details } // namespace Tpetra diff --git a/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_def.hpp b/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_def.hpp index 479733fe8c9d..5e1af8288134 100644 --- a/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_def.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_def.hpp @@ -77,11 +77,6 @@ namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -678,8 +673,7 @@ packCrsGraph typename CrsGraph::buffer_device_type >& export_pids, size_t& constant_num_packets, - const bool pack_pids, - Distributor& /* dist */) + const bool pack_pids) { using Kokkos::View; using crs_graph_type = CrsGraph; @@ -769,8 +763,7 @@ packCrsGraph (const CrsGraph& sourceGraph, Teuchos::Array::packet_type>& exports, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { using Kokkos::HostSpace; using Kokkos::MemoryUnmanaged; @@ -828,7 +821,7 @@ packCrsGraph (const CrsGraph& sourceGraph, PackCrsGraphImpl::packCrsGraph (sourceGraph, exports_dv, num_packets_per_lid_d, export_lids_d, - export_pids_d, constantNumPackets, pack_pids, distor); + export_pids_d, constantNumPackets, pack_pids); // The counts are an output of packCrsGraph, so we have to copy // them back to host. @@ -872,8 +865,7 @@ packCrsGraphNew (const CrsGraph& sourceGraph, typename CrsGraph::buffer_device_type > num_packets_per_lid, size_t& constant_num_packets, - const bool pack_pids, - Distributor& /* dist */) + const bool pack_pids) { using Kokkos::View; using crs_graph_type = CrsGraph; @@ -962,8 +954,7 @@ packCrsGraphWithOwningPIDs const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, const Teuchos::ArrayView& sourcePIDs, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { using Kokkos::HostSpace; using Kokkos::MemoryUnmanaged; @@ -998,7 +989,7 @@ packCrsGraphWithOwningPIDs constexpr bool pack_pids = true; PackCrsGraphImpl::packCrsGraph (sourceGraph, exports_dv, num_packets_per_lid_d, export_lids_d, - export_pids_d, constantNumPackets, pack_pids, distor); + export_pids_d, constantNumPackets, pack_pids); // The counts are an output of packCrsGraph, so we // have to copy them back to host. @@ -1017,8 +1008,7 @@ packCrsGraphWithOwningPIDs Teuchos::Array::packet_type>&, \ const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ - size_t&, \ - Distributor&); \ + size_t&); \ template void \ Details::packCrsGraphNew ( \ const CrsGraph&, \ @@ -1035,8 +1025,7 @@ packCrsGraphWithOwningPIDs size_t*, \ CrsGraph::buffer_device_type>, \ size_t&, \ - const bool, \ - Distributor&); \ + const bool); \ template void \ Details::packCrsGraphWithOwningPIDs ( \ const CrsGraph&, \ @@ -1044,7 +1033,6 @@ packCrsGraphWithOwningPIDs const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ - size_t&, \ - Distributor&); + size_t&); #endif // TPETRA_DETAILS_PACKCRSGRAPH_DEF_HPP diff --git a/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_decl.hpp b/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_decl.hpp index 9cf343d0fd89..f6b05702b389 100644 --- a/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_decl.hpp @@ -80,11 +80,6 @@ template class ArrayView; namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -116,8 +111,6 @@ namespace Details { /// to expect a possibly /// different ("nonconstant") number of packets per local index /// (i.e., a possibly different number of entries per row). /// -/// \param distor [in] The distributor (not used) -/// /// This is the public interface to the pack machinery /// converts passed Teuchos::ArrayView objects to Kokkos::View objects (and /// copies back in to the Teuchos::ArrayView objects, if needed). When @@ -129,8 +122,7 @@ packCrsMatrix (const CrsMatrix& sourceMatrix, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); /// \brief Pack specified entries of the given local sparse matrix for /// communication, for "new" DistObject interface. @@ -158,8 +150,6 @@ packCrsMatrix (const CrsMatrix& sourceMatrix, /// output argument of Tpetra::DistObject::packAndPrepare (which /// see). /// -/// \param distor [in] (Not used.) -/// /// This method implements CrsMatrix::packNew, and thus /// CrsMatrix::packAndPrepare, for the case where the matrix to /// pack has a valid KokkosSparse::CrsMatrix. @@ -172,8 +162,7 @@ packCrsMatrixNew (const CrsMatrix& sourceMatrix, typename DistObject::buffer_device_type>& numPacketsPerLID, const Kokkos::DualView::buffer_device_type>& exportLIDs, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); /// \brief Pack specified entries of the given local sparse matrix for /// communication. @@ -201,8 +190,6 @@ packCrsMatrixNew (const CrsMatrix& sourceMatrix, /// to expect a possibly /// different ("nonconstant") number of packets per local index /// (i.e., a possibly different number of entries per row). /// -/// \param distor [in] The distributor (not used) -/// /// This is the public interface to the pack machinery /// converts passed Teuchos::ArrayView objects to Kokkos::View objects (and /// copies back in to the Teuchos::ArrayView objects, if needed). When @@ -215,8 +202,7 @@ packCrsMatrixWithOwningPIDs (const CrsMatrix& sourceMatrix, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, const Teuchos::ArrayView& sourcePIDs, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); } // namespace Details } // namespace Tpetra diff --git a/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_def.hpp b/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_def.hpp index a0f83397581d..cc2969ac4d31 100644 --- a/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_def.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_def.hpp @@ -81,11 +81,6 @@ namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -731,8 +726,7 @@ packCrsMatrix (const CrsMatrix& sourceMatrix, const Kokkos::View& export_lids, const Kokkos::View& export_pids, size_t& constant_num_packets, - const bool pack_pids, - Distributor& /* dist */) + const bool pack_pids) { ::Tpetra::Details::ProfilingRegion region_pack_crs_matrix( "Tpetra::Details::PackCrsMatrixImpl::packCrsMatrix", @@ -864,8 +858,7 @@ packCrsMatrix (const CrsMatrix& sourceMatrix, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { using local_matrix_device_type = typename CrsMatrix::local_matrix_device_type; using device_type = typename local_matrix_device_type::device_type; @@ -899,7 +892,7 @@ packCrsMatrix (const CrsMatrix& sourceMatrix, constexpr bool pack_pids = false; PackCrsMatrixImpl::packCrsMatrix ( sourceMatrix, exports_dv, num_packets_per_lid_d, export_lids_d, - export_pids_d, constantNumPackets, pack_pids, distor); + export_pids_d, constantNumPackets, pack_pids); // The counts are an output of PackCrsMatrixImpl::packCrsMatrix, so we have to // copy them back to host. @@ -929,9 +922,7 @@ packCrsMatrixNew( Kokkos::DualView::buffer_device_type>& exports, const Kokkos::DualView::buffer_device_type>& numPacketsPerLID, const Kokkos::DualView::buffer_device_type>& exportLIDs, - size_t& constantNumPackets, - Distributor& distor -) + size_t& constantNumPackets) { using device_type = typename CrsMatrix::device_type; using buffer_device_type = typename DistObject::buffer_device_type; @@ -956,7 +947,7 @@ packCrsMatrixNew( ); PackCrsMatrixImpl::packCrsMatrix ( sourceMatrix, exports, numPacketsPerLID_d, exportLIDs_d, - exportPIDs_d, constantNumPackets, pack_pids, distor); + exportPIDs_d, constantNumPackets, pack_pids); } template @@ -966,8 +957,7 @@ packCrsMatrixWithOwningPIDs (const CrsMatrix& sourceMatrix, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, const Teuchos::ArrayView& sourcePIDs, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { typedef typename CrsMatrix::local_matrix_device_type local_matrix_device_type; typedef typename DistObject::buffer_device_type buffer_device_type; @@ -1027,7 +1017,7 @@ packCrsMatrixWithOwningPIDs (const CrsMatrix& sourceMatrix, try { PackCrsMatrixImpl::packCrsMatrix (sourceMatrix, exports_dv, num_packets_per_lid_d, export_lids_d, - export_pids_d, constantNumPackets, pack_pids, distor); + export_pids_d, constantNumPackets, pack_pids); } catch (std::exception& e) { if (verbose) { @@ -1091,22 +1081,19 @@ packCrsMatrixWithOwningPIDs (const CrsMatrix& sourceMatrix, Teuchos::Array&, \ const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ - size_t&, \ - Distributor&); \ + size_t&); \ template void \ Details::packCrsMatrixNew (const CrsMatrix&, \ Kokkos::DualView::buffer_device_type>&, \ const Kokkos::DualView::buffer_device_type>&, \ const Kokkos::DualView::buffer_device_type>&, \ - size_t&, \ - Distributor&); \ + size_t&); \ template void \ Details::packCrsMatrixWithOwningPIDs (const CrsMatrix&, \ Kokkos::DualView::buffer_device_type>&, \ const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ - size_t&, \ - Distributor&); + size_t&); #endif // TPETRA_DETAILS_PACKCRSMATRIX_DEF_HPP diff --git a/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_decl.hpp b/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_decl.hpp index 40e20c3cd5e3..f5ae653c4256 100644 --- a/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_decl.hpp @@ -78,11 +78,6 @@ template class ArrayView; namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -119,8 +114,6 @@ namespace Details { /// to expect a possibly /// different ("nonconstant") number of packets per local index /// (i.e., a possibly different number of entries per row). /// -/// \param distor [in] The distributor (not used) -/// /// \param combineMode [in] the mode to use for combining /// /// \param numSameIds [in] @@ -148,7 +141,6 @@ unpackAndCombineWithOwningPIDsCount( const Teuchos::ArrayView::packet_type> &imports, const Teuchos::ArrayView& numPacketsPerLID, size_t constantNumPackets, - Distributor &distor, CombineMode combineMode, size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, @@ -176,7 +168,6 @@ unpackAndCombineIntoCrsArrays( const Teuchos::ArrayView::packet_type>& imports, const Teuchos::ArrayView& numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode, const size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, diff --git a/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_def.hpp b/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_def.hpp index 0bbb0167f543..55c5c2cc01b8 100644 --- a/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_def.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_def.hpp @@ -76,11 +76,6 @@ namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -838,8 +833,6 @@ unpackAndCombineIntoCrsArrays( /// to expect a possibly /// different ("nonconstant") number of packets per local index /// (i.e., a possibly different number of entries per row). /// -/// \param distor [in] The distributor (not used) -/// /// \param combineMode [in] the mode to use for combining /// /// \param numSameIds [in] @@ -864,7 +857,6 @@ unpackAndCombineWithOwningPIDsCount( const Teuchos::ArrayView::packet_type> &imports, const Teuchos::ArrayView& numPacketsPerLID, size_t /* constantNumPackets */, - Distributor &/* distor */, CombineMode /* combineMode */, size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, @@ -936,7 +928,6 @@ unpackAndCombineIntoCrsArrays( const Teuchos::ArrayView::packet_type>& imports, const Teuchos::ArrayView& numPacketsPerLID, const size_t /* constantNumPackets */, - Distributor& /* distor */, const CombineMode /* combineMode */, const size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, @@ -1075,7 +1066,6 @@ unpackAndCombineIntoCrsArrays( const Teuchos::ArrayView::packet_type>&, \ const Teuchos::ArrayView&, \ const size_t, \ - Distributor&, \ const CombineMode, \ const size_t, \ const Teuchos::ArrayView&, \ @@ -1094,7 +1084,6 @@ unpackAndCombineIntoCrsArrays( const Teuchos::ArrayView::packet_type> &, \ const Teuchos::ArrayView&, \ size_t, \ - Distributor &, \ CombineMode, \ size_t, \ const Teuchos::ArrayView&, \ diff --git a/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_decl.hpp b/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_decl.hpp index 57d8ed190f7b..349a1fa0ca86 100644 --- a/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_decl.hpp @@ -78,11 +78,6 @@ template class ArrayView; namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -133,7 +128,6 @@ unpackCrsMatrixAndCombine (const CrsMatrix& sourceMatrix, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& importLIDs, size_t constantNumPackets, - Distributor & distor, CombineMode combineMode); template @@ -147,7 +141,6 @@ unpackCrsMatrixAndCombineNew( const Kokkos::DualView::buffer_device_type>& importLIDs, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode); /// \brief Special version of Tpetra::Details::unpackCrsMatrixAndCombine @@ -213,7 +206,6 @@ unpackAndCombineWithOwningPIDsCount ( const Teuchos::ArrayView &imports, const Teuchos::ArrayView& numPacketsPerLID, size_t constantNumPackets, - Distributor &distor, CombineMode combineMode, size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, @@ -241,7 +233,6 @@ unpackAndCombineIntoCrsArrays ( const Teuchos::ArrayView& imports, const Teuchos::ArrayView& numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode, const size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, diff --git a/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_def.hpp b/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_def.hpp index 7543b785a400..ed259b127de5 100644 --- a/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_def.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_def.hpp @@ -78,11 +78,6 @@ namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -1203,7 +1198,6 @@ unpackCrsMatrixAndCombine( const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& importLIDs, size_t /* constantNumPackets */, - Distributor & /* distor */, CombineMode combineMode) { using Kokkos::View; @@ -1259,7 +1253,6 @@ unpackCrsMatrixAndCombineNew( const Kokkos::DualView::buffer_device_type>& importLIDs, const size_t /* constantNumPackets */, - Distributor& /* distor */, const CombineMode combineMode) { using Kokkos::View; @@ -1362,7 +1355,6 @@ unpackAndCombineWithOwningPIDsCount ( const Teuchos::ArrayView &imports, const Teuchos::ArrayView& numPacketsPerLID, size_t /* constantNumPackets */, - Distributor &/* distor */, CombineMode /* combineMode */, size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, @@ -1433,7 +1425,6 @@ unpackAndCombineIntoCrsArrays ( const Teuchos::ArrayView& imports, const Teuchos::ArrayView& numPacketsPerLID, const size_t /* constantNumPackets */, - Distributor& /* distor */, const CombineMode /* combineMode */, const size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, @@ -1628,7 +1619,6 @@ unpackAndCombineIntoCrsArrays ( const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ size_t, \ - Distributor&, \ CombineMode); \ template void \ Details::unpackCrsMatrixAndCombineNew ( \ @@ -1637,7 +1627,6 @@ unpackAndCombineIntoCrsArrays ( Kokkos::DualView::buffer_device_type>, \ const Kokkos::DualView::buffer_device_type>&, \ const size_t, \ - Distributor&, \ const CombineMode); \ template void \ Details::unpackAndCombineIntoCrsArrays ( \ @@ -1646,7 +1635,6 @@ unpackAndCombineIntoCrsArrays ( const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ const size_t, \ - Distributor&, \ const CombineMode, \ const size_t, \ const Teuchos::ArrayView&, \ @@ -1666,7 +1654,6 @@ unpackAndCombineIntoCrsArrays ( const Teuchos::ArrayView &, \ const Teuchos::ArrayView&, \ size_t, \ - Distributor &, \ CombineMode, \ size_t, \ const Teuchos::ArrayView&, \ diff --git a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp index 175686c822bc..dffaede3fca6 100644 --- a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp @@ -897,9 +897,6 @@ namespace Tpetra { /// \param constantNumPackets [out] On exit, 0 if the number of /// packets per LID could differ, else (if nonzero) the number /// of packets per LID (which must be constant). - /// - /// \param distor [in] The Distributor object we are using. Most - /// implementations will not use this. virtual void packAndPrepare (const SrcDistObject& source, const Kokkos::DualView& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); /// \brief Perform any unpacking and combining after /// communication. @@ -948,9 +944,6 @@ namespace Tpetra { /// numPacketsPerLID[i] is the number of packets to /// unpack for LID importLIDs[i]. /// - /// \param distor [in] The Distributor object we are using. Most - /// implementations will not use this. - /// /// \param combineMode [in] The CombineMode to use when combining /// the imported entries with existing entries. virtual void @@ -961,7 +954,6 @@ namespace Tpetra { Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode); diff --git a/packages/tpetra/core/src/Tpetra_DistObject_def.hpp b/packages/tpetra/core/src/Tpetra_DistObject_def.hpp index 37f568176335..10e3df8f1673 100644 --- a/packages/tpetra/core/src/Tpetra_DistObject_def.hpp +++ b/packages/tpetra/core/src/Tpetra_DistObject_def.hpp @@ -1637,7 +1637,7 @@ namespace Tpetra { try { this->packAndPrepare (src, exportLIDs, this->exports_, this->numExportPacketsPerLID_, - constantNumPackets, distor); + constantNumPackets); lclSuccess = true; } catch (std::exception& e) { @@ -1659,7 +1659,7 @@ namespace Tpetra { else { this->packAndPrepare (src, exportLIDs, this->exports_, this->numExportPacketsPerLID_, - constantNumPackets, distor); + constantNumPackets); } } @@ -1689,7 +1689,7 @@ namespace Tpetra { try { this->unpackAndCombine (remoteLIDs, this->imports_, this->numImportPacketsPerLID_, - constantNumPackets, distor, CM); + constantNumPackets, CM); lclSuccess = true; } catch (std::exception& e) { @@ -1711,7 +1711,7 @@ namespace Tpetra { else { this->unpackAndCombine (remoteLIDs, this->imports_, this->numImportPacketsPerLID_, - constantNumPackets, distor, CM); + constantNumPackets, CM); } } @@ -1744,8 +1744,7 @@ namespace Tpetra { Kokkos::DualView< size_t*, buffer_device_type>, - size_t&, - Distributor&) + size_t&) {} template @@ -1762,7 +1761,6 @@ namespace Tpetra { size_t*, buffer_device_type> /* numPacketsPerLID */, const size_t /* constantNumPackets */, - Distributor& /* distor */, const CombineMode /* combineMode */) {} diff --git a/packages/tpetra/core/src/Tpetra_MultiVector_decl.hpp b/packages/tpetra/core/src/Tpetra_MultiVector_decl.hpp index 4ad40a64880a..1192859b11c5 100644 --- a/packages/tpetra/core/src/Tpetra_MultiVector_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_MultiVector_decl.hpp @@ -2563,8 +2563,7 @@ namespace Tpetra { Kokkos::DualView< size_t*, buffer_device_type> /* numPacketsPerLID */, - size_t& constantNumPackets, - Distributor& /* distor */); + size_t& constantNumPackets); virtual void unpackAndCombine @@ -2578,7 +2577,6 @@ namespace Tpetra { size_t*, buffer_device_type> /* numPacketsPerLID */, const size_t constantNumPackets, - Distributor& /* distor */, const CombineMode CM); private: diff --git a/packages/tpetra/core/src/Tpetra_MultiVector_def.hpp b/packages/tpetra/core/src/Tpetra_MultiVector_def.hpp index bd5a642286cc..aaed39769b78 100644 --- a/packages/tpetra/core/src/Tpetra_MultiVector_def.hpp +++ b/packages/tpetra/core/src/Tpetra_MultiVector_def.hpp @@ -1365,8 +1365,7 @@ namespace Tpetra { const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, Kokkos::DualView /* numExportPacketsPerLID */, - size_t& constantNumPackets, - Distributor & /* distor */ ) + size_t& constantNumPackets) { using ::Tpetra::Details::Behavior; using ::Tpetra::Details::ProfilingRegion; @@ -1742,7 +1741,6 @@ namespace Tpetra { Kokkos::DualView imports, Kokkos::DualView /* numPacketsPerLID */, const size_t constantNumPackets, - Distributor& /* distor */, const CombineMode CM) { using ::Tpetra::Details::Behavior; diff --git a/packages/tpetra/core/src/Tpetra_Packable.hpp b/packages/tpetra/core/src/Tpetra_Packable.hpp index 05ca5fcfeb63..9b155066d29b 100644 --- a/packages/tpetra/core/src/Tpetra_Packable.hpp +++ b/packages/tpetra/core/src/Tpetra_Packable.hpp @@ -54,9 +54,6 @@ namespace Teuchos { template class ArrayView; } // namespace Teuchos -namespace Tpetra { - class Distributor; // forward declaration -} // namespace Tpetra #endif // DOXYGEN_SHOULD_SKIP_THIS namespace Tpetra { @@ -118,16 +115,11 @@ namespace Tpetra { /// size for each local index). If nonzero, then it is expected /// that the number of packets per local index is constant, and /// that constantNumPackets is that value. - /// - /// \param distor [in] The Distributor object we are using. - /// Implementations may ignore this object. We provide it for - /// consistency with DistObject's packAndPrepare method. virtual void pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor &distor) const = 0; + size_t& constantNumPackets) const = 0; //! Destructor (virtual for memory safety of derived classes). virtual ~Packable () {} diff --git a/packages/tpetra/core/src/Tpetra_RowGraph_decl.hpp b/packages/tpetra/core/src/Tpetra_RowGraph_decl.hpp index a78074bc77e1..7e95073457b6 100644 --- a/packages/tpetra/core/src/Tpetra_RowGraph_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_RowGraph_decl.hpp @@ -322,8 +322,7 @@ namespace Tpetra { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const; + size_t& constantNumPackets) const; //@} }; // class RowGraph } // namespace Tpetra diff --git a/packages/tpetra/core/src/Tpetra_RowGraph_def.hpp b/packages/tpetra/core/src/Tpetra_RowGraph_def.hpp index 13c8b22c3877..8b840128edf9 100644 --- a/packages/tpetra/core/src/Tpetra_RowGraph_def.hpp +++ b/packages/tpetra/core/src/Tpetra_RowGraph_def.hpp @@ -42,8 +42,6 @@ #ifndef TPETRA_ROWGRAPH_DEF_HPP #define TPETRA_ROWGRAPH_DEF_HPP -#include "Tpetra_Distributor.hpp" // avoid error C2027: use of undefined type 'Tpetra::Distributor' at (void) distor below - namespace Tpetra { template void @@ -51,15 +49,13 @@ namespace Tpetra { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const + size_t& constantNumPackets) const { using Teuchos::Array; typedef LocalOrdinal LO; typedef GlobalOrdinal GO; typedef Map map_type; const char tfecfFuncName[] = "packAndPrepare"; - (void) distor; // forestall "unused argument" compiler warning TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC( exportLIDs.size() != numPacketsPerLID.size(), std::runtime_error, diff --git a/packages/tpetra/core/src/Tpetra_RowMatrix_decl.hpp b/packages/tpetra/core/src/Tpetra_RowMatrix_decl.hpp index 382e11b18472..a84e96dc9dcb 100644 --- a/packages/tpetra/core/src/Tpetra_RowMatrix_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_RowMatrix_decl.hpp @@ -565,8 +565,7 @@ namespace Tpetra { packImpl (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const; + size_t& constantNumPackets) const; public: @@ -582,8 +581,7 @@ namespace Tpetra { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const; + size_t& constantNumPackets) const; //@} }; // class RowMatrix } // namespace Tpetra diff --git a/packages/tpetra/core/src/Tpetra_RowMatrix_def.hpp b/packages/tpetra/core/src/Tpetra_RowMatrix_def.hpp index ee6ee36ef063..ad4f52c71ed1 100644 --- a/packages/tpetra/core/src/Tpetra_RowMatrix_def.hpp +++ b/packages/tpetra/core/src/Tpetra_RowMatrix_def.hpp @@ -302,8 +302,7 @@ namespace Tpetra { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor &distor) const + size_t& constantNumPackets) const { #ifdef HAVE_TPETRA_DEBUG const char tfecfFuncName[] = "pack: "; @@ -313,7 +312,7 @@ namespace Tpetra { int lclBad = 0; try { this->packImpl (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); } catch (std::exception& e) { lclBad = 1; msg << e.what (); @@ -342,7 +341,7 @@ namespace Tpetra { } #else this->packImpl (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); #endif // HAVE_TPETRA_DEBUG } @@ -491,8 +490,7 @@ namespace Tpetra { packImpl (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& /* distor */) const + size_t& constantNumPackets) const { using Teuchos::Array; using Teuchos::ArrayView; diff --git a/packages/tpetra/core/test/CrsGraph/CrsGraph_PackUnpack.cpp b/packages/tpetra/core/test/CrsGraph/CrsGraph_PackUnpack.cpp index 09f14dc46622..84e9ad9b3803 100644 --- a/packages/tpetra/core/test/CrsGraph/CrsGraph_PackUnpack.cpp +++ b/packages/tpetra/core/test/CrsGraph/CrsGraph_PackUnpack.cpp @@ -45,7 +45,6 @@ #include "TpetraCore_ETIHelperMacros.h" #include "Tpetra_CrsGraph.hpp" #include "Tpetra_Core.hpp" -#include "Tpetra_Distributor.hpp" #include "Tpetra_Map.hpp" #include "Tpetra_Details_gathervPrint.hpp" #include "Tpetra_Details_packCrsGraph.hpp" @@ -162,7 +161,6 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackThenUnpackAndCombine, LO, GO, NT Array exports; // output argument; to be realloc'd Array numPacketsPerLID (num_loc_rows, 0); // output argument size_t constantNumPackets; // output argument - Tpetra::Distributor distor (comm); // argument required, but not used out << "Calling packCrsGraph" << endl; @@ -171,7 +169,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackThenUnpackAndCombine, LO, GO, NT std::ostringstream msg; try { packCrsGraph(*A, exports, numPacketsPerLID(), exportLIDs(), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { local_op_ok = 0; @@ -331,13 +329,12 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackWithError, LO, GO, NT) Array exports; Array numPacketsPerLID(num_loc_rows, 0); size_t constantNumPackets; - Tpetra::Distributor distor(comm); { int local_op_ok; std::ostringstream msg; try { packCrsGraph(*A, exports, numPacketsPerLID(), exportLIDs(), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { local_op_ok = 0; @@ -374,14 +371,13 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackWithError, LO, GO, NT) Array exports; Array numPacketsPerLID(num_loc_rows, 0); size_t constantNumPackets; - Tpetra::Distributor distor(comm); out << "Calling packCrsGraph" << endl; { int local_op_ok; std::ostringstream msg; try { packCrsGraph(*A, exports, numPacketsPerLID(), exportLIDs(), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { local_op_ok = 0; diff --git a/packages/tpetra/core/test/CrsGraph/CrsGraph_UnpackIntoStaticGraph.cpp b/packages/tpetra/core/test/CrsGraph/CrsGraph_UnpackIntoStaticGraph.cpp index 93bce1345f00..c45620c22e2e 100644 --- a/packages/tpetra/core/test/CrsGraph/CrsGraph_UnpackIntoStaticGraph.cpp +++ b/packages/tpetra/core/test/CrsGraph/CrsGraph_UnpackIntoStaticGraph.cpp @@ -45,7 +45,6 @@ #include "TpetraCore_ETIHelperMacros.h" #include "Tpetra_CrsGraph.hpp" #include "Tpetra_Core.hpp" -#include "Tpetra_Distributor.hpp" #include "Tpetra_Map.hpp" #include "Tpetra_Details_gathervPrint.hpp" #include "Tpetra_Details_packCrsGraph.hpp" @@ -134,13 +133,6 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackThenUnpackAndCombine, LO, GO, NT auto num_packets_per_lid = Array(num_loc_rows, 0); // output argument size_t const_num_packets; // output argument - // We're not actually communicating in this test; we just need the Distributor - // for the interface of packCrsGraph (which doesn't use it). Consider changing - // packCrsGraph's interface so it doesn't take a Distributor? No, because - // Distributor has index permutation information that we could use to pack in - // a particular order and thus avoid the slow path in Distributor::doPosts. - auto distor = Tpetra::Distributor(comm); - out << "Calling packCrsGraph" << endl; { @@ -148,7 +140,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackThenUnpackAndCombine, LO, GO, NT std::ostringstream msg; try { packCrsGraph(*B, exports, num_packets_per_lid(), export_lids(), - const_num_packets, distor); + const_num_packets); local_op_ok = 1; } catch (std::exception& e) { local_op_ok = 0; @@ -178,7 +170,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackThenUnpackAndCombine, LO, GO, NT int local_op_ok; std::ostringstream msg; unpackCrsGraphAndCombine(*A, exports, num_packets_per_lid(), - export_lids(), const_num_packets, distor, Tpetra::REPLACE); + export_lids(), const_num_packets, Tpetra::REPLACE); local_op_ok = 1; TEST_ASSERT(local_op_ok == 1); diff --git a/packages/tpetra/core/test/CrsMatrix/CrsMatrix_PackUnpack.cpp b/packages/tpetra/core/test/CrsMatrix/CrsMatrix_PackUnpack.cpp index 5331c90da1f8..a3f81430adc0 100644 --- a/packages/tpetra/core/test/CrsMatrix/CrsMatrix_PackUnpack.cpp +++ b/packages/tpetra/core/test/CrsMatrix/CrsMatrix_PackUnpack.cpp @@ -43,7 +43,6 @@ #include "TpetraCore_ETIHelperMacros.h" #include "Tpetra_CrsMatrix.hpp" #include "Tpetra_Core.hpp" -#include "Tpetra_Distributor.hpp" #include "Tpetra_Map.hpp" #include "Tpetra_Details_gathervPrint.hpp" #include "Tpetra_Details_packCrsMatrix.hpp" @@ -170,7 +169,6 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackThenUnpackAndCombine, SC, LO, G Array exports; // output argument; to be realloc'd Array numPacketsPerLID (num_loc_rows, 0); // output argument size_t constantNumPackets; // output argument - Tpetra::Distributor distor (comm); // argument required, but not used out << "Calling packCrsMatrix" << endl; @@ -180,7 +178,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackThenUnpackAndCombine, SC, LO, G try { packCrsMatrix (*A, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { @@ -220,7 +218,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackThenUnpackAndCombine, SC, LO, G try { unpackCrsMatrixAndCombine (*B, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor, + constantNumPackets, Tpetra::REPLACE); local_op_ok = 1; } @@ -319,7 +317,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackThenUnpackAndCombine, SC, LO, G try { unpackCrsMatrixAndCombine (*B, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor, + constantNumPackets, Tpetra::ADD); local_op_ok = 1; } @@ -438,14 +436,13 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackWithError, SC, LO, GO, NT) Array exports; Array numPacketsPerLID(num_loc_rows, 0); size_t constantNumPackets; - Tpetra::Distributor distor(comm); { int local_op_ok; std::ostringstream msg; try { packCrsMatrix (*A, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { @@ -483,7 +480,6 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackWithError, SC, LO, GO, NT) Array exports; Array numPacketsPerLID(num_loc_rows, 0); size_t constantNumPackets; - Tpetra::Distributor distor(comm); out << "Calling packCrsMatrix" << endl; { int local_op_ok; @@ -491,7 +487,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackWithError, SC, LO, GO, NT) try { packCrsMatrix (*A, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { @@ -546,7 +542,6 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackPartial, SC, LO, GO, NT) Array exports; // output argument; to be realloc'd Array numPacketsPerLID (num_loc_rows, 0); // output argument size_t constantNumPackets; // output argument - Tpetra::Distributor distor (comm); // argument required, but not used out << "Calling packCrsMatrix" << endl; @@ -556,7 +551,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackPartial, SC, LO, GO, NT) try { packCrsMatrix (*A, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { @@ -596,7 +591,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackPartial, SC, LO, GO, NT) try { unpackCrsMatrixAndCombine (*B, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor, + constantNumPackets, Tpetra::REPLACE); local_op_ok = 1; } diff --git a/packages/tpetra/core/test/CrsMatrix/CrsMatrix_createDeepCopy.cpp b/packages/tpetra/core/test/CrsMatrix/CrsMatrix_createDeepCopy.cpp index 349977191014..da5894ede2ea 100644 --- a/packages/tpetra/core/test/CrsMatrix/CrsMatrix_createDeepCopy.cpp +++ b/packages/tpetra/core/test/CrsMatrix/CrsMatrix_createDeepCopy.cpp @@ -264,11 +264,10 @@ class MyRowGraph : public Tpetra::RowGraph { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Tpetra::Distributor& distor) const override + size_t& constantNumPackets) const override { return G_->pack (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); } private: @@ -502,11 +501,10 @@ class MyRowMatrix : public Tpetra::RowMatrix { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Tpetra::Distributor& distor) const override + size_t& constantNumPackets) const override { return A_->pack (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); } void diff --git a/packages/tpetra/core/test/ImportExport2/ImportExport2_UnitTests.cpp b/packages/tpetra/core/test/ImportExport2/ImportExport2_UnitTests.cpp index d30147695045..95ec17d71174 100644 --- a/packages/tpetra/core/test/ImportExport2/ImportExport2_UnitTests.cpp +++ b/packages/tpetra/core/test/ImportExport2/ImportExport2_UnitTests.cpp @@ -2303,7 +2303,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL( Import_Util, UnpackAndCombineWithOwningPIDs, } Tpetra::Details::packCrsMatrixWithOwningPIDs( *A, exports, numExportPackets(), Importer->getExportLIDs(), - SourcePids(), constantNumPackets, distor); + SourcePids(), constantNumPackets); if (verbose) { std::ostringstream os; os << *prefix << "Done with packCrsMatrixWithOwningPIDs" << std::endl; @@ -2370,7 +2370,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL( Import_Util, UnpackAndCombineWithOwningPIDs, size_t nnz2 = unpackAndCombineWithOwningPIDsCount (*A, Importer->getRemoteLIDs (), imports (), numImportPackets (), - constantNumPackets, distor, + constantNumPackets, Tpetra::INSERT, Importer->getNumSameIDs (), Importer->getPermuteToLIDs (), @@ -2406,7 +2406,6 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL( Import_Util, UnpackAndCombineWithOwningPIDs, imports (), numImportPackets (), constantNumPackets, - distor, Tpetra::INSERT, Importer->getNumSameIDs (), Importer->getPermuteToLIDs (), From 6dc9c660d12f778395d5f3a25a87c4434ec90a54 Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Mon, 16 Aug 2021 09:35:00 -0600 Subject: [PATCH 02/70] Ifpack2: update usage of the Tpetra matrix pack API --- packages/ifpack2/test/unit_tests/Ifpack2_UnitTestHelpers.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestHelpers.hpp b/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestHelpers.hpp index 14aa05e3926a..b11516eaf06c 100644 --- a/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestHelpers.hpp +++ b/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestHelpers.hpp @@ -1070,9 +1070,8 @@ Teuchos::RCP > c pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Tpetra::Distributor& distor) const - {A_->pack(exportLIDs,exports,numPacketsPerLID,constantNumPackets,distor);} + size_t& constantNumPackets) const + {A_->pack(exportLIDs,exports,numPacketsPerLID,constantNumPackets);} private: Teuchos::RCP > A_; From 11aadcce4c187d47be304d236f4d41b5d98c9524 Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Sat, 14 Aug 2021 15:40:10 -0600 Subject: [PATCH 03/70] Tpetra: Remove Distributor from pack/unpack interfaces --- .../core/inout/Tpetra_Details_CooMatrix.hpp | 4 +- .../core/src/Tpetra_BlockCrsMatrix_decl.hpp | 4 +- .../core/src/Tpetra_BlockCrsMatrix_def.hpp | 4 +- .../core/src/Tpetra_BlockMultiVector_decl.hpp | 4 +- .../core/src/Tpetra_BlockMultiVector_def.hpp | 4 +- .../tpetra/core/src/Tpetra_CrsGraph_decl.hpp | 13 ++----- .../tpetra/core/src/Tpetra_CrsGraph_def.hpp | 31 +++++++-------- .../tpetra/core/src/Tpetra_CrsMatrix_decl.hpp | 14 ++----- .../tpetra/core/src/Tpetra_CrsMatrix_def.hpp | 38 +++++++------------ .../src/Tpetra_Details_packCrsGraph_decl.hpp | 20 ++-------- .../src/Tpetra_Details_packCrsGraph_def.hpp | 30 +++++---------- .../src/Tpetra_Details_packCrsMatrix_decl.hpp | 20 ++-------- .../src/Tpetra_Details_packCrsMatrix_def.hpp | 33 +++++----------- ..._Details_unpackCrsGraphAndCombine_decl.hpp | 9 ----- ...a_Details_unpackCrsGraphAndCombine_def.hpp | 11 ------ ...Details_unpackCrsMatrixAndCombine_decl.hpp | 9 ----- ..._Details_unpackCrsMatrixAndCombine_def.hpp | 13 ------- .../core/src/Tpetra_DistObject_decl.hpp | 10 +---- .../tpetra/core/src/Tpetra_DistObject_def.hpp | 12 +++--- .../core/src/Tpetra_MultiVector_decl.hpp | 4 +- .../core/src/Tpetra_MultiVector_def.hpp | 4 +- packages/tpetra/core/src/Tpetra_Packable.hpp | 10 +---- .../tpetra/core/src/Tpetra_RowGraph_decl.hpp | 3 +- .../tpetra/core/src/Tpetra_RowGraph_def.hpp | 6 +-- .../tpetra/core/src/Tpetra_RowMatrix_decl.hpp | 6 +-- .../tpetra/core/src/Tpetra_RowMatrix_def.hpp | 10 ++--- .../test/CrsGraph/CrsGraph_PackUnpack.cpp | 10 ++--- .../CrsGraph_UnpackIntoStaticGraph.cpp | 12 +----- .../test/CrsMatrix/CrsMatrix_PackUnpack.cpp | 19 ++++------ .../CrsMatrix/CrsMatrix_createDeepCopy.cpp | 10 ++--- .../ImportExport2/ImportExport2_UnitTests.cpp | 5 +-- 31 files changed, 99 insertions(+), 283 deletions(-) diff --git a/packages/tpetra/core/inout/Tpetra_Details_CooMatrix.hpp b/packages/tpetra/core/inout/Tpetra_Details_CooMatrix.hpp index 1a32d7e7f611..6d8a159274e1 100644 --- a/packages/tpetra/core/inout/Tpetra_Details_CooMatrix.hpp +++ b/packages/tpetra/core/inout/Tpetra_Details_CooMatrix.hpp @@ -1209,8 +1209,7 @@ class CooMatrix : public ::Tpetra::DistObject { buffer_device_type>& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - ::Tpetra::Distributor& /* distor */) + size_t& constantNumPackets) { using Teuchos::Comm; using Teuchos::RCP; @@ -1406,7 +1405,6 @@ class CooMatrix : public ::Tpetra::DistObject { Kokkos::DualView numPacketsPerLID, const size_t /* constantNumPackets */, - ::Tpetra::Distributor& /* distor */, const ::Tpetra::CombineMode /* combineMode */) { using Teuchos::Comm; diff --git a/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_decl.hpp b/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_decl.hpp index 06b10eef9890..ce7b1f7cd89e 100644 --- a/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_decl.hpp @@ -740,8 +740,7 @@ class BlockCrsMatrix : buffer_device_type>& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& /* distor */) override; + size_t& constantNumPackets) override; virtual void unpackAndCombine @@ -752,7 +751,6 @@ class BlockCrsMatrix : Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& /* distor */, const CombineMode combineMode) override; //@} diff --git a/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_def.hpp b/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_def.hpp index 6e35e3cf2b72..1da618cd6d1e 100644 --- a/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_def.hpp +++ b/packages/tpetra/core/src/Tpetra_BlockCrsMatrix_def.hpp @@ -2377,8 +2377,7 @@ class GetLocalDiagCopy { buffer_device_type>& exports, // output Kokkos::DualView numPacketsPerLID, // output - size_t& constantNumPackets, - Distributor& /* distor */) + size_t& constantNumPackets) { using ::Tpetra::Details::Behavior; using ::Tpetra::Details::dualViewStatusToString; @@ -2634,7 +2633,6 @@ class GetLocalDiagCopy { Kokkos::DualView numPacketsPerLID, const size_t /* constantNumPackets */, - Distributor& /* distor */, const CombineMode combineMode) { using ::Tpetra::Details::Behavior; diff --git a/packages/tpetra/core/src/Tpetra_BlockMultiVector_decl.hpp b/packages/tpetra/core/src/Tpetra_BlockMultiVector_decl.hpp index d7f9b0a82141..0250ff716c43 100644 --- a/packages/tpetra/core/src/Tpetra_BlockMultiVector_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_BlockMultiVector_decl.hpp @@ -665,8 +665,7 @@ class BlockMultiVector : buffer_device_type>& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); virtual void unpackAndCombine @@ -677,7 +676,6 @@ class BlockMultiVector : Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode); //@} diff --git a/packages/tpetra/core/src/Tpetra_BlockMultiVector_def.hpp b/packages/tpetra/core/src/Tpetra_BlockMultiVector_def.hpp index 96603bd020b5..b9a44c49a199 100644 --- a/packages/tpetra/core/src/Tpetra_BlockMultiVector_def.hpp +++ b/packages/tpetra/core/src/Tpetra_BlockMultiVector_def.hpp @@ -571,8 +571,7 @@ packAndPrepare buffer_device_type>& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { TEUCHOS_TEST_FOR_EXCEPTION (true, std::logic_error, @@ -590,7 +589,6 @@ unpackAndCombine Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode) { TEUCHOS_TEST_FOR_EXCEPTION diff --git a/packages/tpetra/core/src/Tpetra_CrsGraph_decl.hpp b/packages/tpetra/core/src/Tpetra_CrsGraph_decl.hpp index a7bc7f167021..ed4f882a85ab 100644 --- a/packages/tpetra/core/src/Tpetra_CrsGraph_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_CrsGraph_decl.hpp @@ -1302,22 +1302,19 @@ namespace Tpetra { const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) override; + size_t& constantNumPackets) override; virtual void pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const override; + size_t& constantNumPackets) const override; void packFillActive (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const; + size_t& constantNumPackets) const; void packFillActiveNew (const Kokkos::DualView& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const; + size_t& constantNumPackets) const; virtual void unpackAndCombine @@ -1338,7 +1334,6 @@ namespace Tpetra { Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode) override; //@} diff --git a/packages/tpetra/core/src/Tpetra_CrsGraph_def.hpp b/packages/tpetra/core/src/Tpetra_CrsGraph_def.hpp index 06c7ceb0edbc..8e609554dde0 100644 --- a/packages/tpetra/core/src/Tpetra_CrsGraph_def.hpp +++ b/packages/tpetra/core/src/Tpetra_CrsGraph_def.hpp @@ -5690,8 +5690,7 @@ namespace Tpetra { buffer_device_type>& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { using Tpetra::Details::ProfilingRegion; using GO = global_ordinal_type; @@ -5759,7 +5758,7 @@ namespace Tpetra { ArrayView numPacketsPerLID_av (numPacketsPerLID_h.data (), numPacketsPerLID_h.extent (0)); srcRowGraphPtr->pack (exportLIDs_av, exports_a, numPacketsPerLID_av, - constantNumPackets, distor); + constantNumPackets); const size_t newSize = static_cast (exports_a.size ()); if (static_cast (exports.extent (0)) != newSize) { using exports_dv_type = Kokkos::DualView; @@ -5788,11 +5787,11 @@ namespace Tpetra { using Tpetra::Details::packCrsGraphNew; packCrsGraphNew (*srcCrsGraphPtr, exportLIDs, exportPIDs, exports, numPacketsPerLID, - constantNumPackets, false, distor); + constantNumPackets, false); } else { srcCrsGraphPtr->packFillActiveNew (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); } if (verbose) { @@ -5808,19 +5807,18 @@ namespace Tpetra { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const + size_t& constantNumPackets) const { auto col_map = this->getColMap(); // packCrsGraph requires k_rowPtrsPacked to be set if( !col_map.is_null() && (rowPtrsPacked_dev_.extent(0) != 0 || getRowMap()->getNodeNumElements() ==0)) { using Tpetra::Details::packCrsGraph; packCrsGraph(*this, exports, numPacketsPerLID, - exportLIDs, constantNumPackets, distor); + exportLIDs, constantNumPackets); } else { this->packFillActive(exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); } } @@ -5830,8 +5828,7 @@ namespace Tpetra { packFillActive (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& /* distor */) const + size_t& constantNumPackets) const { using std::endl; using LO = LocalOrdinal; @@ -6021,8 +6018,7 @@ namespace Tpetra { buffer_device_type>& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const + size_t& constantNumPackets) const { using std::endl; using LO = local_ordinal_type; @@ -6251,7 +6247,6 @@ namespace Tpetra { Kokkos::DualView numPacketsPerLID, const size_t /* constantNumPackets */, - Distributor& /* distor */, const CombineMode /* combineMode */ ) { using Details::ProfilingRegion; @@ -7067,7 +7062,7 @@ namespace Tpetra { // The basic algorithm here is: // - // 1. Call the moral equivalent of "distor.do" to handle the import. + // 1. Call the moral equivalent of "Distor.do" to handle the import. // 2. Copy all the Imported and Copy/Permuted data into the raw // CrsGraph pointers, still using GIDs. // 3. Call an optimized version of MakeColMap that avoids the @@ -7274,7 +7269,7 @@ namespace Tpetra { // Pack & Prepare w/ owning PIDs packCrsGraphWithOwningPIDs(*this, destGraph->exports_, numExportPacketsPerLID, ExportLIDs, - SourcePids, constantNumPackets, Distor); + SourcePids, constantNumPackets); } // Do the exchange of remote data. @@ -7401,7 +7396,7 @@ namespace Tpetra { size_t mynnz = unpackAndCombineWithOwningPIDsCount(*this, RemoteLIDs, hostImports, numImportPacketsPerLID, - constantNumPackets, Distor, INSERT, + constantNumPackets, INSERT, NumSameIDs, PermuteToLIDs, PermuteFromLIDs); size_t N = BaseRowMap->getNodeNumElements(); @@ -7427,7 +7422,7 @@ namespace Tpetra { // takes five methods. unpackAndCombineIntoCrsArrays(*this, RemoteLIDs, hostImports, numImportPacketsPerLID, constantNumPackets, - Distor, INSERT, NumSameIDs, PermuteToLIDs, + INSERT, NumSameIDs, PermuteToLIDs, PermuteFromLIDs, N, mynnz, MyPID, CSR_rowptr(), CSR_colind_GID(), SourcePids(), TargetPids); diff --git a/packages/tpetra/core/src/Tpetra_CrsMatrix_decl.hpp b/packages/tpetra/core/src/Tpetra_CrsMatrix_decl.hpp index 7bc82914263f..555e10b451d9 100644 --- a/packages/tpetra/core/src/Tpetra_CrsMatrix_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_CrsMatrix_decl.hpp @@ -3074,8 +3074,7 @@ namespace Tpetra { buffer_device_type>& exportLIDs, Kokkos::DualView& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) override; + size_t& constantNumPackets) override; private: /// \brief Unpack the imported column indices and values, and @@ -3087,7 +3086,6 @@ namespace Tpetra { Kokkos::DualView imports, Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode, const bool verbose); @@ -3100,7 +3098,6 @@ namespace Tpetra { Kokkos::DualView imports, Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode); public: @@ -3119,7 +3116,6 @@ namespace Tpetra { Kokkos::DualView imports, Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode CM) override; /// \brief Pack this object's data for an Import or Export. @@ -3138,8 +3134,6 @@ namespace Tpetra { /// output, then that number gives the constant number of /// entries for all packed rows on all processes in the /// matrix's communicator. - /// \param distor [in/out] The Distributor object which implements - /// the Import or Export operation that is calling this method. /// /// \subsection Tpetra_CrsMatrix_packNew_summary Packing scheme /// @@ -3233,8 +3227,7 @@ namespace Tpetra { packNew (const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, const Kokkos::DualView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& dist) const; + size_t& constantNumPackets) const; private: /// \brief Pack this matrix (part of implementation of packAndPrepare). @@ -3247,8 +3240,7 @@ namespace Tpetra { packNonStaticNew (const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, const Kokkos::DualView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const; + size_t& constantNumPackets) const; /// \brief Pack data for the current row to send. /// diff --git a/packages/tpetra/core/src/Tpetra_CrsMatrix_def.hpp b/packages/tpetra/core/src/Tpetra_CrsMatrix_def.hpp index 6b89763912da..0147280f6659 100644 --- a/packages/tpetra/core/src/Tpetra_CrsMatrix_def.hpp +++ b/packages/tpetra/core/src/Tpetra_CrsMatrix_def.hpp @@ -6521,8 +6521,7 @@ CrsMatrix:: const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { using Details::Behavior; using Details::dualViewStatusToString; @@ -6599,7 +6598,7 @@ CrsMatrix:: } try { srcCrsMat->packNew (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); } catch (std::exception& e) { lclBad = 1; @@ -6659,7 +6658,7 @@ CrsMatrix:: // Teuchos::Array* objects. try { srcRowMat->pack (exportLIDs_av, exports_a, numPacketsPerLID_av, - constantNumPackets, distor); + constantNumPackets); } catch (std::exception& e) { lclBad = 1; @@ -6996,19 +6995,18 @@ CrsMatrix:: packNew (const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, const Kokkos::DualView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& dist) const + size_t& constantNumPackets) const { // The call to packNew in packAndPrepare catches and handles any exceptions. Details::ProfilingRegion region_pack_new("Tpetra::CrsMatrix::packNew", "Import/Export"); if (this->isStaticGraph ()) { using ::Tpetra::Details::packCrsMatrixNew; packCrsMatrixNew (*this, exports, numPacketsPerLID, exportLIDs, - constantNumPackets, dist); + constantNumPackets); } else { this->packNonStaticNew (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, dist); + constantNumPackets); } } @@ -7018,8 +7016,7 @@ CrsMatrix:: packNonStaticNew (const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, const Kokkos::DualView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& /* distor */) const + size_t& constantNumPackets) const { using Details::Behavior; using Details::dualViewStatusToString; @@ -7302,7 +7299,6 @@ CrsMatrix:: Kokkos::DualView imports, Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode) { using Details::Behavior; @@ -7374,7 +7370,7 @@ CrsMatrix:: int lclBad = 0; try { unpackAndCombineImpl(importLIDs, imports, numPacketsPerLID, - constantNumPackets, distor, combineMode, + constantNumPackets, combineMode, verbose); } catch (std::exception& e) { lclBad = 1; @@ -7401,7 +7397,7 @@ CrsMatrix:: } else { unpackAndCombineImpl(importLIDs, imports, numPacketsPerLID, - constantNumPackets, distor, combineMode, + constantNumPackets, combineMode, verbose); } @@ -7430,7 +7426,6 @@ CrsMatrix:: Kokkos::DualView imports, Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor & distor, const CombineMode combineMode, const bool verbose) { @@ -7460,7 +7455,7 @@ CrsMatrix:: using Details::unpackCrsMatrixAndCombineNew; unpackCrsMatrixAndCombineNew(*this, imports, numPacketsPerLID, importLIDs, constantNumPackets, - distor, combineMode); + combineMode); } else { { @@ -7495,7 +7490,7 @@ CrsMatrix:: unpackAndCombineImplNonStatic(importLIDs, imports, numPacketsPerLID, constantNumPackets, - distor, combineMode); + combineMode); } if (verbose) { @@ -7514,7 +7509,6 @@ CrsMatrix:: Kokkos::DualView imports, Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode) { using Kokkos::View; @@ -8348,7 +8342,7 @@ CrsMatrix:: // The basic algorithm here is: // - // 1. Call the moral equivalent of "distor.do" to handle the import. + // 1. Call the moral equivalent of "Distor.do" to handle the import. // 2. Copy all the Imported and Copy/Permuted data into the raw // CrsMatrix / CrsGraphData pointers, still using GIDs. // 3. Call an optimized version of MakeColMap that avoids the @@ -8603,8 +8597,7 @@ CrsMatrix:: numExportPacketsPerLID, ExportLIDs, SourcePids, - constantNumPackets, - Distor); + constantNumPackets); } catch (std::exception& e) { errStrm << "Proc " << myRank << ": packCrsMatrixWithOwningPIDs threw: " @@ -8650,8 +8643,7 @@ CrsMatrix:: numExportPacketsPerLID, ExportLIDs, SourcePids, - constantNumPackets, - Distor); + constantNumPackets); if (verbose) { std::ostringstream os; os << *verbosePrefix << "Done with packCrsMatrixWithOwningPIDs" @@ -8892,7 +8884,6 @@ CrsMatrix:: hostImports, numImportPacketsPerLID, constantNumPackets, - Distor, INSERT, NumSameIDs, PermuteToLIDs, @@ -8938,7 +8929,6 @@ CrsMatrix:: hostImports, numImportPacketsPerLID, constantNumPackets, - Distor, INSERT, NumSameIDs, PermuteToLIDs, diff --git a/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_decl.hpp b/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_decl.hpp index e7c2c395fca6..08f569731930 100644 --- a/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_decl.hpp @@ -79,11 +79,6 @@ template class ArrayView; namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -112,8 +107,6 @@ namespace Details { /// to expect a possibly /// different ("nonconstant") number of packets per local index /// (i.e., a possibly different number of entries per row). /// -/// \param distor [in] The distributor (not used) -/// /// This is the public interface to the pack machinery /// converts passed Teuchos::ArrayView objects to Kokkos::View objects (and /// copies back in to the Teuchos::ArrayView objects, if needed). When @@ -125,8 +118,7 @@ packCrsGraph (const CrsGraph& sourceGraph, Teuchos::Array::packet_type>& exports, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); /// \brief Pack specified entries of the given local sparse graph for /// communication, for "new" DistObject interface. @@ -152,8 +144,6 @@ packCrsGraph (const CrsGraph& sourceGraph, /// output argument of Tpetra::DistObject::packAndPrepare (which /// see). /// -/// \param distor [in] (Not used.) -/// /// This method implements CrsGraph::packNew, and thus /// CrsGraph::packAndPrepare, for the case where the graph to /// pack has a valid KokkosSparse::CrsGraph. @@ -177,8 +167,7 @@ packCrsGraphNew (const CrsGraph& sourceGraph, typename CrsGraph::buffer_device_type > numPacketsPerLID, size_t& constantNumPackets, - const bool pack_pids, - Distributor& distor); + const bool pack_pids); /// \brief Pack specified entries of the given local sparse graph for /// communication. @@ -203,8 +192,6 @@ packCrsGraphNew (const CrsGraph& sourceGraph, /// to expect a possibly /// different ("nonconstant") number of packets per local index /// (i.e., a possibly different number of entries per row). /// -/// \param distor [in] The distributor (not used) -/// /// This is the public interface to the pack machinery /// converts passed Teuchos::ArrayView objects to Kokkos::View objects (and /// copies back in to the Teuchos::ArrayView objects, if needed). When @@ -219,8 +206,7 @@ packCrsGraphWithOwningPIDs (const CrsGraph& sourceGraph, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, const Teuchos::ArrayView& sourcePIDs, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); } // namespace Details } // namespace Tpetra diff --git a/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_def.hpp b/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_def.hpp index 479733fe8c9d..5e1af8288134 100644 --- a/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_def.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_packCrsGraph_def.hpp @@ -77,11 +77,6 @@ namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -678,8 +673,7 @@ packCrsGraph typename CrsGraph::buffer_device_type >& export_pids, size_t& constant_num_packets, - const bool pack_pids, - Distributor& /* dist */) + const bool pack_pids) { using Kokkos::View; using crs_graph_type = CrsGraph; @@ -769,8 +763,7 @@ packCrsGraph (const CrsGraph& sourceGraph, Teuchos::Array::packet_type>& exports, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { using Kokkos::HostSpace; using Kokkos::MemoryUnmanaged; @@ -828,7 +821,7 @@ packCrsGraph (const CrsGraph& sourceGraph, PackCrsGraphImpl::packCrsGraph (sourceGraph, exports_dv, num_packets_per_lid_d, export_lids_d, - export_pids_d, constantNumPackets, pack_pids, distor); + export_pids_d, constantNumPackets, pack_pids); // The counts are an output of packCrsGraph, so we have to copy // them back to host. @@ -872,8 +865,7 @@ packCrsGraphNew (const CrsGraph& sourceGraph, typename CrsGraph::buffer_device_type > num_packets_per_lid, size_t& constant_num_packets, - const bool pack_pids, - Distributor& /* dist */) + const bool pack_pids) { using Kokkos::View; using crs_graph_type = CrsGraph; @@ -962,8 +954,7 @@ packCrsGraphWithOwningPIDs const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, const Teuchos::ArrayView& sourcePIDs, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { using Kokkos::HostSpace; using Kokkos::MemoryUnmanaged; @@ -998,7 +989,7 @@ packCrsGraphWithOwningPIDs constexpr bool pack_pids = true; PackCrsGraphImpl::packCrsGraph (sourceGraph, exports_dv, num_packets_per_lid_d, export_lids_d, - export_pids_d, constantNumPackets, pack_pids, distor); + export_pids_d, constantNumPackets, pack_pids); // The counts are an output of packCrsGraph, so we // have to copy them back to host. @@ -1017,8 +1008,7 @@ packCrsGraphWithOwningPIDs Teuchos::Array::packet_type>&, \ const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ - size_t&, \ - Distributor&); \ + size_t&); \ template void \ Details::packCrsGraphNew ( \ const CrsGraph&, \ @@ -1035,8 +1025,7 @@ packCrsGraphWithOwningPIDs size_t*, \ CrsGraph::buffer_device_type>, \ size_t&, \ - const bool, \ - Distributor&); \ + const bool); \ template void \ Details::packCrsGraphWithOwningPIDs ( \ const CrsGraph&, \ @@ -1044,7 +1033,6 @@ packCrsGraphWithOwningPIDs const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ - size_t&, \ - Distributor&); + size_t&); #endif // TPETRA_DETAILS_PACKCRSGRAPH_DEF_HPP diff --git a/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_decl.hpp b/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_decl.hpp index 9cf343d0fd89..f6b05702b389 100644 --- a/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_decl.hpp @@ -80,11 +80,6 @@ template class ArrayView; namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -116,8 +111,6 @@ namespace Details { /// to expect a possibly /// different ("nonconstant") number of packets per local index /// (i.e., a possibly different number of entries per row). /// -/// \param distor [in] The distributor (not used) -/// /// This is the public interface to the pack machinery /// converts passed Teuchos::ArrayView objects to Kokkos::View objects (and /// copies back in to the Teuchos::ArrayView objects, if needed). When @@ -129,8 +122,7 @@ packCrsMatrix (const CrsMatrix& sourceMatrix, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); /// \brief Pack specified entries of the given local sparse matrix for /// communication, for "new" DistObject interface. @@ -158,8 +150,6 @@ packCrsMatrix (const CrsMatrix& sourceMatrix, /// output argument of Tpetra::DistObject::packAndPrepare (which /// see). /// -/// \param distor [in] (Not used.) -/// /// This method implements CrsMatrix::packNew, and thus /// CrsMatrix::packAndPrepare, for the case where the matrix to /// pack has a valid KokkosSparse::CrsMatrix. @@ -172,8 +162,7 @@ packCrsMatrixNew (const CrsMatrix& sourceMatrix, typename DistObject::buffer_device_type>& numPacketsPerLID, const Kokkos::DualView::buffer_device_type>& exportLIDs, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); /// \brief Pack specified entries of the given local sparse matrix for /// communication. @@ -201,8 +190,6 @@ packCrsMatrixNew (const CrsMatrix& sourceMatrix, /// to expect a possibly /// different ("nonconstant") number of packets per local index /// (i.e., a possibly different number of entries per row). /// -/// \param distor [in] The distributor (not used) -/// /// This is the public interface to the pack machinery /// converts passed Teuchos::ArrayView objects to Kokkos::View objects (and /// copies back in to the Teuchos::ArrayView objects, if needed). When @@ -215,8 +202,7 @@ packCrsMatrixWithOwningPIDs (const CrsMatrix& sourceMatrix, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, const Teuchos::ArrayView& sourcePIDs, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); } // namespace Details } // namespace Tpetra diff --git a/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_def.hpp b/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_def.hpp index a0f83397581d..cc2969ac4d31 100644 --- a/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_def.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_packCrsMatrix_def.hpp @@ -81,11 +81,6 @@ namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -731,8 +726,7 @@ packCrsMatrix (const CrsMatrix& sourceMatrix, const Kokkos::View& export_lids, const Kokkos::View& export_pids, size_t& constant_num_packets, - const bool pack_pids, - Distributor& /* dist */) + const bool pack_pids) { ::Tpetra::Details::ProfilingRegion region_pack_crs_matrix( "Tpetra::Details::PackCrsMatrixImpl::packCrsMatrix", @@ -864,8 +858,7 @@ packCrsMatrix (const CrsMatrix& sourceMatrix, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { using local_matrix_device_type = typename CrsMatrix::local_matrix_device_type; using device_type = typename local_matrix_device_type::device_type; @@ -899,7 +892,7 @@ packCrsMatrix (const CrsMatrix& sourceMatrix, constexpr bool pack_pids = false; PackCrsMatrixImpl::packCrsMatrix ( sourceMatrix, exports_dv, num_packets_per_lid_d, export_lids_d, - export_pids_d, constantNumPackets, pack_pids, distor); + export_pids_d, constantNumPackets, pack_pids); // The counts are an output of PackCrsMatrixImpl::packCrsMatrix, so we have to // copy them back to host. @@ -929,9 +922,7 @@ packCrsMatrixNew( Kokkos::DualView::buffer_device_type>& exports, const Kokkos::DualView::buffer_device_type>& numPacketsPerLID, const Kokkos::DualView::buffer_device_type>& exportLIDs, - size_t& constantNumPackets, - Distributor& distor -) + size_t& constantNumPackets) { using device_type = typename CrsMatrix::device_type; using buffer_device_type = typename DistObject::buffer_device_type; @@ -956,7 +947,7 @@ packCrsMatrixNew( ); PackCrsMatrixImpl::packCrsMatrix ( sourceMatrix, exports, numPacketsPerLID_d, exportLIDs_d, - exportPIDs_d, constantNumPackets, pack_pids, distor); + exportPIDs_d, constantNumPackets, pack_pids); } template @@ -966,8 +957,7 @@ packCrsMatrixWithOwningPIDs (const CrsMatrix& sourceMatrix, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& exportLIDs, const Teuchos::ArrayView& sourcePIDs, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { typedef typename CrsMatrix::local_matrix_device_type local_matrix_device_type; typedef typename DistObject::buffer_device_type buffer_device_type; @@ -1027,7 +1017,7 @@ packCrsMatrixWithOwningPIDs (const CrsMatrix& sourceMatrix, try { PackCrsMatrixImpl::packCrsMatrix (sourceMatrix, exports_dv, num_packets_per_lid_d, export_lids_d, - export_pids_d, constantNumPackets, pack_pids, distor); + export_pids_d, constantNumPackets, pack_pids); } catch (std::exception& e) { if (verbose) { @@ -1091,22 +1081,19 @@ packCrsMatrixWithOwningPIDs (const CrsMatrix& sourceMatrix, Teuchos::Array&, \ const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ - size_t&, \ - Distributor&); \ + size_t&); \ template void \ Details::packCrsMatrixNew (const CrsMatrix&, \ Kokkos::DualView::buffer_device_type>&, \ const Kokkos::DualView::buffer_device_type>&, \ const Kokkos::DualView::buffer_device_type>&, \ - size_t&, \ - Distributor&); \ + size_t&); \ template void \ Details::packCrsMatrixWithOwningPIDs (const CrsMatrix&, \ Kokkos::DualView::buffer_device_type>&, \ const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ - size_t&, \ - Distributor&); + size_t&); #endif // TPETRA_DETAILS_PACKCRSMATRIX_DEF_HPP diff --git a/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_decl.hpp b/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_decl.hpp index 40e20c3cd5e3..f5ae653c4256 100644 --- a/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_decl.hpp @@ -78,11 +78,6 @@ template class ArrayView; namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -119,8 +114,6 @@ namespace Details { /// to expect a possibly /// different ("nonconstant") number of packets per local index /// (i.e., a possibly different number of entries per row). /// -/// \param distor [in] The distributor (not used) -/// /// \param combineMode [in] the mode to use for combining /// /// \param numSameIds [in] @@ -148,7 +141,6 @@ unpackAndCombineWithOwningPIDsCount( const Teuchos::ArrayView::packet_type> &imports, const Teuchos::ArrayView& numPacketsPerLID, size_t constantNumPackets, - Distributor &distor, CombineMode combineMode, size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, @@ -176,7 +168,6 @@ unpackAndCombineIntoCrsArrays( const Teuchos::ArrayView::packet_type>& imports, const Teuchos::ArrayView& numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode, const size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, diff --git a/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_def.hpp b/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_def.hpp index 0bbb0167f543..55c5c2cc01b8 100644 --- a/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_def.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_unpackCrsGraphAndCombine_def.hpp @@ -76,11 +76,6 @@ namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -838,8 +833,6 @@ unpackAndCombineIntoCrsArrays( /// to expect a possibly /// different ("nonconstant") number of packets per local index /// (i.e., a possibly different number of entries per row). /// -/// \param distor [in] The distributor (not used) -/// /// \param combineMode [in] the mode to use for combining /// /// \param numSameIds [in] @@ -864,7 +857,6 @@ unpackAndCombineWithOwningPIDsCount( const Teuchos::ArrayView::packet_type> &imports, const Teuchos::ArrayView& numPacketsPerLID, size_t /* constantNumPackets */, - Distributor &/* distor */, CombineMode /* combineMode */, size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, @@ -936,7 +928,6 @@ unpackAndCombineIntoCrsArrays( const Teuchos::ArrayView::packet_type>& imports, const Teuchos::ArrayView& numPacketsPerLID, const size_t /* constantNumPackets */, - Distributor& /* distor */, const CombineMode /* combineMode */, const size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, @@ -1075,7 +1066,6 @@ unpackAndCombineIntoCrsArrays( const Teuchos::ArrayView::packet_type>&, \ const Teuchos::ArrayView&, \ const size_t, \ - Distributor&, \ const CombineMode, \ const size_t, \ const Teuchos::ArrayView&, \ @@ -1094,7 +1084,6 @@ unpackAndCombineIntoCrsArrays( const Teuchos::ArrayView::packet_type> &, \ const Teuchos::ArrayView&, \ size_t, \ - Distributor &, \ CombineMode, \ size_t, \ const Teuchos::ArrayView&, \ diff --git a/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_decl.hpp b/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_decl.hpp index 57d8ed190f7b..349a1fa0ca86 100644 --- a/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_decl.hpp @@ -78,11 +78,6 @@ template class ArrayView; namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -133,7 +128,6 @@ unpackCrsMatrixAndCombine (const CrsMatrix& sourceMatrix, const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& importLIDs, size_t constantNumPackets, - Distributor & distor, CombineMode combineMode); template @@ -147,7 +141,6 @@ unpackCrsMatrixAndCombineNew( const Kokkos::DualView::buffer_device_type>& importLIDs, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode); /// \brief Special version of Tpetra::Details::unpackCrsMatrixAndCombine @@ -213,7 +206,6 @@ unpackAndCombineWithOwningPIDsCount ( const Teuchos::ArrayView &imports, const Teuchos::ArrayView& numPacketsPerLID, size_t constantNumPackets, - Distributor &distor, CombineMode combineMode, size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, @@ -241,7 +233,6 @@ unpackAndCombineIntoCrsArrays ( const Teuchos::ArrayView& imports, const Teuchos::ArrayView& numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode, const size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, diff --git a/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_def.hpp b/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_def.hpp index 7543b785a400..ed259b127de5 100644 --- a/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_def.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_unpackCrsMatrixAndCombine_def.hpp @@ -78,11 +78,6 @@ namespace Tpetra { -#ifndef DOXYGEN_SHOULD_SKIP_THIS -// Forward declaration of Distributor -class Distributor; -#endif // DOXYGEN_SHOULD_SKIP_THIS - // // Users must never rely on anything in the Details namespace. // @@ -1203,7 +1198,6 @@ unpackCrsMatrixAndCombine( const Teuchos::ArrayView& numPacketsPerLID, const Teuchos::ArrayView& importLIDs, size_t /* constantNumPackets */, - Distributor & /* distor */, CombineMode combineMode) { using Kokkos::View; @@ -1259,7 +1253,6 @@ unpackCrsMatrixAndCombineNew( const Kokkos::DualView::buffer_device_type>& importLIDs, const size_t /* constantNumPackets */, - Distributor& /* distor */, const CombineMode combineMode) { using Kokkos::View; @@ -1362,7 +1355,6 @@ unpackAndCombineWithOwningPIDsCount ( const Teuchos::ArrayView &imports, const Teuchos::ArrayView& numPacketsPerLID, size_t /* constantNumPackets */, - Distributor &/* distor */, CombineMode /* combineMode */, size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, @@ -1433,7 +1425,6 @@ unpackAndCombineIntoCrsArrays ( const Teuchos::ArrayView& imports, const Teuchos::ArrayView& numPacketsPerLID, const size_t /* constantNumPackets */, - Distributor& /* distor */, const CombineMode /* combineMode */, const size_t numSameIDs, const Teuchos::ArrayView& permuteToLIDs, @@ -1628,7 +1619,6 @@ unpackAndCombineIntoCrsArrays ( const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ size_t, \ - Distributor&, \ CombineMode); \ template void \ Details::unpackCrsMatrixAndCombineNew ( \ @@ -1637,7 +1627,6 @@ unpackAndCombineIntoCrsArrays ( Kokkos::DualView::buffer_device_type>, \ const Kokkos::DualView::buffer_device_type>&, \ const size_t, \ - Distributor&, \ const CombineMode); \ template void \ Details::unpackAndCombineIntoCrsArrays ( \ @@ -1646,7 +1635,6 @@ unpackAndCombineIntoCrsArrays ( const Teuchos::ArrayView&, \ const Teuchos::ArrayView&, \ const size_t, \ - Distributor&, \ const CombineMode, \ const size_t, \ const Teuchos::ArrayView&, \ @@ -1666,7 +1654,6 @@ unpackAndCombineIntoCrsArrays ( const Teuchos::ArrayView &, \ const Teuchos::ArrayView&, \ size_t, \ - Distributor &, \ CombineMode, \ size_t, \ const Teuchos::ArrayView&, \ diff --git a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp index 175686c822bc..dffaede3fca6 100644 --- a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp @@ -897,9 +897,6 @@ namespace Tpetra { /// \param constantNumPackets [out] On exit, 0 if the number of /// packets per LID could differ, else (if nonzero) the number /// of packets per LID (which must be constant). - /// - /// \param distor [in] The Distributor object we are using. Most - /// implementations will not use this. virtual void packAndPrepare (const SrcDistObject& source, const Kokkos::DualView& exports, Kokkos::DualView numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); /// \brief Perform any unpacking and combining after /// communication. @@ -948,9 +944,6 @@ namespace Tpetra { /// numPacketsPerLID[i] is the number of packets to /// unpack for LID importLIDs[i]. /// - /// \param distor [in] The Distributor object we are using. Most - /// implementations will not use this. - /// /// \param combineMode [in] The CombineMode to use when combining /// the imported entries with existing entries. virtual void @@ -961,7 +954,6 @@ namespace Tpetra { Kokkos::DualView numPacketsPerLID, const size_t constantNumPackets, - Distributor& distor, const CombineMode combineMode); diff --git a/packages/tpetra/core/src/Tpetra_DistObject_def.hpp b/packages/tpetra/core/src/Tpetra_DistObject_def.hpp index 37f568176335..10e3df8f1673 100644 --- a/packages/tpetra/core/src/Tpetra_DistObject_def.hpp +++ b/packages/tpetra/core/src/Tpetra_DistObject_def.hpp @@ -1637,7 +1637,7 @@ namespace Tpetra { try { this->packAndPrepare (src, exportLIDs, this->exports_, this->numExportPacketsPerLID_, - constantNumPackets, distor); + constantNumPackets); lclSuccess = true; } catch (std::exception& e) { @@ -1659,7 +1659,7 @@ namespace Tpetra { else { this->packAndPrepare (src, exportLIDs, this->exports_, this->numExportPacketsPerLID_, - constantNumPackets, distor); + constantNumPackets); } } @@ -1689,7 +1689,7 @@ namespace Tpetra { try { this->unpackAndCombine (remoteLIDs, this->imports_, this->numImportPacketsPerLID_, - constantNumPackets, distor, CM); + constantNumPackets, CM); lclSuccess = true; } catch (std::exception& e) { @@ -1711,7 +1711,7 @@ namespace Tpetra { else { this->unpackAndCombine (remoteLIDs, this->imports_, this->numImportPacketsPerLID_, - constantNumPackets, distor, CM); + constantNumPackets, CM); } } @@ -1744,8 +1744,7 @@ namespace Tpetra { Kokkos::DualView< size_t*, buffer_device_type>, - size_t&, - Distributor&) + size_t&) {} template @@ -1762,7 +1761,6 @@ namespace Tpetra { size_t*, buffer_device_type> /* numPacketsPerLID */, const size_t /* constantNumPackets */, - Distributor& /* distor */, const CombineMode /* combineMode */) {} diff --git a/packages/tpetra/core/src/Tpetra_MultiVector_decl.hpp b/packages/tpetra/core/src/Tpetra_MultiVector_decl.hpp index 4ad40a64880a..1192859b11c5 100644 --- a/packages/tpetra/core/src/Tpetra_MultiVector_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_MultiVector_decl.hpp @@ -2563,8 +2563,7 @@ namespace Tpetra { Kokkos::DualView< size_t*, buffer_device_type> /* numPacketsPerLID */, - size_t& constantNumPackets, - Distributor& /* distor */); + size_t& constantNumPackets); virtual void unpackAndCombine @@ -2578,7 +2577,6 @@ namespace Tpetra { size_t*, buffer_device_type> /* numPacketsPerLID */, const size_t constantNumPackets, - Distributor& /* distor */, const CombineMode CM); private: diff --git a/packages/tpetra/core/src/Tpetra_MultiVector_def.hpp b/packages/tpetra/core/src/Tpetra_MultiVector_def.hpp index bd5a642286cc..aaed39769b78 100644 --- a/packages/tpetra/core/src/Tpetra_MultiVector_def.hpp +++ b/packages/tpetra/core/src/Tpetra_MultiVector_def.hpp @@ -1365,8 +1365,7 @@ namespace Tpetra { const Kokkos::DualView& exportLIDs, Kokkos::DualView& exports, Kokkos::DualView /* numExportPacketsPerLID */, - size_t& constantNumPackets, - Distributor & /* distor */ ) + size_t& constantNumPackets) { using ::Tpetra::Details::Behavior; using ::Tpetra::Details::ProfilingRegion; @@ -1742,7 +1741,6 @@ namespace Tpetra { Kokkos::DualView imports, Kokkos::DualView /* numPacketsPerLID */, const size_t constantNumPackets, - Distributor& /* distor */, const CombineMode CM) { using ::Tpetra::Details::Behavior; diff --git a/packages/tpetra/core/src/Tpetra_Packable.hpp b/packages/tpetra/core/src/Tpetra_Packable.hpp index 05ca5fcfeb63..9b155066d29b 100644 --- a/packages/tpetra/core/src/Tpetra_Packable.hpp +++ b/packages/tpetra/core/src/Tpetra_Packable.hpp @@ -54,9 +54,6 @@ namespace Teuchos { template class ArrayView; } // namespace Teuchos -namespace Tpetra { - class Distributor; // forward declaration -} // namespace Tpetra #endif // DOXYGEN_SHOULD_SKIP_THIS namespace Tpetra { @@ -118,16 +115,11 @@ namespace Tpetra { /// size for each local index). If nonzero, then it is expected /// that the number of packets per local index is constant, and /// that constantNumPackets is that value. - /// - /// \param distor [in] The Distributor object we are using. - /// Implementations may ignore this object. We provide it for - /// consistency with DistObject's packAndPrepare method. virtual void pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor &distor) const = 0; + size_t& constantNumPackets) const = 0; //! Destructor (virtual for memory safety of derived classes). virtual ~Packable () {} diff --git a/packages/tpetra/core/src/Tpetra_RowGraph_decl.hpp b/packages/tpetra/core/src/Tpetra_RowGraph_decl.hpp index a78074bc77e1..7e95073457b6 100644 --- a/packages/tpetra/core/src/Tpetra_RowGraph_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_RowGraph_decl.hpp @@ -322,8 +322,7 @@ namespace Tpetra { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const; + size_t& constantNumPackets) const; //@} }; // class RowGraph } // namespace Tpetra diff --git a/packages/tpetra/core/src/Tpetra_RowGraph_def.hpp b/packages/tpetra/core/src/Tpetra_RowGraph_def.hpp index 13c8b22c3877..8b840128edf9 100644 --- a/packages/tpetra/core/src/Tpetra_RowGraph_def.hpp +++ b/packages/tpetra/core/src/Tpetra_RowGraph_def.hpp @@ -42,8 +42,6 @@ #ifndef TPETRA_ROWGRAPH_DEF_HPP #define TPETRA_ROWGRAPH_DEF_HPP -#include "Tpetra_Distributor.hpp" // avoid error C2027: use of undefined type 'Tpetra::Distributor' at (void) distor below - namespace Tpetra { template void @@ -51,15 +49,13 @@ namespace Tpetra { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const + size_t& constantNumPackets) const { using Teuchos::Array; typedef LocalOrdinal LO; typedef GlobalOrdinal GO; typedef Map map_type; const char tfecfFuncName[] = "packAndPrepare"; - (void) distor; // forestall "unused argument" compiler warning TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC( exportLIDs.size() != numPacketsPerLID.size(), std::runtime_error, diff --git a/packages/tpetra/core/src/Tpetra_RowMatrix_decl.hpp b/packages/tpetra/core/src/Tpetra_RowMatrix_decl.hpp index 382e11b18472..a84e96dc9dcb 100644 --- a/packages/tpetra/core/src/Tpetra_RowMatrix_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_RowMatrix_decl.hpp @@ -565,8 +565,7 @@ namespace Tpetra { packImpl (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const; + size_t& constantNumPackets) const; public: @@ -582,8 +581,7 @@ namespace Tpetra { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& distor) const; + size_t& constantNumPackets) const; //@} }; // class RowMatrix } // namespace Tpetra diff --git a/packages/tpetra/core/src/Tpetra_RowMatrix_def.hpp b/packages/tpetra/core/src/Tpetra_RowMatrix_def.hpp index ee6ee36ef063..ad4f52c71ed1 100644 --- a/packages/tpetra/core/src/Tpetra_RowMatrix_def.hpp +++ b/packages/tpetra/core/src/Tpetra_RowMatrix_def.hpp @@ -302,8 +302,7 @@ namespace Tpetra { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor &distor) const + size_t& constantNumPackets) const { #ifdef HAVE_TPETRA_DEBUG const char tfecfFuncName[] = "pack: "; @@ -313,7 +312,7 @@ namespace Tpetra { int lclBad = 0; try { this->packImpl (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); } catch (std::exception& e) { lclBad = 1; msg << e.what (); @@ -342,7 +341,7 @@ namespace Tpetra { } #else this->packImpl (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); #endif // HAVE_TPETRA_DEBUG } @@ -491,8 +490,7 @@ namespace Tpetra { packImpl (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Distributor& /* distor */) const + size_t& constantNumPackets) const { using Teuchos::Array; using Teuchos::ArrayView; diff --git a/packages/tpetra/core/test/CrsGraph/CrsGraph_PackUnpack.cpp b/packages/tpetra/core/test/CrsGraph/CrsGraph_PackUnpack.cpp index 09f14dc46622..84e9ad9b3803 100644 --- a/packages/tpetra/core/test/CrsGraph/CrsGraph_PackUnpack.cpp +++ b/packages/tpetra/core/test/CrsGraph/CrsGraph_PackUnpack.cpp @@ -45,7 +45,6 @@ #include "TpetraCore_ETIHelperMacros.h" #include "Tpetra_CrsGraph.hpp" #include "Tpetra_Core.hpp" -#include "Tpetra_Distributor.hpp" #include "Tpetra_Map.hpp" #include "Tpetra_Details_gathervPrint.hpp" #include "Tpetra_Details_packCrsGraph.hpp" @@ -162,7 +161,6 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackThenUnpackAndCombine, LO, GO, NT Array exports; // output argument; to be realloc'd Array numPacketsPerLID (num_loc_rows, 0); // output argument size_t constantNumPackets; // output argument - Tpetra::Distributor distor (comm); // argument required, but not used out << "Calling packCrsGraph" << endl; @@ -171,7 +169,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackThenUnpackAndCombine, LO, GO, NT std::ostringstream msg; try { packCrsGraph(*A, exports, numPacketsPerLID(), exportLIDs(), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { local_op_ok = 0; @@ -331,13 +329,12 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackWithError, LO, GO, NT) Array exports; Array numPacketsPerLID(num_loc_rows, 0); size_t constantNumPackets; - Tpetra::Distributor distor(comm); { int local_op_ok; std::ostringstream msg; try { packCrsGraph(*A, exports, numPacketsPerLID(), exportLIDs(), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { local_op_ok = 0; @@ -374,14 +371,13 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackWithError, LO, GO, NT) Array exports; Array numPacketsPerLID(num_loc_rows, 0); size_t constantNumPackets; - Tpetra::Distributor distor(comm); out << "Calling packCrsGraph" << endl; { int local_op_ok; std::ostringstream msg; try { packCrsGraph(*A, exports, numPacketsPerLID(), exportLIDs(), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { local_op_ok = 0; diff --git a/packages/tpetra/core/test/CrsGraph/CrsGraph_UnpackIntoStaticGraph.cpp b/packages/tpetra/core/test/CrsGraph/CrsGraph_UnpackIntoStaticGraph.cpp index 93bce1345f00..c45620c22e2e 100644 --- a/packages/tpetra/core/test/CrsGraph/CrsGraph_UnpackIntoStaticGraph.cpp +++ b/packages/tpetra/core/test/CrsGraph/CrsGraph_UnpackIntoStaticGraph.cpp @@ -45,7 +45,6 @@ #include "TpetraCore_ETIHelperMacros.h" #include "Tpetra_CrsGraph.hpp" #include "Tpetra_Core.hpp" -#include "Tpetra_Distributor.hpp" #include "Tpetra_Map.hpp" #include "Tpetra_Details_gathervPrint.hpp" #include "Tpetra_Details_packCrsGraph.hpp" @@ -134,13 +133,6 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackThenUnpackAndCombine, LO, GO, NT auto num_packets_per_lid = Array(num_loc_rows, 0); // output argument size_t const_num_packets; // output argument - // We're not actually communicating in this test; we just need the Distributor - // for the interface of packCrsGraph (which doesn't use it). Consider changing - // packCrsGraph's interface so it doesn't take a Distributor? No, because - // Distributor has index permutation information that we could use to pack in - // a particular order and thus avoid the slow path in Distributor::doPosts. - auto distor = Tpetra::Distributor(comm); - out << "Calling packCrsGraph" << endl; { @@ -148,7 +140,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackThenUnpackAndCombine, LO, GO, NT std::ostringstream msg; try { packCrsGraph(*B, exports, num_packets_per_lid(), export_lids(), - const_num_packets, distor); + const_num_packets); local_op_ok = 1; } catch (std::exception& e) { local_op_ok = 0; @@ -178,7 +170,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(CrsGraph, PackThenUnpackAndCombine, LO, GO, NT int local_op_ok; std::ostringstream msg; unpackCrsGraphAndCombine(*A, exports, num_packets_per_lid(), - export_lids(), const_num_packets, distor, Tpetra::REPLACE); + export_lids(), const_num_packets, Tpetra::REPLACE); local_op_ok = 1; TEST_ASSERT(local_op_ok == 1); diff --git a/packages/tpetra/core/test/CrsMatrix/CrsMatrix_PackUnpack.cpp b/packages/tpetra/core/test/CrsMatrix/CrsMatrix_PackUnpack.cpp index 5331c90da1f8..a3f81430adc0 100644 --- a/packages/tpetra/core/test/CrsMatrix/CrsMatrix_PackUnpack.cpp +++ b/packages/tpetra/core/test/CrsMatrix/CrsMatrix_PackUnpack.cpp @@ -43,7 +43,6 @@ #include "TpetraCore_ETIHelperMacros.h" #include "Tpetra_CrsMatrix.hpp" #include "Tpetra_Core.hpp" -#include "Tpetra_Distributor.hpp" #include "Tpetra_Map.hpp" #include "Tpetra_Details_gathervPrint.hpp" #include "Tpetra_Details_packCrsMatrix.hpp" @@ -170,7 +169,6 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackThenUnpackAndCombine, SC, LO, G Array exports; // output argument; to be realloc'd Array numPacketsPerLID (num_loc_rows, 0); // output argument size_t constantNumPackets; // output argument - Tpetra::Distributor distor (comm); // argument required, but not used out << "Calling packCrsMatrix" << endl; @@ -180,7 +178,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackThenUnpackAndCombine, SC, LO, G try { packCrsMatrix (*A, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { @@ -220,7 +218,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackThenUnpackAndCombine, SC, LO, G try { unpackCrsMatrixAndCombine (*B, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor, + constantNumPackets, Tpetra::REPLACE); local_op_ok = 1; } @@ -319,7 +317,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackThenUnpackAndCombine, SC, LO, G try { unpackCrsMatrixAndCombine (*B, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor, + constantNumPackets, Tpetra::ADD); local_op_ok = 1; } @@ -438,14 +436,13 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackWithError, SC, LO, GO, NT) Array exports; Array numPacketsPerLID(num_loc_rows, 0); size_t constantNumPackets; - Tpetra::Distributor distor(comm); { int local_op_ok; std::ostringstream msg; try { packCrsMatrix (*A, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { @@ -483,7 +480,6 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackWithError, SC, LO, GO, NT) Array exports; Array numPacketsPerLID(num_loc_rows, 0); size_t constantNumPackets; - Tpetra::Distributor distor(comm); out << "Calling packCrsMatrix" << endl; { int local_op_ok; @@ -491,7 +487,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackWithError, SC, LO, GO, NT) try { packCrsMatrix (*A, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { @@ -546,7 +542,6 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackPartial, SC, LO, GO, NT) Array exports; // output argument; to be realloc'd Array numPacketsPerLID (num_loc_rows, 0); // output argument size_t constantNumPackets; // output argument - Tpetra::Distributor distor (comm); // argument required, but not used out << "Calling packCrsMatrix" << endl; @@ -556,7 +551,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackPartial, SC, LO, GO, NT) try { packCrsMatrix (*A, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor); + constantNumPackets); local_op_ok = 1; } catch (std::exception& e) { @@ -596,7 +591,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CrsMatrix, PackPartial, SC, LO, GO, NT) try { unpackCrsMatrixAndCombine (*B, exports, numPacketsPerLID (), exportLIDs ().getConst (), - constantNumPackets, distor, + constantNumPackets, Tpetra::REPLACE); local_op_ok = 1; } diff --git a/packages/tpetra/core/test/CrsMatrix/CrsMatrix_createDeepCopy.cpp b/packages/tpetra/core/test/CrsMatrix/CrsMatrix_createDeepCopy.cpp index 349977191014..da5894ede2ea 100644 --- a/packages/tpetra/core/test/CrsMatrix/CrsMatrix_createDeepCopy.cpp +++ b/packages/tpetra/core/test/CrsMatrix/CrsMatrix_createDeepCopy.cpp @@ -264,11 +264,10 @@ class MyRowGraph : public Tpetra::RowGraph { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Tpetra::Distributor& distor) const override + size_t& constantNumPackets) const override { return G_->pack (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); } private: @@ -502,11 +501,10 @@ class MyRowMatrix : public Tpetra::RowMatrix { pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Tpetra::Distributor& distor) const override + size_t& constantNumPackets) const override { return A_->pack (exportLIDs, exports, numPacketsPerLID, - constantNumPackets, distor); + constantNumPackets); } void diff --git a/packages/tpetra/core/test/ImportExport2/ImportExport2_UnitTests.cpp b/packages/tpetra/core/test/ImportExport2/ImportExport2_UnitTests.cpp index d30147695045..95ec17d71174 100644 --- a/packages/tpetra/core/test/ImportExport2/ImportExport2_UnitTests.cpp +++ b/packages/tpetra/core/test/ImportExport2/ImportExport2_UnitTests.cpp @@ -2303,7 +2303,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL( Import_Util, UnpackAndCombineWithOwningPIDs, } Tpetra::Details::packCrsMatrixWithOwningPIDs( *A, exports, numExportPackets(), Importer->getExportLIDs(), - SourcePids(), constantNumPackets, distor); + SourcePids(), constantNumPackets); if (verbose) { std::ostringstream os; os << *prefix << "Done with packCrsMatrixWithOwningPIDs" << std::endl; @@ -2370,7 +2370,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL( Import_Util, UnpackAndCombineWithOwningPIDs, size_t nnz2 = unpackAndCombineWithOwningPIDsCount (*A, Importer->getRemoteLIDs (), imports (), numImportPackets (), - constantNumPackets, distor, + constantNumPackets, Tpetra::INSERT, Importer->getNumSameIDs (), Importer->getPermuteToLIDs (), @@ -2406,7 +2406,6 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL( Import_Util, UnpackAndCombineWithOwningPIDs, imports (), numImportPackets (), constantNumPackets, - distor, Tpetra::INSERT, Importer->getNumSameIDs (), Importer->getPermuteToLIDs (), From dd824dd676829c463c1a0d7b5662fabb60bcacfc Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Mon, 16 Aug 2021 09:35:00 -0600 Subject: [PATCH 04/70] Ifpack2: update usage of the Tpetra matrix pack API --- packages/ifpack2/test/unit_tests/Ifpack2_UnitTestHelpers.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestHelpers.hpp b/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestHelpers.hpp index 14aa05e3926a..b11516eaf06c 100644 --- a/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestHelpers.hpp +++ b/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestHelpers.hpp @@ -1070,9 +1070,8 @@ Teuchos::RCP > c pack (const Teuchos::ArrayView& exportLIDs, Teuchos::Array& exports, const Teuchos::ArrayView& numPacketsPerLID, - size_t& constantNumPackets, - Tpetra::Distributor& distor) const - {A_->pack(exportLIDs,exports,numPacketsPerLID,constantNumPackets,distor);} + size_t& constantNumPackets) const + {A_->pack(exportLIDs,exports,numPacketsPerLID,constantNumPackets);} private: Teuchos::RCP > A_; From 6149ac5824035c4ff255e606c395f4e83cbbe12e Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Sat, 14 Aug 2021 14:21:52 -0600 Subject: [PATCH 05/70] Add a temporary method to Distributor to get the Plan for things that need it --- packages/tpetra/core/src/Tpetra_Distributor.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/tpetra/core/src/Tpetra_Distributor.hpp b/packages/tpetra/core/src/Tpetra_Distributor.hpp index 49a21b7fa15e..ecf2ed9c22e9 100644 --- a/packages/tpetra/core/src/Tpetra_Distributor.hpp +++ b/packages/tpetra/core/src/Tpetra_Distributor.hpp @@ -751,6 +751,11 @@ namespace Tpetra { Teuchos::Describable::verbLevel_default) const; //@} + /// \brief Get this Distributor's DistributorPlan + /// + /// FIXME: Delete this method when it's no longer needed for non-blocking + /// communication in the DistObject + const Details::DistributorPlan& getPlan() const { return plan_; } private: Details::DistributorPlan plan_; Details::DistributorActor actor_; From e630cab733b0f5af3e2729c6d93f5c20ec486247 Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Mon, 16 Aug 2021 13:41:16 -0600 Subject: [PATCH 06/70] Tpetra: Add DistributorActor to DistObject --- packages/tpetra/core/src/Tpetra_DistObject_decl.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp index dffaede3fca6..77fef1821d2b 100644 --- a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp @@ -43,6 +43,7 @@ /// \file Tpetra_DistObject_decl.hpp /// \brief Declaration of the Tpetra::DistObject class +#include "Tpetra_Details_DistributorActor.hpp" #include "Tpetra_Map.hpp" #include "Tpetra_Import.hpp" #include "Tpetra_Export.hpp" @@ -1038,6 +1039,8 @@ namespace Tpetra { private: using this_type = DistObject; + Details::DistributorActor distributorActor_; + #ifdef HAVE_TPETRA_TRANSFER_TIMERS Teuchos::RCP doXferTimer_; Teuchos::RCP copyAndPermuteTimer_; From c29a8a285eda2095a5bb88b59b4b453bf9afba3b Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Mon, 16 Aug 2021 15:56:12 -0600 Subject: [PATCH 07/70] Tpetra: delete distributor arg from DistObject pack/unpack helpers --- packages/tpetra/core/src/Tpetra_DistObject_decl.hpp | 4 +--- packages/tpetra/core/src/Tpetra_DistObject_def.hpp | 8 +++----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp index 77fef1821d2b..05a1a92b89c5 100644 --- a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp @@ -797,12 +797,10 @@ namespace Tpetra { void doPackAndPrepare(const SrcDistObject& src, const Kokkos::DualView& exportLIDs, - size_t& constantNumPackets, - Distributor& distor); + size_t& constantNumPackets); void doUnpackAndCombine(const Kokkos::DualView& remoteLIDs, size_t constantNumPackets, - Distributor& distor, CombineMode CM); /// \name Methods implemented by subclasses and used by doTransfer(). diff --git a/packages/tpetra/core/src/Tpetra_DistObject_def.hpp b/packages/tpetra/core/src/Tpetra_DistObject_def.hpp index 10e3df8f1673..0c745232216d 100644 --- a/packages/tpetra/core/src/Tpetra_DistObject_def.hpp +++ b/packages/tpetra/core/src/Tpetra_DistObject_def.hpp @@ -1049,7 +1049,7 @@ namespace Tpetra { std::cerr << os.str (); } - doPackAndPrepare(src, exportLIDs, constantNumPackets, distor); + doPackAndPrepare(src, exportLIDs, constantNumPackets); if (commOnHost) { this->exports_.sync_host(); } @@ -1326,7 +1326,7 @@ namespace Tpetra { os << *prefix << "8. unpackAndCombine" << endl; std::cerr << os.str (); } - doUnpackAndCombine(remoteLIDs, constantNumPackets, distor, CM); + doUnpackAndCombine(remoteLIDs, constantNumPackets, CM); } // if (needCommunication) } // if (CM != ZERO) @@ -1600,8 +1600,7 @@ namespace Tpetra { DistObject:: doPackAndPrepare(const SrcDistObject& src, const Kokkos::DualView& exportLIDs, - size_t& constantNumPackets, - Distributor& distor) + size_t& constantNumPackets) { using Details::ProfilingRegion; using std::endl; @@ -1668,7 +1667,6 @@ namespace Tpetra { DistObject:: doUnpackAndCombine(const Kokkos::DualView& remoteLIDs, size_t constantNumPackets, - Distributor& distor, CombineMode CM) { using Details::ProfilingRegion; From 13b7ede55a3b9dc6798b674b76a2f913f662402f Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Mon, 16 Aug 2021 16:34:59 -0600 Subject: [PATCH 08/70] Tpetra: DistObject's doPosts/Waits take a Plan and use the local actor --- .../core/src/Tpetra_DistObject_decl.hpp | 4 +- .../tpetra/core/src/Tpetra_DistObject_def.hpp | 62 +++++++++++-------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp index 05a1a92b89c5..38cb37a3098e 100644 --- a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp @@ -784,7 +784,7 @@ namespace Tpetra { const CombineMode CM, const bool restrictedMode); - void doPosts(Distributor& distor, + void doPosts(const Details::DistributorPlan& distributorPlan, size_t constantNumPackets, bool commOnHost, ReverseOption revOp, @@ -792,7 +792,7 @@ namespace Tpetra { const bool canTryAliasing, const CombineMode CM); - void doWaits(Distributor& distor, + void doWaits(const Details::DistributorPlan& distributorPlan, ReverseOption revOp); void doPackAndPrepare(const SrcDistObject& src, diff --git a/packages/tpetra/core/src/Tpetra_DistObject_def.hpp b/packages/tpetra/core/src/Tpetra_DistObject_def.hpp index 0c745232216d..c3b4550eb751 100644 --- a/packages/tpetra/core/src/Tpetra_DistObject_def.hpp +++ b/packages/tpetra/core/src/Tpetra_DistObject_def.hpp @@ -887,6 +887,7 @@ namespace Tpetra { const size_t numSameIDs = transfer.getNumSameIDs (); Distributor& distor = transfer.getDistributor (); + const Details::DistributorPlan& distributorPlan = distor.getPlan(); TEUCHOS_TEST_FOR_EXCEPTION (debug && restrictedMode && @@ -1122,7 +1123,7 @@ namespace Tpetra { std::cerr << os.str (); } - doPosts(distor, constantNumPackets, commOnHost, revOp, prefix, canTryAliasing, CM); + doPosts(distributorPlan, constantNumPackets, commOnHost, revOp, prefix, canTryAliasing, CM); } // if (needCommunication) } // if (CM != ZERO) } @@ -1241,6 +1242,7 @@ namespace Tpetra { } Distributor& distor = transfer.getDistributor (); + const Details::DistributorPlan& distributorPlan = distor.getPlan(); TEUCHOS_TEST_FOR_EXCEPTION (debug && restrictedMode && @@ -1319,7 +1321,7 @@ namespace Tpetra { } } else { - doWaits(distor, revOp); + doWaits(distributorPlan, revOp); if (verbose) { std::ostringstream os; @@ -1346,7 +1348,7 @@ namespace Tpetra { template void DistObject:: - doPosts(Distributor& distor, + doPosts(const Details::DistributorPlan& distributorPlan, size_t constantNumPackets, bool commOnHost, ReverseOption revOp, @@ -1391,10 +1393,10 @@ namespace Tpetra { std::cerr << os.str (); } if (revOp == DoReverse) { - distor.doReversePostsAndWaits (numExp_h, 1, numImp_h); + distributorActor_.doPostsAndWaits(*distributorPlan.getReversePlan(), numExp_h, 1, numImp_h); } else { - distor.doPostsAndWaits (numExp_h, 1, numImp_h); + distributorActor_.doPostsAndWaits(distributorPlan, numExp_h, 1, numImp_h); } if (verbose) { @@ -1422,10 +1424,10 @@ namespace Tpetra { std::cerr << os.str (); } if (revOp == DoReverse) { - distor.doReversePostsAndWaits (numExp_d, 1, numImp_d); + distributorActor_.doPostsAndWaits(*distributorPlan.getReversePlan(), numExp_d, 1, numImp_d); } else { - distor.doPostsAndWaits (numExp_d, 1, numImp_d); + distributorActor_.doPostsAndWaits(distributorPlan, numExp_d, 1, numImp_d); } if (verbose) { @@ -1487,15 +1489,17 @@ namespace Tpetra { if (commOnHost) { this->imports_.modify_host (); if (revOp == DoReverse) { - distor.doReversePosts - (create_const_view (this->exports_.view_host ()), + distributorActor_.doPosts + (*distributorPlan.getReversePlan(), + create_const_view (this->exports_.view_host ()), numExportPacketsPerLID_av, this->imports_.view_host (), numImportPacketsPerLID_av); } else { - distor.doPosts - (create_const_view (this->exports_.view_host ()), + distributorActor_.doPosts + (distributorPlan, + create_const_view (this->exports_.view_host ()), numExportPacketsPerLID_av, this->imports_.view_host (), numImportPacketsPerLID_av); @@ -1505,15 +1509,17 @@ namespace Tpetra { Kokkos::fence(); // for UVM this->imports_.modify_device (); if (revOp == DoReverse) { - distor.doReversePosts - (create_const_view (this->exports_.view_device ()), + distributorActor_.doPosts + (*distributorPlan.getReversePlan(), + create_const_view (this->exports_.view_device ()), numExportPacketsPerLID_av, this->imports_.view_device (), numImportPacketsPerLID_av); } else { - distor.doPosts - (create_const_view (this->exports_.view_device ()), + distributorActor_.doPosts + (distributorPlan, + create_const_view (this->exports_.view_device ()), numExportPacketsPerLID_av, this->imports_.view_device (), numImportPacketsPerLID_av); @@ -1550,14 +1556,16 @@ namespace Tpetra { if (commOnHost) { this->imports_.modify_host (); if (revOp == DoReverse) { - distor.doReversePosts - (create_const_view (this->exports_.view_host ()), + distributorActor_.doPosts + (*distributorPlan.getReversePlan(), + create_const_view (this->exports_.view_host ()), constantNumPackets, this->imports_.view_host ()); } else { - distor.doPosts - (create_const_view (this->exports_.view_host ()), + distributorActor_.doPosts + (distributorPlan, + create_const_view (this->exports_.view_host ()), constantNumPackets, this->imports_.view_host ()); } @@ -1566,14 +1574,16 @@ namespace Tpetra { Kokkos::fence(); // for UVM this->imports_.modify_device (); if (revOp == DoReverse) { - distor.doReversePosts - (create_const_view (this->exports_.view_device ()), + distributorActor_.doPosts + (*distributorPlan.getReversePlan(), + create_const_view (this->exports_.view_device ()), constantNumPackets, this->imports_.view_device ()); } else { - distor.doPosts - (create_const_view (this->exports_.view_device ()), + distributorActor_.doPosts + (distributorPlan, + create_const_view (this->exports_.view_device ()), constantNumPackets, this->imports_.view_device ()); } @@ -1584,14 +1594,14 @@ namespace Tpetra { template void DistObject:: - doWaits(Distributor& distor, + doWaits(const Details::DistributorPlan& distributorPlan, ReverseOption revOp) { if (revOp == DoReverse) { - distor.doReverseWaits(); + distributorActor_.doWaits(*distributorPlan.getReversePlan()); } else { - distor.doWaits(); + distributorActor_.doWaits(distributorPlan); } } From d302bafe54f0d85bb14cb1023520145d19257065 Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Fri, 20 Aug 2021 08:07:54 -0600 Subject: [PATCH 09/70] TrilinosCouplings: Hook up Stratimikos-Ifpack2 in Tpetra Maxwell example --- packages/trilinoscouplings/cmake/Dependencies.cmake | 2 +- .../cmake/TrilinosCouplings_config.h.in | 3 +++ .../examples/scaling/example_Maxwell_Tpetra.cpp | 9 +++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/trilinoscouplings/cmake/Dependencies.cmake b/packages/trilinoscouplings/cmake/Dependencies.cmake index d09284c3e805..5918fd8d77e9 100644 --- a/packages/trilinoscouplings/cmake/Dependencies.cmake +++ b/packages/trilinoscouplings/cmake/Dependencies.cmake @@ -1,7 +1,7 @@ SET(LIB_REQUIRED_DEP_PACKAGES) SET(LIB_OPTIONAL_DEP_PACKAGES Amesos AztecOO Belos EpetraExt Ifpack Isorropia ML MueLu NOX Stokhos Zoltan) SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES Amesos AvatarT AztecOO Epetra EpetraExt Ifpack Intrepid Intrepid2 Isorropia KokkosContainers KokkosCore KokkosKernels ML MueLu MueLu Pamgen SEACASExodus SEACASNemesis Sacado STKIO STKMesh Stokhos Stratimikos Teko TeuchosKokkosComm TeuchosKokkosCompat Tpetra TriKota Zoltan) +SET(TEST_OPTIONAL_DEP_PACKAGES Amesos AvatarT AztecOO Epetra EpetraExt Ifpack Ifpack2 Intrepid Intrepid2 Isorropia KokkosContainers KokkosCore KokkosKernels ML MueLu MueLu Pamgen SEACASExodus SEACASNemesis Sacado STKIO STKMesh Stokhos Stratimikos Teko TeuchosKokkosComm TeuchosKokkosCompat Tpetra TriKota Zoltan) SET(LIB_REQUIRED_DEP_TPLS) SET(LIB_OPTIONAL_DEP_TPLS) SET(TEST_REQUIRED_DEP_TPLS) diff --git a/packages/trilinoscouplings/cmake/TrilinosCouplings_config.h.in b/packages/trilinoscouplings/cmake/TrilinosCouplings_config.h.in index 4e952b3ca9ed..f03d64b5f582 100644 --- a/packages/trilinoscouplings/cmake/TrilinosCouplings_config.h.in +++ b/packages/trilinoscouplings/cmake/TrilinosCouplings_config.h.in @@ -63,6 +63,9 @@ /* Define if TASMANIAN TPL is enabled in TrilinosCouplings */ #cmakedefine HAVE_TRILINOSCOUPLINGS_TASMANIAN +/* Define if want to build Ifpack2 */ +#cmakedefine HAVE_TRILINOSCOUPLINGS_IFPACK2 + /* Define if want to build intrepid */ #cmakedefine HAVE_TRILINOSCOUPLINGS_INTREPID diff --git a/packages/trilinoscouplings/examples/scaling/example_Maxwell_Tpetra.cpp b/packages/trilinoscouplings/examples/scaling/example_Maxwell_Tpetra.cpp index 5233e5bdf30a..d21699d6bcda 100644 --- a/packages/trilinoscouplings/examples/scaling/example_Maxwell_Tpetra.cpp +++ b/packages/trilinoscouplings/examples/scaling/example_Maxwell_Tpetra.cpp @@ -160,6 +160,9 @@ #ifdef HAVE_TRILINOSCOUPLINGS_STRATIMIKOS #include "Stratimikos_DefaultLinearSolverBuilder.hpp" #include +#ifdef HAVE_TRILINOSCOUPLINGS_IFPACK2 +#include +#endif #endif //#include "TrilinosCouplings_IntrepidPoissonExampleHelpers.hpp" @@ -2651,6 +2654,12 @@ void TestPreconditioner_Stratimikos(char ProblemType[], /* Stratimikos setup */ Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder; Stratimikos::enableMueLuRefMaxwell(linearSolverBuilder); // Register MueLu as a Stratimikos preconditioner strategy. +#ifdef HAVE_TRILINOSCOUPLINGS_IFPACK2 + // Register Ifpack2 as a Stratimikos preconditioner strategy. + typedef Thyra::PreconditionerFactoryBase Base; + typedef Thyra::Ifpack2PreconditionerFactory > Impl; + linearSolverBuilder.setPreconditioningStrategyFactory(Teuchos::abstractFactoryStd(), "Ifpack2"); +#endif linearSolverBuilder.setParameterList(rcp(&SList,false)); RCP > lowsFactory = createLinearSolveStrategy(linearSolverBuilder); From 5c2c51e6162f90e60a89d0460605f57bb7bde34a Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Fri, 20 Aug 2021 12:08:11 -0600 Subject: [PATCH 10/70] MueLu: Fixes for distributed classical --- .../MueLu_CoalesceDropFactory_def.hpp | 23 +++++++++++------- .../MueLu_ClassicalMapFactory_def.hpp | 1 + .../Classical/MueLu_ClassicalPFactory_def.hpp | 24 ++++++++++--------- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_def.hpp b/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_def.hpp index 6847f6afedc4..2422498d5442 100644 --- a/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_def.hpp +++ b/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_def.hpp @@ -439,14 +439,14 @@ namespace MueLu { ArrayRCP negMaxOffDiagonal; if(useSignedClassical) { if(ghostedBlockNumber.is_null()) { - if (GetVerbLevel() & Statistics1) - GetOStream(Statistics1) << "Calculating max point off-diagonal"<::GetMatrixMaxMinusOffDiagonal(*A); + if (GetVerbLevel() & Statistics1) + GetOStream(Statistics1) << "Calculated max point off-diagonal" << std::endl; } else { - if (GetVerbLevel() & Statistics1) - GetOStream(Statistics1) << "Calculating max block off-diagonal"<::GetMatrixMaxMinusOffDiagonal(*A,*ghostedBlockNumber); + if (GetVerbLevel() & Statistics1) + GetOStream(Statistics1) << "Calculating max block off-diagonal" << std::endl; } } else { @@ -455,10 +455,15 @@ namespace MueLu { } ArrayRCP boundaryNodes = Teuchos::arcp_const_cast(MueLu::Utilities::DetectDirichletRows(*A, dirichletThreshold)); if (rowSumTol > 0.) { - if(ghostedBlockNumber.is_null()) + if(ghostedBlockNumber.is_null()) { + if (GetVerbLevel() & Statistics1) + GetOStream(Statistics1) << "Applying point row sum criterion." << std::endl; Utilities::ApplyRowSumCriterion(*A, rowSumTol, boundaryNodes); - else - Utilities::ApplyRowSumCriterion(*A, *ghostedBlockNumber, rowSumTol, boundaryNodes); + } else { + if (GetVerbLevel() & Statistics1) + GetOStream(Statistics1) << "Applying block row sum criterion." << std::endl; + Utilities::ApplyRowSumCriterion(*A, *ghostedBlockNumber, rowSumTol, boundaryNodes); + } } LO realnnz = 0; @@ -1835,8 +1840,8 @@ typedef Teuchos::ScalarTraits STS; for (LO colID = 0; colID < Teuchos::as(indices.size()); colID++) { LO col = indices[colID]; LO col_block = col_block_number[col]; - - if(row_block == col_block) { + + if((row_block == col_block) && (col < Teuchos::as(inputGraph->GetDomainMap()->getNodeNumElements()))) { columns[realnnz++] = col; rownnz++; } else diff --git a/packages/muelu/src/Transfers/Classical/MueLu_ClassicalMapFactory_def.hpp b/packages/muelu/src/Transfers/Classical/MueLu_ClassicalMapFactory_def.hpp index 25825e8b8114..6814f9656079 100644 --- a/packages/muelu/src/Transfers/Classical/MueLu_ClassicalMapFactory_def.hpp +++ b/packages/muelu/src/Transfers/Classical/MueLu_ClassicalMapFactory_def.hpp @@ -187,6 +187,7 @@ namespace MueLu { RCP mv; + GetOStream(Statistics1) << "Reading FC splitting from " << color_file << ", using map file " << map_file << ". On rank " << A->getRowMap()->getComm()->getRank() << " local size is " << A->getRowMap()->getNodeNumElements() << std::endl; if(mapfile) { fclose(mapfile); RCP colorMap = Xpetra::IO::ReadMap(map_file, A->getRowMap()->lib(), A->getRowMap()->getComm()); diff --git a/packages/muelu/src/Transfers/Classical/MueLu_ClassicalPFactory_def.hpp b/packages/muelu/src/Transfers/Classical/MueLu_ClassicalPFactory_def.hpp index 0b4f15d77609..752875add280 100644 --- a/packages/muelu/src/Transfers/Classical/MueLu_ClassicalPFactory_def.hpp +++ b/packages/muelu/src/Transfers/Classical/MueLu_ClassicalPFactory_def.hpp @@ -224,14 +224,15 @@ namespace MueLu { RCP Acrs = rcp_dynamic_cast(A)->getCrsMatrix(); RCP Aghost_crs = CrsMatrixFactory::Build(Acrs,*remoteOnlyImporter,A->getDomainMap(),remoteOnlyImporter->getTargetMap()); Aghost = rcp(new CrsMatrixWrap(Aghost_crs)); - // We also may need need to ghost myPointType for Aghos - RCP Importer2 = Aghost->getCrsGraph()->getImporter(); - if(Importer2.is_null()) { - RCP fc_splitting_ghost_nonconst = LocalOrdinalVectorFactory::Build(Aghost->getColMap()); - fc_splitting_ghost_nonconst->doImport(*owned_fc_splitting,*Importer,Xpetra::INSERT); - fc_splitting_ghost = fc_splitting_ghost_nonconst; - myPointType_ghost = fc_splitting_ghost->getData(0); - } + // CAG: It seems that this isn't actually needed? + // We also may need need to ghost myPointType for Aghost + // RCP Importer2 = Aghost->getCrsGraph()->getImporter(); + // if(Importer2.is_null()) { + // RCP fc_splitting_ghost_nonconst = LocalOrdinalVectorFactory::Build(Aghost->getColMap()); + // fc_splitting_ghost_nonconst->doImport(*owned_fc_splitting,*Importer,Xpetra::INSERT); + // fc_splitting_ghost = fc_splitting_ghost_nonconst; + // myPointType_ghost = fc_splitting_ghost->getData(0); + // } } /* Generate the ghosted Coarse map using the "Tuminaro maneuver" (if needed)*/ @@ -351,7 +352,8 @@ namespace MueLu { // NOTE: ParameterList validator will check this guy so we don't really need an "else" here #ifdef CMS_DEBUG - Xpetra::IO::Write("classical_p.mat", *P); + Xpetra::IO::Write("classical_p.mat."+std::to_string(coarseLevel.GetLevelID()), *P); + // Xpetra::IO::WriteLocal("classical_p.mat."+std::to_string(coarseLevel.GetLevelID()), *P); #endif // Save output @@ -1076,9 +1078,9 @@ GenerateStrengthFlags(const Matrix & A,const GraphBase & graph, Teuchos::Array A_indices[j+1]) { is_ok=false; break;} + if (A_indices[j] >= A_indices[j+1]) { is_ok=false; break;} for(LO j=0; j G_indices[j+1]) { is_ok=false; break;} + if (G_indices[j] >= G_indices[j+1]) { is_ok=false; break;} TEUCHOS_TEST_FOR_EXCEPTION(!is_ok, Exceptions::RuntimeError,"ClassicalPFactory: Exected A and Graph to be sorted"); // Now cycle through and set the flags - if the edge is in G it is strong From 272467603e4d46d27743aa39ca2117e5d8633b43 Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Fri, 20 Aug 2021 12:10:19 -0600 Subject: [PATCH 11/70] TrilinosCouplings: Add stacked timer to Maxwell Tpetra example --- .../scaling/example_Maxwell_Tpetra.cpp | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/trilinoscouplings/examples/scaling/example_Maxwell_Tpetra.cpp b/packages/trilinoscouplings/examples/scaling/example_Maxwell_Tpetra.cpp index d21699d6bcda..f8e9f4821620 100644 --- a/packages/trilinoscouplings/examples/scaling/example_Maxwell_Tpetra.cpp +++ b/packages/trilinoscouplings/examples/scaling/example_Maxwell_Tpetra.cpp @@ -126,6 +126,7 @@ #include "Teuchos_XMLParameterListHelpers.hpp" #include "Teuchos_Comm.hpp" #include "Teuchos_OrdinalTraits.hpp" +#include "Teuchos_StackedTimer.hpp" // Shards includes #include "Shards_CellTopology.hpp" @@ -407,6 +408,7 @@ int body(int argc, char *argv[]) { bool verbose, debug, jiggle, dump; std::string solverName; double scaling = 1.0; + bool use_stacked_timer; // Set default values of command-line arguments. nx = 10; @@ -419,6 +421,7 @@ int body(int argc, char *argv[]) { debug = false; jiggle = false; dump = false; + use_stacked_timer = false; // Parse and validate command-line arguments. Teuchos::CommandLineProcessor cmdp (false, true); cmdp.setOption ("nx", &nx, "Number of cells along the x dimension"); @@ -443,6 +446,8 @@ int body(int argc, char *argv[]) { "Whether to randomly perturb the mesh."); cmdp.setOption ("dump", "nodump", &dump, "Whether to dump data."); + cmdp.setOption ("stacked-timer", "no-stacked-timer", &use_stacked_timer, + "Run with or without stacked timer output"); if (MyPID == 0) { std::cout \ @@ -487,6 +492,14 @@ int body(int argc, char *argv[]) { return EXIT_SUCCESS; } + Teuchos::RCP stacked_timer; + if (use_stacked_timer) { + stacked_timer = rcp(new Teuchos::StackedTimer("TC Maxwell Tpetra")); + Teuchos::RCP out = Teuchos::rcp(new Teuchos::FancyOStream(Teuchos::rcpFromRef(std::cout))); + stacked_timer->setVerboseOstream(out); + Teuchos::TimeMonitor::setStackedTimer(stacked_timer); + } + long long * node_comm_proc_ids = NULL; long long * node_cmap_node_cnts = NULL; long long * node_cmap_ids = NULL; @@ -2458,7 +2471,14 @@ int body(int argc, char *argv[]) { } // Summarize timings - Teuchos::TimeMonitor::report (comm.ptr(), std::cout); + if (use_stacked_timer) { + Teuchos::RCP out = Teuchos::rcp(new Teuchos::FancyOStream(Teuchos::rcpFromRef(std::cout))); + stacked_timer->stop("TC Maxwell Tpetra"); + Teuchos::StackedTimer::OutputOptions options; + options.output_fraction = options.output_histogram = options.output_minmax = true; + stacked_timer->report(*out, comm, options); + } else + Teuchos::TimeMonitor::report (comm.ptr(), std::cout); return 0; } From a489ff0dcf4becb486c627b248394ef478bc35d0 Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Fri, 20 Aug 2021 12:11:09 -0600 Subject: [PATCH 12/70] Xpetra IO: Fix warning --- packages/xpetra/sup/Utils/Xpetra_IO.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/xpetra/sup/Utils/Xpetra_IO.hpp b/packages/xpetra/sup/Utils/Xpetra_IO.hpp index f34bce044e8d..a8485fb7a02b 100644 --- a/packages/xpetra/sup/Utils/Xpetra_IO.hpp +++ b/packages/xpetra/sup/Utils/Xpetra_IO.hpp @@ -993,7 +993,7 @@ namespace Xpetra { rowptr2_RCP.resize(rowptr.size()); ArrayView rowptr2 = rowptr2_RCP(); - for (size_t j = 0; j(rowptr.size()); j++) rowptr2[j] = rowptr[j]; Teuchos::MatrixMarket::Raw::Writer writer; From 79fc92af1114a84183118319ffd5390b2cfb93cc Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Fri, 20 Aug 2021 12:20:16 -0600 Subject: [PATCH 13/70] MueLu: Enable rebalancing of "BlockNumber" --- .../MueLu_ParameterListInterpreter_def.hpp | 49 ++++++++++--------- .../MueLu_RebalanceTransferFactory_def.hpp | 38 ++++++++++++++ 2 files changed, 63 insertions(+), 24 deletions(-) diff --git a/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp b/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp index 473872b06b33..5f13dcc95f29 100644 --- a/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp +++ b/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp @@ -671,10 +671,7 @@ namespace MueLu { UpdateFactoryManager_RAP(paramList, defaultList, manager, levelID, keeps); // == BlockNumber Transfer == - // NOTE: You would think this would be levelID > 0, but you'd be wrong, since the FactoryManager is basically - // offset by a level from the things which actually do the work. - if(useBlockNumber_ && levelID > 0) - UpdateFactoryManager_LocalOrdinalTransfer("BlockNumber",multigridAlgo,paramList,defaultList,manager,levelID,keeps); + UpdateFactoryManager_LocalOrdinalTransfer("BlockNumber",multigridAlgo,paramList,defaultList,manager,levelID,keeps); // === Coordinates === @@ -1288,7 +1285,7 @@ namespace MueLu { template void ParameterListInterpreter:: UpdateFactoryManager_RAP(ParameterList& paramList, const ParameterList& defaultList, FactoryManager& manager, - int /* levelID */, std::vector& keeps) const + int levelID, std::vector& keeps) const { if (paramList.isParameter("A") && !paramList.get >("A").is_null()) { // We have user matrix A @@ -1443,30 +1440,30 @@ namespace MueLu { void ParameterListInterpreter:: UpdateFactoryManager_LocalOrdinalTransfer(const std::string & VarName, const std::string &multigridAlgo,ParameterList& paramList, const ParameterList& /* defaultList */, FactoryManager& manager, int levelID, std::vector& /* keeps */) const - { - if(levelID > 0){ - RCP fact = rcp(new LocalOrdinalTransferFactory(VarName,multigridAlgo)); - if(multigridAlgo == "classical") - fact->SetFactory("P Graph", manager.GetFactory("P Graph")); - else - fact->SetFactory("Aggregates", manager.GetFactory("Aggregates")); - fact->SetFactory("CoarseMap", manager.GetFactory("CoarseMap")); + { + // NOTE: You would think this would be levelID > 0, but you'd be wrong, since the FactoryManager is basically + // offset by a level from the things which actually do the work. + if (useBlockNumber_ && (levelID > 0)) { + auto RAP = rcp_const_cast(rcp_dynamic_cast(manager.GetFactory("A"))); + auto RAPs = rcp_const_cast(rcp_dynamic_cast(manager.GetFactory("A"))); + if (!RAP.is_null() || !RAPs.is_null()) { + RCP fact = rcp(new LocalOrdinalTransferFactory(VarName,multigridAlgo)); + if(multigridAlgo == "classical") + fact->SetFactory("P Graph", manager.GetFactory("P Graph")); + else + fact->SetFactory("Aggregates", manager.GetFactory("Aggregates")); + fact->SetFactory("CoarseMap", manager.GetFactory("CoarseMap")); - fact->SetFactory(VarName, this->GetFactoryManager(levelID-1)->GetFactory(VarName)); + fact->SetFactory(VarName, this->GetFactoryManager(levelID-1)->GetFactory(VarName)); - manager.SetFactory(VarName, fact); + manager.SetFactory(VarName, fact); - auto RAP = rcp_const_cast(rcp_dynamic_cast(manager.GetFactory("A"))); - if (!RAP.is_null()) { - RAP->AddTransferFactory(manager.GetFactory(VarName)); - } else { - auto RAPs = rcp_const_cast(rcp_dynamic_cast(manager.GetFactory("A"))); - RAPs->AddTransferFactory(manager.GetFactory(VarName)); + if (!RAP.is_null()) + RAP->AddTransferFactory(manager.GetFactory(VarName)); + else + RAPs->AddTransferFactory(manager.GetFactory(VarName)); } } - else { - throw std::runtime_error(std::string("UpdateFactoryManager_LocalOrdinalTransfer should not be run on level ") + std::to_string(levelID)); - } } @@ -1730,6 +1727,10 @@ namespace MueLu { manager.SetFactory("P", newP); if (useCoordinates_) manager.SetFactory("Coordinates", newP); + if (useBlockNumber_ && (levelID > 0)) { + newP->SetFactory("BlockNumber", manager.GetFactory("BlockNumber")); + manager.SetFactory("BlockNumber", newP); + } // Rebalanced R auto newR = rcp(new RebalanceTransferFactory()); diff --git a/packages/muelu/src/Rebalancing/MueLu_RebalanceTransferFactory_def.hpp b/packages/muelu/src/Rebalancing/MueLu_RebalanceTransferFactory_def.hpp index 4e25afe7dac7..fd9cdcce16ab 100644 --- a/packages/muelu/src/Rebalancing/MueLu_RebalanceTransferFactory_def.hpp +++ b/packages/muelu/src/Rebalancing/MueLu_RebalanceTransferFactory_def.hpp @@ -90,6 +90,7 @@ namespace MueLu { validParamList->set< RCP >("R", null, "Factory of the restriction operator that need to be rebalanced (only used if type=Restriction)"); validParamList->set< RCP >("Nullspace", null, "Factory of the nullspace that need to be rebalanced (only used if type=Interpolation)"); validParamList->set< RCP >("Coordinates", null, "Factory of the coordinates that need to be rebalanced (only used if type=Interpolation)"); + validParamList->set< RCP >("BlockNumber", null, "Factory of the block ids that need to be rebalanced (only used if type=Interpolation)"); validParamList->set< RCP >("Importer", null, "Factory of the importer object used for the rebalancing"); validParamList->set< int > ("write start", -1, "First level at which coordinates should be written to file"); validParamList->set< int > ("write end", -1, "Last level at which coordinates should be written to file"); @@ -111,6 +112,8 @@ namespace MueLu { Input(coarseLevel, "Nullspace"); if (pL.get< RCP >("Coordinates") != Teuchos::null) Input(coarseLevel, "Coordinates"); + if (pL.get< RCP >("BlockNumber") != Teuchos::null) + Input(coarseLevel, "BlockNumber"); } else { if (pL.get("transpose: use implicit") == false) @@ -138,6 +141,13 @@ namespace MueLu { Xpetra::IO::magnitudeType,LO,GO,NO>::Write(fileName, *fineCoords); } + if (writeStart == 0 && fineLevel.GetLevelID() == 0 && writeStart <= writeEnd && IsAvailable(fineLevel, "BlockNumber")) { + std::string fileName = "BlockNumber_level_0.m"; + RCP fineBlockNumber = fineLevel.Get< RCP >("BlockNumber"); + if (fineBlockNumber != Teuchos::null) + Xpetra::IO::Write(fileName, *fineBlockNumber); + } + RCP importer = Get >(coarseLevel, "Importer"); if (implicit) { // Save the importer, we'll need it for solve @@ -218,6 +228,10 @@ namespace MueLu { if (IsAvailable(coarseLevel, "Coordinates")) Set(coarseLevel, "Coordinates", Get< RCP >(coarseLevel, "Coordinates")); + if (pL.isParameter("BlockNumber") && pL.get< RCP >("BlockNumber") != Teuchos::null) + if (IsAvailable(coarseLevel, "BlockNumber")) + Set(coarseLevel, "BlockNumber", Get< RCP >(coarseLevel, "BlockNumber")); + return; } @@ -274,6 +288,30 @@ namespace MueLu { Xpetra::IO::magnitudeType,LO,GO,NO>::Write(fileName, *permutedCoords); } + if (pL.isParameter("BlockNumber") && + pL.get< RCP >("BlockNumber") != Teuchos::null && + IsAvailable(coarseLevel, "BlockNumber")) { + RCP BlockNumber = Get >(coarseLevel, "BlockNumber"); + + // This line must be after the Get call + SubFactoryMonitor subM(*this, "Rebalancing BlockNumber", coarseLevel); + + RCP permutedBlockNumber = LocalOrdinalVectorFactory::Build(importer->getTargetMap(), false); + permutedBlockNumber->doImport(*BlockNumber, *importer, Xpetra::INSERT); + + if (pL.isParameter("repartition: use subcommunicators") == true && pL.get("repartition: use subcommunicators") == true) + permutedBlockNumber->replaceMap(permutedBlockNumber->getMap()->removeEmptyProcesses()); + + if (permutedBlockNumber->getMap() == Teuchos::null) + permutedBlockNumber = Teuchos::null; + + Set(coarseLevel, "BlockNumber", permutedBlockNumber); + + std::string fileName = "rebalanced_BlockNumber_level_" + toString(coarseLevel.GetLevelID()) + ".m"; + if (writeStart <= coarseLevel.GetLevelID() && coarseLevel.GetLevelID() <= writeEnd && permutedBlockNumber->getMap() != Teuchos::null) + Xpetra::IO::Write(fileName, *permutedBlockNumber); + } + if (IsAvailable(coarseLevel, "Nullspace")) { RCP nullspace = Get< RCP >(coarseLevel, "Nullspace"); From 8e1016ae090b97f343611318f8fab9e9ca2f4686 Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Sun, 22 Aug 2021 16:49:16 -0600 Subject: [PATCH 14/70] MueLu CoalesceDropFactory: Add symmetrization of coloring graph --- packages/muelu/doc/UsersGuide/masterList.xml | 10 + .../muelu/doc/UsersGuide/paramlist_hidden.tex | 2 + .../MueLu_CoalesceDropFactory_decl.hpp | 2 +- .../MueLu_CoalesceDropFactory_def.hpp | 192 ++++++++++++------ .../MueLu_ParameterListInterpreter_def.hpp | 1 + .../muelu/src/MueCentral/MueLu_MasterList.cpp | 3 + 6 files changed, 145 insertions(+), 65 deletions(-) diff --git a/packages/muelu/doc/UsersGuide/masterList.xml b/packages/muelu/doc/UsersGuide/masterList.xml index 5b9e34ef1e36..088665579fa2 100644 --- a/packages/muelu/doc/UsersGuide/masterList.xml +++ b/packages/muelu/doc/UsersGuide/masterList.xml @@ -643,6 +643,16 @@ + + aggregation: coloring: localize color graph + bool + true + false + Localize the coloring graph generated by CoalesceDropFactory for SOC and for coloring + parameter not existing in ML + + + aggregation: enable phase 1 bool diff --git a/packages/muelu/doc/UsersGuide/paramlist_hidden.tex b/packages/muelu/doc/UsersGuide/paramlist_hidden.tex index 4a154f5c6d3f..d49769110253 100644 --- a/packages/muelu/doc/UsersGuide/paramlist_hidden.tex +++ b/packages/muelu/doc/UsersGuide/paramlist_hidden.tex @@ -126,6 +126,8 @@ \cbb{aggregation: coloring: use color graph}{bool}{false}{Have CoalesceDropFactory generate a seperate graph for SOC and for coloring} +\cbb{aggregation: coloring: localize color graph}{bool}{true}{Localize the coloring graph generated by CoalesceDropFactory for SOC and for coloring} + \cbb{aggregation: enable phase 1}{bool}{true}{Turn on/off phase 1 of aggregation} \cbb{aggregation: enable phase 2a}{bool}{true}{Turn on/off phase 2a of aggregation} diff --git a/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_decl.hpp b/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_decl.hpp index 1dde7f48d7bb..231cb1b78d18 100644 --- a/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_decl.hpp +++ b/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_decl.hpp @@ -179,7 +179,7 @@ namespace MueLu { Teuchos::RCP > BlockDiagonalize(Level & currentLevel,const RCP & A, bool generate_matrix) const; // When we want to decouple a block diagonal system via a *graph* - void BlockDiagonalizeGraph(const RCP & inputGraph, const RCP & ghostedBlockNumber, RCP & outputGraph) const; + void BlockDiagonalizeGraph(const RCP & inputGraph, const RCP & ghostedBlockNumber, RCP & outputGraph, RCP & importer) const; }; //class CoalesceDropFactory diff --git a/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_def.hpp b/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_def.hpp index 2422498d5442..b65228188264 100644 --- a/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_def.hpp +++ b/packages/muelu/src/Graph/MatrixTransformation/MueLu_CoalesceDropFactory_def.hpp @@ -59,6 +59,8 @@ #include #include +#include + #include "MueLu_CoalesceDropFactory_decl.hpp" #include "MueLu_AmalgamationFactory.hpp" @@ -74,6 +76,10 @@ #include "MueLu_PreDropFunctionConstVal.hpp" #include "MueLu_Utilities.hpp" +#ifdef HAVE_XPETRA_TPETRA +#include "Tpetra_CrsGraphTransposer.hpp" +#endif + #include #include #include @@ -131,6 +137,7 @@ namespace MueLu { } SET_VALID_ENTRY("aggregation: distance laplacian algo"); SET_VALID_ENTRY("aggregation: classical algo"); + SET_VALID_ENTRY("aggregation: coloring: localize color graph"); #undef SET_VALID_ENTRY validParamList->set< bool > ("lightweight wrap", true, "Experimental option for lightweight graph access"); @@ -646,22 +653,25 @@ namespace MueLu { // If we're doing signed classical, we might want to block-diagonalize *after* the dropping if(generateColoringGraph) { RCP colorGraph; - BlockDiagonalizeGraph(graph,ghostedBlockNumber,colorGraph); + RCP importer = A->getCrsGraph()->getImporter(); + BlockDiagonalizeGraph(graph,ghostedBlockNumber,colorGraph,importer); Set(currentLevel, "Coloring Graph",colorGraph); - //#define CMS_DUMP + // #define CMS_DUMP #ifdef CMS_DUMP { - int rank = graph->GetDomainMap()->getComm()->getRank(); - { - std::ofstream ofs(std::string("m_color_graph_") + std::to_string(currentLevel.GetLevelID())+std::string("_") + std::to_string(rank) + std::string(".dat"),std::ofstream::out); - RCP fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(ofs)); - colorGraph->print(*fancy,Debug); - } - { - std::ofstream ofs(std::string("m_regular_graph_") + std::to_string(currentLevel.GetLevelID())+std::string("_") + std::to_string(rank) + std::string(".dat"),std::ofstream::out); - RCP fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(ofs)); - graph->print(*fancy,Debug); - } + Xpetra::IO::Write("m_regular_graph."+std::to_string(currentLevel.GetLevelID()), *rcp_dynamic_cast(graph)->GetCrsGraph()); + Xpetra::IO::Write("m_color_graph."+std::to_string(currentLevel.GetLevelID()), *rcp_dynamic_cast(colorGraph)->GetCrsGraph()); + // int rank = graph->GetDomainMap()->getComm()->getRank(); + // { + // std::ofstream ofs(std::string("m_color_graph_") + std::to_string(currentLevel.GetLevelID())+std::string("_") + std::to_string(rank) + std::string(".dat"),std::ofstream::out); + // RCP fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(ofs)); + // colorGraph->print(*fancy,Debug); + // } + // { + // std::ofstream ofs(std::string("m_regular_graph_") + std::to_string(currentLevel.GetLevelID())+std::string("_") + std::to_string(rank) + std::string(".dat"),std::ofstream::out); + // RCP fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(ofs)); + // graph->print(*fancy,Debug); + // } } #endif @@ -1811,64 +1821,118 @@ namespace MueLu { template - void CoalesceDropFactory::BlockDiagonalizeGraph(const RCP & inputGraph, const RCP & ghostedBlockNumber, RCP & outputGraph) const { -typedef Teuchos::ScalarTraits STS; + void CoalesceDropFactory::BlockDiagonalizeGraph(const RCP & inputGraph, const RCP & ghostedBlockNumber, RCP & outputGraph, RCP & importer) const { + typedef Teuchos::ScalarTraits STS; - TEUCHOS_TEST_FOR_EXCEPTION(ghostedBlockNumber.is_null(), Exceptions::RuntimeError, "BlockDiagonalizeGraph(): ghostedBlockNumber is null."); -// const ParameterList & pL = GetParameterList(); + TEUCHOS_TEST_FOR_EXCEPTION(ghostedBlockNumber.is_null(), Exceptions::RuntimeError, "BlockDiagonalizeGraph(): ghostedBlockNumber is null."); + const ParameterList & pL = GetParameterList(); - GetOStream(Statistics1) << "Using BlockDiagonal Graph after Dropping (with provided blocking)"<("aggregation: coloring: localize color graph"); - // Accessors for block numbers - Teuchos::ArrayRCP row_block_number = ghostedBlockNumber->getData(0); - Teuchos::ArrayRCP col_block_number = ghostedBlockNumber->getData(0); - - // allocate space for the local graph - ArrayRCP rows_mat; - ArrayRCP rows_graph,columns; - - rows_graph.resize(inputGraph->GetNodeNumVertices()+1); - columns.resize(inputGraph->GetNodeNumEdges()); - - LO realnnz = 0; - GO numDropped = 0, numTotal = 0; - for (LO row = 0; row < Teuchos::as(inputGraph->GetDomainMap()->getNodeNumElements()); ++row) { - LO row_block = row_block_number[row]; - ArrayView indices = inputGraph->getNeighborVertices(row); + GetOStream(Statistics1) << "Using BlockDiagonal Graph after Dropping (with provided blocking)"; + if (localizeColoringGraph) + GetOStream(Statistics1) << ", with localization" <(indices.size()); colID++) { - LO col = indices[colID]; - LO col_block = col_block_number[col]; + // Accessors for block numbers + Teuchos::ArrayRCP row_block_number = ghostedBlockNumber->getData(0); + Teuchos::ArrayRCP col_block_number = ghostedBlockNumber->getData(0); - if((row_block == col_block) && (col < Teuchos::as(inputGraph->GetDomainMap()->getNodeNumElements()))) { - columns[realnnz++] = col; - rownnz++; - } else - numDropped++; - } - rows_graph[row+1] = realnnz; - } + // allocate space for the local graph + ArrayRCP rows_mat; + ArrayRCP rows_graph,columns; + rows_graph.resize(inputGraph->GetNodeNumVertices()+1); + columns.resize(inputGraph->GetNodeNumEdges()); + + LO realnnz = 0; + GO numDropped = 0, numTotal = 0; + const LO numRows = Teuchos::as(inputGraph->GetDomainMap()->getNodeNumElements()); + if (localizeColoringGraph) { + + for (LO row = 0; row < numRows; ++row) { + LO row_block = row_block_number[row]; + ArrayView indices = inputGraph->getNeighborVertices(row); + + LO rownnz = 0; + for (LO colID = 0; colID < Teuchos::as(indices.size()); colID++) { + LO col = indices[colID]; + LO col_block = col_block_number[col]; + + if((row_block == col_block) && (col < numRows)) { + columns[realnnz++] = col; + rownnz++; + } else + numDropped++; + } + rows_graph[row+1] = realnnz; + } + } else { + // ghosting of boundary node map + Teuchos::ArrayRCP boundaryNodes = inputGraph->GetBoundaryNodeMap(); + auto boundaryNodesVector = Xpetra::VectorFactory::Build(inputGraph->GetDomainMap()); + for (size_t i=0; iGetNodeNumVertices(); i++) + boundaryNodesVector->getDataNonConst(0)[i] = boundaryNodes[i]; + // Xpetra::IO::Write("boundary",*boundaryNodesVector); + auto boundaryColumnVector = Xpetra::VectorFactory::Build(inputGraph->GetImportMap()); + boundaryColumnVector->doImport(*boundaryNodesVector,*importer, Xpetra::INSERT); + auto boundaryColumn = boundaryColumnVector->getData(0); + + for (LO row = 0; row < numRows; ++row) { + LO row_block = row_block_number[row]; + ArrayView indices = inputGraph->getNeighborVertices(row); + + LO rownnz = 0; + for (LO colID = 0; colID < Teuchos::as(indices.size()); colID++) { + LO col = indices[colID]; + LO col_block = col_block_number[col]; + + if((row_block == col_block) && ((row == col) || (boundaryColumn[col] == 0))) { + columns[realnnz++] = col; + rownnz++; + } else + numDropped++; + } + rows_graph[row+1] = realnnz; + } + } - columns.resize(realnnz); - numTotal = inputGraph->GetNodeNumEdges(); - - if (GetVerbLevel() & Statistics1) { - RCP > comm = inputGraph->GetDomainMap()->getComm(); - GO numGlobalTotal, numGlobalDropped; - MueLu_sumAll(comm, numTotal, numGlobalTotal); - MueLu_sumAll(comm, numDropped, numGlobalDropped); - GetOStream(Statistics1) << "Number of dropped entries in block-diagonalized matrix graph: " << numGlobalDropped << "/" << numGlobalTotal; - if (numGlobalTotal != 0) - GetOStream(Statistics1) << " (" << 100*Teuchos::as(numGlobalDropped)/Teuchos::as(numGlobalTotal) << "%)"; - GetOStream(Statistics1) << std::endl; - } - - outputGraph = rcp(new LWGraph(rows_graph, columns, inputGraph->GetDomainMap(), inputGraph->GetImportMap(), "block-diagonalized graph of A")); - outputGraph->SetBoundaryNodeMap(inputGraph->GetBoundaryNodeMap()); - - + columns.resize(realnnz); + numTotal = inputGraph->GetNodeNumEdges(); + + if (GetVerbLevel() & Statistics1) { + RCP > comm = inputGraph->GetDomainMap()->getComm(); + GO numGlobalTotal, numGlobalDropped; + MueLu_sumAll(comm, numTotal, numGlobalTotal); + MueLu_sumAll(comm, numDropped, numGlobalDropped); + GetOStream(Statistics1) << "Number of dropped entries in block-diagonalized matrix graph: " << numGlobalDropped << "/" << numGlobalTotal; + if (numGlobalTotal != 0) + GetOStream(Statistics1) << " (" << 100*Teuchos::as(numGlobalDropped)/Teuchos::as(numGlobalTotal) << "%)"; + GetOStream(Statistics1) << std::endl; + } + + if (localizeColoringGraph) { + outputGraph = rcp(new LWGraph(rows_graph, columns, inputGraph->GetDomainMap(), inputGraph->GetImportMap(), "block-diagonalized graph of A")); + outputGraph->SetBoundaryNodeMap(inputGraph->GetBoundaryNodeMap()); + } else { + TEUCHOS_ASSERT(inputGraph->GetDomainMap()->lib() == Xpetra::UseTpetra); +#ifdef HAVE_XPETRA_TPETRA + auto outputGraph2 = rcp(new LWGraph(rows_graph, columns, inputGraph->GetDomainMap(), inputGraph->GetImportMap(), "block-diagonalized graph of A")); + + auto tpGraph = Xpetra::toTpetra(rcp_const_cast(outputGraph2->GetCrsGraph())); + auto sym = rcp(new Tpetra::CrsGraphTransposer(tpGraph)); + auto tpGraphSym = sym->symmetrize(); + + auto rowsSym = tpGraphSym->getNodeRowPtrs(); + ArrayRCP rows_graphSym; + rows_graphSym.resize(rowsSym.size()); + for (LO row = 0; row < rowsSym.size(); row++) + rows_graphSym[row] = rowsSym[row]; + outputGraph = rcp(new LWGraph(rows_graphSym, tpGraphSym->getNodePackedIndices(), inputGraph->GetDomainMap(), Xpetra::toXpetra(tpGraphSym->getColMap()), "block-diagonalized graph of A")); + outputGraph->SetBoundaryNodeMap(inputGraph->GetBoundaryNodeMap()); +#endif + } } diff --git a/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp b/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp index 5f13dcc95f29..2dd59644d87d 100644 --- a/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp +++ b/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp @@ -1073,6 +1073,7 @@ namespace MueLu { MUELU_TEST_AND_SET_PARAM_2LIST(paramList, defaultList, "aggregation: distance laplacian algo", std::string, dropParams); MUELU_TEST_AND_SET_PARAM_2LIST(paramList, defaultList, "aggregation: classical algo", std::string, dropParams); MUELU_TEST_AND_SET_PARAM_2LIST(paramList, defaultList, "aggregation: distance laplacian directional weights",Teuchos::Array,dropParams); + MUELU_TEST_AND_SET_PARAM_2LIST(paramList, defaultList, "aggregation: coloring: localize color graph", bool, dropParams); if (useKokkos_) { MUELU_TEST_AND_SET_PARAM_2LIST(paramList, defaultList, "filtered matrix: use lumping", bool, dropParams); MUELU_TEST_AND_SET_PARAM_2LIST(paramList, defaultList, "filtered matrix: reuse graph", bool, dropParams); diff --git a/packages/muelu/src/MueCentral/MueLu_MasterList.cpp b/packages/muelu/src/MueCentral/MueLu_MasterList.cpp index 3452824f7a74..ed38c3353ed0 100644 --- a/packages/muelu/src/MueCentral/MueLu_MasterList.cpp +++ b/packages/muelu/src/MueCentral/MueLu_MasterList.cpp @@ -228,6 +228,7 @@ namespace MueLu { "" "" "" + "" "" "" "" @@ -664,6 +665,8 @@ namespace MueLu { ("aggregation: coloring: use color graph","aggregation: coloring: use color graph") + ("aggregation: coloring: localize color graph","aggregation: coloring: localize color graph") + ("aggregation: enable phase 1","aggregation: enable phase 1") ("aggregation: enable phase 2a","aggregation: enable phase 2a") From 6bbd3bbccea32f9aa17b099c5f66b3b55e9be5f2 Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Sun, 22 Aug 2021 16:50:24 -0600 Subject: [PATCH 15/70] Tpetra: Add CrsGraphTransposer --- packages/tpetra/core/src/CMakeLists.txt | 6 +- .../src/Tpetra_CrsGraphTransposer_decl.hpp | 126 +++++ .../src/Tpetra_CrsGraphTransposer_def.hpp | 521 ++++++++++++++++++ .../src/Tpetra_CrsGraphTransposer_fwd.hpp | 59 ++ 4 files changed, 711 insertions(+), 1 deletion(-) create mode 100644 packages/tpetra/core/src/Tpetra_CrsGraphTransposer_decl.hpp create mode 100644 packages/tpetra/core/src/Tpetra_CrsGraphTransposer_def.hpp create mode 100644 packages/tpetra/core/src/Tpetra_CrsGraphTransposer_fwd.hpp diff --git a/packages/tpetra/core/src/CMakeLists.txt b/packages/tpetra/core/src/CMakeLists.txt index 51054675d103..88ec363df3bc 100644 --- a/packages/tpetra/core/src/CMakeLists.txt +++ b/packages/tpetra/core/src/CMakeLists.txt @@ -38,7 +38,7 @@ FUNCTION(TPETRA_SLG_MACRO_NAME NAME_OUT NAME_IN) ELSE() #long double is a special name; add _ to macro name STRING(COMPARE EQUAL "${NAME_IN}" "long double" IS_LONG_DOUBLE) - IF(IS_LONG_DOUBLE) + IF(IS_LONG_DOUBLE) SET(${NAME_OUT} "LONG_DOUBLE" PARENT_SCOPE) ELSE() # Convert to upper case, convert double colons to underscores, @@ -701,6 +701,10 @@ IF (${PACKAGE_NAME}_ENABLE_EXPLICIT_INSTANTIATION) TPETRA_PROCESS_ALL_SLGN_TEMPLATES(ROWMATRIXTRANSPOSER_OUTPUT_FILES "Tpetra_ETI_SC_LO_GO_NT.tmpl" "RowMatrixTransposer" "ROWMATRIXTRANSPOSER" "${CrsMatrix_ETI_SCALARS}" "${TpetraCore_ETI_LORDS}" "${TpetraCore_ETI_GORDS}" "${TpetraCore_ETI_NODES}" TRUE) LIST(APPEND SOURCES ${ROWMATRIXTRANSPOSER_OUTPUT_FILES}) + # Generate ETI .cpp files for Tpetra::CrsGraphTransposer. + TPETRA_PROCESS_ALL_LGN_TEMPLATES(CRSGRAPHTRANSPOSER_OUTPUT_FILES "Tpetra_ETI_LO_GO_NT.tmpl" "CrsGraphTransposer" "CRSGRAPHTRANSPOSER" "${TpetraCore_ETI_LORDS}" "${TpetraCore_ETI_GORDS}" "${TpetraCore_ETI_NODES}") + LIST(APPEND SOURCES ${CRSGRAPHTRANSPOSER_OUTPUT_FILES}) + # Generate ETI .cpp files for Tpetra::BlockCrsMatrix. TPETRA_PROCESS_ALL_SLGN_TEMPLATES(BLOCKCRSMATRIX_OUTPUT_FILES "Tpetra_ETI_SC_LO_GO_NT.tmpl" "BlockCrsMatrix" "BLOCKCRSMATRIX" "${TpetraCore_ETI_SCALARS_NO_ORDS}" "${TpetraCore_ETI_LORDS}" "${TpetraCore_ETI_GORDS}" "${TpetraCore_ETI_NODES}" FALSE) LIST(APPEND SOURCES ${BLOCKCRSMATRIX_OUTPUT_FILES}) diff --git a/packages/tpetra/core/src/Tpetra_CrsGraphTransposer_decl.hpp b/packages/tpetra/core/src/Tpetra_CrsGraphTransposer_decl.hpp new file mode 100644 index 000000000000..055eb0742b73 --- /dev/null +++ b/packages/tpetra/core/src/Tpetra_CrsGraphTransposer_decl.hpp @@ -0,0 +1,126 @@ + +// @HEADER +// *********************************************************************** +// +// Tpetra: Templated Linear Algebra Services Package +// Copyright (2008) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Michael A. Heroux (maherou@sandia.gov) +// +// ************************************************************************ +// @HEADER + +#ifndef TPETRA_CRSGRAPHTRANSPOSER_DECL_HPP +#define TPETRA_CRSGRAPHTRANSPOSER_DECL_HPP + +/// \file Tpetra_CrsGraphTransposer_decl.hpp +/// +/// Declaration of Tpetra::CrsGraphTransposer. + +#include "Tpetra_CrsGraphTransposer_fwd.hpp" +#include "Tpetra_CrsGraph_fwd.hpp" +#include "Tpetra_Map_fwd.hpp" +#include "Teuchos_RCP.hpp" +#include + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace Teuchos { + // Forward declaration of ParameterList + class ParameterList; +} // namespace Teuchos +#endif // DOXYGEN_SHOULD_SKIP_THIS + +namespace Tpetra { + +/// \class CrsGraphTransposer +/// \brief Construct and (optionally) redistribute the explicitly +/// stored transpose of a CrsGraph. +/// +/// This class is based on the EpetraExt version. It first transposes +/// the graph to an intermediate version with overlapping row map. +/// That graph is then converted to a final version whose row map is +/// "unique", i.e., a row is wholly owned by one process. +/// +/// This class takes the same template parameters as CrsGraph. +template +class CrsGraphTransposer { +public: + //! @name Typedefs + //@{ + typedef LocalOrdinal local_ordinal_type; + typedef GlobalOrdinal global_ordinal_type; + typedef Node node_type; + + typedef Map map_type; + typedef CrsGraph crs_graph_type; + + //@} + //! @name Constructors + //@{ + + //! Constructor that takes the graph to transpose. + CrsGraphTransposer (const Teuchos::RCP& origGraph,const std::string & label = std::string()); + + //@} + //! @name Methods for computing the explicit transpose. + //@{ + + //! Compute and return graph+graph^T of the graph given to the constructor. + Teuchos::RCP symmetrize(const Teuchos::RCP ¶ms=Teuchos::null); + + //! Compute and return the transpose of the graph given to the constructor. + Teuchos::RCP createTranspose(const Teuchos::RCP ¶ms=Teuchos::null); + + /// \brief Compute and return the transpose of the graph given to the constructor. + /// + /// In this call, we (potentially) leave the graph with an + /// overlapping row Map. This is a perfectly valid graph, but + /// won't work correctly with some routines in Ifpack or Muelu. + /// + /// \warning This routine leaves overlapping rows. Unless you're + /// sure that's OK, call createTranspose() instead. + Teuchos::RCP createTransposeLocal(const Teuchos::RCP ¶ms=Teuchos::null); + +private: + //! The original graph to be transposed. + Teuchos::RCP origGraph_; + + //! Label for timers + std::string label_; +}; + +} // namespace Tpetra + +#endif /* TPETRA_CRSGRAPHTRANSPOSER_DECL_HPP */ diff --git a/packages/tpetra/core/src/Tpetra_CrsGraphTransposer_def.hpp b/packages/tpetra/core/src/Tpetra_CrsGraphTransposer_def.hpp new file mode 100644 index 000000000000..28e63f3fcd78 --- /dev/null +++ b/packages/tpetra/core/src/Tpetra_CrsGraphTransposer_def.hpp @@ -0,0 +1,521 @@ +// @HEADER +// *********************************************************************** +// +// Tpetra: Templated Linear Algebra Services Package +// Copyright (2008) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Michael A. Heroux (maherou@sandia.gov) +// +// ************************************************************************ +// @HEADER + +#ifndef TPETRA_CRSGRAPHTRANSPOSER_DEF_HPP +#define TPETRA_CRSGRAPHTRANSPOSER_DEF_HPP + +#include "Tpetra_CrsGraph.hpp" +#include "Tpetra_Export.hpp" +#include "Tpetra_Details_computeOffsets.hpp" +#include "Tpetra_Details_makeColMap.hpp" +#include "Tpetra_Details_shortSort.hpp" +#include "Teuchos_ParameterList.hpp" +#include "Teuchos_TimeMonitor.hpp" +#include "KokkosKernels_SparseUtils.hpp" +#include "KokkosKernels_Handle.hpp" +#include "KokkosSparse_spadd.hpp" + +namespace Tpetra { + + template + struct ConvertLocalToGlobalFunctor + { + ConvertLocalToGlobalFunctor( + const LocalIndicesType& colindsOrig_, + const GlobalIndicesType& colindsConverted_, + const ColMapType& colmap_) : + colindsOrig (colindsOrig_), + colindsConverted (colindsConverted_), + colmap (colmap_) + {} + KOKKOS_INLINE_FUNCTION void + operator() (const GO i) const + { + colindsConverted(i) = colmap.getGlobalElement(colindsOrig(i)); + } + LocalIndicesType colindsOrig; + GlobalIndicesType colindsConverted; + ColMapType colmap; + }; + + template + struct ConvertGlobalToLocalFunctor + { + ConvertGlobalToLocalFunctor(LOView& lids_, const GOView& gids_, const LocalMap localColMap_) + : lids(lids_), gids(gids_), localColMap(localColMap_) + {} + + KOKKOS_FUNCTION void operator() (const GO i) const + { + lids(i) = localColMap.getLocalElement(gids(i)); + } + + LOView lids; + const GOView gids; + const LocalMap localColMap; + }; + + + template + struct SortedNumericIndicesOnlyFunctor { + + SortedNumericIndicesOnlyFunctor(const ArowptrsT& Arowptrs_, + const BrowptrsT& Browptrs_, + const CrowptrsT& Crowptrs_, + const AcolindsT& Acolinds_, + const BcolindsT& Bcolinds_, + const CcolindsT& Ccolinds_) + : Arowptrs(Arowptrs_), + Browptrs(Browptrs_), + Crowptrs(Crowptrs_), + Acolinds(Acolinds_), + Bcolinds(Bcolinds_), + Ccolinds(Ccolinds_) {} + + KOKKOS_INLINE_FUNCTION void operator()(const ordinal_type i) const + { + const ordinal_type ORDINAL_MAX = Kokkos::ArithTraits::max(); + + // count the union of nonzeros in Arow and Brow + size_type ai = 0; + size_type bi = 0; + size_type Arowstart = Arowptrs(i); + size_type Arowlen = Arowptrs(i + 1) - Arowstart; + size_type Browstart = Browptrs(i); + size_type Browlen = Browptrs(i + 1) - Browstart; + ordinal_type Acol = (Arowlen == 0) ? ORDINAL_MAX : Acolinds(Arowstart); + ordinal_type Bcol = (Browlen == 0) ? ORDINAL_MAX : Bcolinds(Browstart); + size_type Coffset = Crowptrs(i); + while (Acol != ORDINAL_MAX || Bcol != ORDINAL_MAX) + { + ordinal_type Ccol = (Acol < Bcol) ? Acol : Bcol; + while(Acol == Ccol) + { + ai++; + if(ai == Arowlen) + Acol = ORDINAL_MAX; + else + Acol = Acolinds(Arowstart + ai); + } + while(Bcol == Ccol) + { + bi++; + if(bi == Browlen) + Bcol = ORDINAL_MAX; + else + Bcol = Bcolinds(Browstart + bi); + } + Ccolinds(Coffset) = Ccol; + Coffset++; + } + } + + const ArowptrsT Arowptrs; + const BrowptrsT Browptrs; + const CrowptrsT Crowptrs; + const AcolindsT Acolinds; + const BcolindsT Bcolinds; + CcolindsT Ccolinds; + }; + + template + struct UnsortedNumericIndicesOnlyFunctor { + + UnsortedNumericIndicesOnlyFunctor( + const ArowptrsT Arowptrs_, const BrowptrsT Browptrs_, const CrowptrsT Crowptrs_, + const AcolindsT Acolinds_, const BcolindsT Bcolinds_, CcolindsT Ccolinds_, + const CcolindsT Apos_, const CcolindsT Bpos_) + : Arowptrs(Arowptrs_), + Browptrs(Browptrs_), + Crowptrs(Crowptrs_), + Acolinds(Acolinds_), + Bcolinds(Bcolinds_), + Ccolinds(Ccolinds_), + Apos(Apos_), + Bpos(Bpos_) {} + + KOKKOS_INLINE_FUNCTION void operator()(const ordinal_type i) const { + size_type CrowStart = Crowptrs(i); + size_type ArowStart = Arowptrs(i); + size_type ArowEnd = Arowptrs(i + 1); + size_type BrowStart = Browptrs(i); + size_type BrowEnd = Browptrs(i + 1); + // add in A entries, while setting C colinds + for (size_type j = ArowStart; j < ArowEnd; j++) { + Ccolinds(CrowStart + Apos(j)) = Acolinds(j); + } + // add in B entries, while setting C colinds + for (size_type j = BrowStart; j < BrowEnd; j++) { + Ccolinds(CrowStart + Bpos(j)) = Bcolinds(j); + } + } + const ArowptrsT Arowptrs; + const BrowptrsT Browptrs; + const CrowptrsT Crowptrs; + const AcolindsT Acolinds; + const BcolindsT Bcolinds; + CcolindsT Ccolinds; + const CcolindsT Apos; + const CcolindsT Bpos; + }; + + + template + CrsGraphTransposer:: + CrsGraphTransposer (const Teuchos::RCP& origGraph, + const std::string& label) + : origGraph_ (origGraph), label_ (label) + {} + + template + Teuchos::RCP > + CrsGraphTransposer:: + symmetrize (const Teuchos::RCP ¶ms) + { + using Teuchos::RCP; + using device_type = typename Node::device_type; + using execution_space = typename device_type::execution_space; + using range_type = Kokkos::RangePolicy; + using local_graph_device_type = typename crs_graph_type::local_graph_device_type; + using impl_scalar_type = ::Tpetra::Details::DefaultTypes::scalar_type; + using row_ptrs_array = typename local_graph_device_type::row_map_type::non_const_type ; + using col_inds_array = typename local_graph_device_type::entries_type::non_const_type; + using local_map_type = typename map_type::local_map_type; + using global_col_inds_array = typename Kokkos::View; + + auto graph = origGraph_; + auto domain_map = graph->getDomainMap(); + auto range_map = graph->getRangeMap(); + auto row_map = graph->getRowMap(); + auto col_map = graph->getColMap(); + RCP col_map_sym; + RCP > importer; + + TEUCHOS_ASSERT(domain_map->isSameAs(*range_map)); + TEUCHOS_ASSERT(domain_map->isSameAs(*row_map)); + + // Do the transpose + RCP graphT = createTranspose (params); + + auto col_map_T = graphT->getColMap(); + TEUCHOS_ASSERT(!col_map_T.is_null()); + TEUCHOS_ASSERT(domain_map->isSameAs(*graphT->getDomainMap())); + + bool graphSorted = graph->isSorted(); + bool graphTSorted = graphT->isSorted(); + bool sorted = graphSorted && graphTSorted; + bool matchingColMaps = col_map->isSameAs(*col_map_T); + + auto lclGraph = graph->getLocalGraphDevice(); + auto lclGraphT = graphT->getLocalGraphDevice(); + + using KKH_LO = KokkosKernels::Experimental::KokkosKernelsHandle; + using KKH_GO = KokkosKernels::Experimental::KokkosKernelsHandle; + + auto rowptrs = lclGraph.row_map; + auto rowptrsT = lclGraphT.row_map; + auto colinds = lclGraph.entries; + auto colindsT = lclGraphT.entries; + + auto nrows = rowptrs.extent(0) - 1; + auto rowptrsSym = row_ptrs_array(Kokkos::ViewAllocateWithoutInitializing("row ptrs sym"), nrows + 1); + + col_inds_array colindsSym; + + if(!matchingColMaps) { + // convert indices of local graph to GlobalOrdinal + auto lclColmap = col_map->getLocalMap(); + global_col_inds_array colindsConverted(Kokkos::ViewAllocateWithoutInitializing("colinds (converted)"), colinds.extent(0)); + ConvertLocalToGlobalFunctor convert(colinds, colindsConverted, lclColmap); + Kokkos::parallel_for("colInds (converted)", range_type(0, colinds.extent(0)), convert); + + // convert indices of local graphT to GlobalOrdinal + auto lclColmapT = col_map_T->getLocalMap(); + global_col_inds_array colindsTConverted(Kokkos::ViewAllocateWithoutInitializing("colindsT (converted)"), colindsT.extent(0)); + ConvertLocalToGlobalFunctor convertT(colindsT, colindsTConverted, lclColmapT); + Kokkos::parallel_for("colIndsT (converted)", range_type(0, colindsT.extent(0)), convertT); + + // sum graph and graphT in GlobalOrdinal + KKH_GO handle; + handle.create_spadd_handle(false); + auto addHandle = handle.get_spadd_handle(); + + global_col_inds_array globalColindsSym; + + KokkosSparse::Experimental::spadd_symbolic + + (&handle, rowptrs, colindsConverted, rowptrsT, colindsTConverted, rowptrsSym); + globalColindsSym = global_col_inds_array(Kokkos::ViewAllocateWithoutInitializing("global colinds sym"), addHandle->get_c_nnz()); + + UnsortedNumericIndicesOnlyFunctor< + size_t, GlobalOrdinal, + typename row_ptrs_array::const_type, typename row_ptrs_array::const_type, row_ptrs_array, + typename global_col_inds_array::const_type, typename global_col_inds_array::const_type, global_col_inds_array> + unsortedNumeric(rowptrs, rowptrsT, rowptrsSym, + colindsConverted, colindsTConverted, globalColindsSym, + addHandle->get_a_pos(), addHandle->get_b_pos()); + Kokkos::parallel_for("KokkosSparse::SpAdd:Numeric::InputNotSorted", + range_type(0, nrows), unsortedNumeric); + + // build column map for graphSym + Tpetra::Details::makeColMap + (col_map_sym, domain_map, globalColindsSym); + + // convert indices of local graphSym to LocalOrdinal + auto lclColmapSym = col_map_sym->getLocalMap(); + colindsSym = col_inds_array("colindsSym", globalColindsSym.extent(0)); + ConvertGlobalToLocalFunctor convertSym(colindsSym, globalColindsSym, lclColmapSym); + Kokkos::parallel_for(range_type(0, globalColindsSym.extent(0)), convertSym); + + } else { + + // sum graph and graphT in LocalOrdinal + KKH_LO handle; + handle.create_spadd_handle(sorted); + auto addHandle = handle.get_spadd_handle(); + + KokkosSparse::Experimental::spadd_symbolic + + (&handle, rowptrs, colinds, rowptrsT, colindsT, rowptrsSym); + colindsSym = col_inds_array(Kokkos::ViewAllocateWithoutInitializing("C colinds"), addHandle->get_c_nnz()); + + if (sorted) { + SortedNumericIndicesOnlyFunctor< + size_t, LocalOrdinal, + typename row_ptrs_array::const_type, typename row_ptrs_array::const_type, row_ptrs_array, + typename col_inds_array::const_type, typename col_inds_array::const_type, col_inds_array> + sortedNumeric(rowptrs, rowptrsT, rowptrsSym, + colinds, colindsT, colindsSym); + Kokkos::parallel_for("KokkosSparse::SpAdd:Numeric::InputSorted", + range_type(0, nrows), sortedNumeric); + + } else { + UnsortedNumericIndicesOnlyFunctor< + size_t, LocalOrdinal, + typename row_ptrs_array::const_type, typename row_ptrs_array::const_type, row_ptrs_array, + typename col_inds_array::const_type, typename col_inds_array::const_type, col_inds_array> + unsortedNumeric(rowptrs, rowptrsT, rowptrsSym, + colinds, colindsT, colindsSym, + addHandle->get_a_pos(), addHandle->get_b_pos()); + Kokkos::parallel_for("KokkosSparse::SpAdd:Numeric::InputNotSorted", + range_type(0, nrows), unsortedNumeric); + } + + // column map for graphSym is graph's column map + col_map_sym = col_map; + importer = graph->getImporter(); + } + + bool sort = true; + if (sort) + KokkosKernels::Impl::sort_crs_graph(rowptrsSym, colindsSym); + + local_graph_device_type lclGraphSym = local_graph_device_type(colindsSym, rowptrsSym); + + RCP graphParams = Teuchos::null; + if(!sort) { + graphParams = rcp(new Teuchos::ParameterList); + graphParams->set("sorted", false); + } + + return rcp (new crs_graph_type (lclGraphSym, + row_map, + col_map_sym, + domain_map, + range_map, + importer, + Teuchos::null, + graphParams)); + } + + template + Teuchos::RCP > + CrsGraphTransposer:: + createTranspose (const Teuchos::RCP ¶ms) + { + using Teuchos::RCP; + // Do the local transpose + RCP transGraphWithSharedRows = createTransposeLocal (params); + +#ifdef HAVE_TPETRA_MMM_TIMINGS + const std::string prefix = std::string ("Tpetra ") + label_ + ": "; + using Teuchos::TimeMonitor; + TimeMonitor MM (*TimeMonitor::getNewTimer (prefix + "Transpose TAFC")); +#endif + + // If transGraphWithSharedRows has an exporter, that's what we + // want. If it doesn't, the rows aren't actually shared, and we're + // done! + using export_type = Export; + RCP exporter = + transGraphWithSharedRows->getExporter (); + if (exporter.is_null ()) { + return transGraphWithSharedRows; + } + else { + Teuchos::ParameterList labelList; +#ifdef HAVE_TPETRA_MMM_TIMINGS + labelList.set("Timer Label", label_); +#endif + if(! params.is_null ()) { + const char paramName[] = "compute global constants"; + labelList.set (paramName, params->get (paramName, true)); + } + // Use the Export object to do a fused Export and fillComplete. + // This always sorts the local graph after communication, so + // no need to set "sorted = false" in parameters. + return exportAndFillCompleteCrsGraph + (transGraphWithSharedRows, *exporter, Teuchos::null, + Teuchos::null, Teuchos::rcpFromRef (labelList)); + } + } + + template + Teuchos::RCP > + CrsGraphTransposer:: + createTransposeLocal (const Teuchos::RCP ¶ms) + { + using Teuchos::RCP; + using Teuchos::rcp; + using Teuchos::rcp_dynamic_cast; + using LO = LocalOrdinal; + using GO = GlobalOrdinal; + using import_type = Tpetra::Import; + using export_type = Tpetra::Export; + +#ifdef HAVE_TPETRA_MMM_TIMINGS + std::string prefix = std::string("Tpetra ") + label_ + ": "; + using Teuchos::TimeMonitor; + TimeMonitor MM (*TimeMonitor::getNewTimer (prefix + "Transpose Local")); +#endif + + const bool sort = [&] () { + constexpr bool sortDefault = true; // see #4607 discussion + const char sortParamName[] = "sort"; + return params.get () == nullptr ? sortDefault : + params->get (sortParamName, sortDefault); + } (); + + using local_graph_device_type = typename crs_graph_type::local_graph_device_type; + local_graph_device_type lclGraph = origGraph_->getLocalGraphDevice (); + + //Allocate views and call the other version of transpose_graph + using c_rowmap_t = typename local_graph_device_type::row_map_type; + using c_entries_t = typename local_graph_device_type::entries_type; + using rowmap_t = typename local_graph_device_type::row_map_type::non_const_type; + using entries_t = typename local_graph_device_type::entries_type::non_const_type; + LocalOrdinal numCols = origGraph_->getColMap()->getNodeNumElements(); + rowmap_t lclGraphT_rowmap("Transpose rowmap", numCols + 1); + entries_t lclGraphT_entries( + Kokkos::ViewAllocateWithoutInitializing("Transpose entries"), lclGraph.entries.extent(0)); + KokkosKernels::Impl::transpose_graph< + c_rowmap_t, c_entries_t, + rowmap_t, entries_t, + rowmap_t, typename local_graph_device_type::execution_space>( + lclGraph.numRows(), numCols, + lclGraph.row_map, lclGraph.entries, + lclGraphT_rowmap, lclGraphT_entries); + + if (sort) + KokkosKernels::Impl::sort_crs_graph< + typename local_graph_device_type::execution_space, + rowmap_t, entries_t>( + lclGraphT_rowmap, + lclGraphT_entries); + + //And construct the transpose local_graph_device_type + local_graph_device_type lclGraphT = local_graph_device_type(lclGraphT_entries, lclGraphT_rowmap); + + // Prebuild the importers and exporters the no-communication way, + // flipping the importers and exporters around. + const auto origExport = origGraph_->getExporter (); + RCP myImport = origExport.is_null () ? + Teuchos::null : rcp (new import_type (*origExport)); + const auto origImport = origGraph_->getImporter (); + RCP myExport = origImport.is_null () ? + Teuchos::null : rcp (new export_type (*origImport)); + + RCP graphParams = Teuchos::null; + if(!sort) { + graphParams = rcp(new Teuchos::ParameterList); + graphParams->set("sorted", false); + } + + return rcp (new crs_graph_type (lclGraphT, + origGraph_->getColMap (), + origGraph_->getRowMap (), + origGraph_->getRangeMap (), + origGraph_->getDomainMap (), + myImport, myExport, graphParams)); + } + + // + // Explicit instantiation macro + // + // Must be expanded from within the Tpetra namespace! + // + +#define TPETRA_CRSGRAPHTRANSPOSER_INSTANT(LO,GO,NODE) \ + template class CrsGraphTransposer< LO , GO , NODE >; + +} // namespace Tpetra + +#endif diff --git a/packages/tpetra/core/src/Tpetra_CrsGraphTransposer_fwd.hpp b/packages/tpetra/core/src/Tpetra_CrsGraphTransposer_fwd.hpp new file mode 100644 index 000000000000..f03d84702acc --- /dev/null +++ b/packages/tpetra/core/src/Tpetra_CrsGraphTransposer_fwd.hpp @@ -0,0 +1,59 @@ +// @HEADER +// *********************************************************************** +// +// Tpetra: Templated Linear Algebra Services Package +// Copyright (2008) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Michael A. Heroux (maherou@sandia.gov) +// +// ************************************************************************ +// @HEADER + +#ifndef TPETRA_CRSGRAPHTRANSPOSER_FWD_HPP +#define TPETRA_CRSGRAPHTRANSPOSER_FWD_HPP + +#include "Tpetra_Details_DefaultTypes.hpp" + +/// \file Tpetra_CrsGraphTransposer_fwd.hpp +/// \brief Forward declaration of Tpetra::CrsGraphTransposer + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace Tpetra { +template +class CrsGraphTransposer; +} // namespace Tpetra +#endif // DOXYGEN_SHOULD_SKIP_THIS + +#endif // TPETRA_CRSGRAPHTRANSPOSER_FWD_HPP From 629659aabcb019a21144b489815dd0cddba2301d Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Sun, 22 Aug 2021 16:50:59 -0600 Subject: [PATCH 16/70] Tpetra: Fix some timer names --- packages/tpetra/core/ext/TpetraExt_MatrixMatrix_def.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/tpetra/core/ext/TpetraExt_MatrixMatrix_def.hpp b/packages/tpetra/core/ext/TpetraExt_MatrixMatrix_def.hpp index 96e9e8763d33..a11686002906 100644 --- a/packages/tpetra/core/ext/TpetraExt_MatrixMatrix_def.hpp +++ b/packages/tpetra/core/ext/TpetraExt_MatrixMatrix_def.hpp @@ -3221,7 +3221,7 @@ addUnsorted( handle.create_spadd_handle(false); auto addHandle = handle.get_spadd_handle(); #ifdef HAVE_TPETRA_MMM_TIMINGS - auto MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("TpetraExt::MatrixMatrix::add() sorted symbolic"))); + auto MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("TpetraExt::MatrixMatrix::add() unsorted symbolic"))); #endif KokkosSparse::Experimental::spadd_symbolic get_c_nnz()); #ifdef HAVE_TPETRA_MMM_TIMINGS MM = Teuchos::null; - MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("TpetraExt::MatrixMatrix::add() sorted kernel: sorted numeric"))); + MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("TpetraExt::MatrixMatrix::add() unsorted kernel: unsorted numeric"))); #endif KokkosSparse::Experimental::spadd_numeric(&handle, Arowptrs, Acolinds, Avals, scalarA, From 362f27496360bd7bce72a36e09bcbff80fa2b1b5 Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Mon, 23 Aug 2021 12:43:07 -0600 Subject: [PATCH 17/70] Tpetra: take reverseOption out of DistObject::doWaits --- .../tpetra/core/src/Tpetra_DistObject_decl.hpp | 3 +-- packages/tpetra/core/src/Tpetra_DistObject_def.hpp | 14 ++++---------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp index 38cb37a3098e..016e1e96c7c7 100644 --- a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp @@ -792,8 +792,7 @@ namespace Tpetra { const bool canTryAliasing, const CombineMode CM); - void doWaits(const Details::DistributorPlan& distributorPlan, - ReverseOption revOp); + void doWaits(const Details::DistributorPlan& distributorPlan); void doPackAndPrepare(const SrcDistObject& src, const Kokkos::DualView& exportLIDs, diff --git a/packages/tpetra/core/src/Tpetra_DistObject_def.hpp b/packages/tpetra/core/src/Tpetra_DistObject_def.hpp index c3b4550eb751..a126fdfd409c 100644 --- a/packages/tpetra/core/src/Tpetra_DistObject_def.hpp +++ b/packages/tpetra/core/src/Tpetra_DistObject_def.hpp @@ -1242,7 +1242,7 @@ namespace Tpetra { } Distributor& distor = transfer.getDistributor (); - const Details::DistributorPlan& distributorPlan = distor.getPlan(); + const Details::DistributorPlan& distributorPlan = (revOp == DoForward) ? distor.getPlan() : *distor.getPlan().getReversePlan(); TEUCHOS_TEST_FOR_EXCEPTION (debug && restrictedMode && @@ -1321,7 +1321,7 @@ namespace Tpetra { } } else { - doWaits(distributorPlan, revOp); + doWaits(distributorPlan); if (verbose) { std::ostringstream os; @@ -1594,15 +1594,9 @@ namespace Tpetra { template void DistObject:: - doWaits(const Details::DistributorPlan& distributorPlan, - ReverseOption revOp) + doWaits(const Details::DistributorPlan& distributorPlan) { - if (revOp == DoReverse) { - distributorActor_.doWaits(*distributorPlan.getReversePlan()); - } - else { - distributorActor_.doWaits(distributorPlan); - } + distributorActor_.doWaits(distributorPlan); } template From e0f107ce62a5fbe657484b123914b308dec26acf Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Mon, 23 Aug 2021 12:50:08 -0600 Subject: [PATCH 18/70] Tpetra: take reverseOption out of DistObject::doPosts --- .../core/src/Tpetra_DistObject_decl.hpp | 3 - .../tpetra/core/src/Tpetra_DistObject_def.hpp | 123 +++++------------- 2 files changed, 31 insertions(+), 95 deletions(-) diff --git a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp index 016e1e96c7c7..45eac4cd0119 100644 --- a/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_DistObject_decl.hpp @@ -787,13 +787,10 @@ namespace Tpetra { void doPosts(const Details::DistributorPlan& distributorPlan, size_t constantNumPackets, bool commOnHost, - ReverseOption revOp, std::shared_ptr prefix, const bool canTryAliasing, const CombineMode CM); - void doWaits(const Details::DistributorPlan& distributorPlan); - void doPackAndPrepare(const SrcDistObject& src, const Kokkos::DualView& exportLIDs, size_t& constantNumPackets); diff --git a/packages/tpetra/core/src/Tpetra_DistObject_def.hpp b/packages/tpetra/core/src/Tpetra_DistObject_def.hpp index a126fdfd409c..73cc655062fe 100644 --- a/packages/tpetra/core/src/Tpetra_DistObject_def.hpp +++ b/packages/tpetra/core/src/Tpetra_DistObject_def.hpp @@ -887,7 +887,7 @@ namespace Tpetra { const size_t numSameIDs = transfer.getNumSameIDs (); Distributor& distor = transfer.getDistributor (); - const Details::DistributorPlan& distributorPlan = distor.getPlan(); + const Details::DistributorPlan& distributorPlan = (revOp == DoForward) ? distor.getPlan() : *distor.getPlan().getReversePlan(); TEUCHOS_TEST_FOR_EXCEPTION (debug && restrictedMode && @@ -1123,7 +1123,7 @@ namespace Tpetra { std::cerr << os.str (); } - doPosts(distributorPlan, constantNumPackets, commOnHost, revOp, prefix, canTryAliasing, CM); + doPosts(distributorPlan, constantNumPackets, commOnHost, prefix, canTryAliasing, CM); } // if (needCommunication) } // if (CM != ZERO) } @@ -1321,7 +1321,7 @@ namespace Tpetra { } } else { - doWaits(distributorPlan); + distributorActor_.doWaits(distributorPlan); if (verbose) { std::ostringstream os; @@ -1351,7 +1351,6 @@ namespace Tpetra { doPosts(const Details::DistributorPlan& distributorPlan, size_t constantNumPackets, bool commOnHost, - ReverseOption revOp, std::shared_ptr prefix, const bool canTryAliasing, const CombineMode CM) @@ -1387,17 +1386,11 @@ namespace Tpetra { // MPI communication happens here. if (verbose) { std::ostringstream os; - os << *prefix << "Call do" - << (revOp == DoReverse ? "Reverse" : "") << "PostsAndWaits" + os << *prefix << "Call doPostsAndWaits" << endl; std::cerr << os.str (); } - if (revOp == DoReverse) { - distributorActor_.doPostsAndWaits(*distributorPlan.getReversePlan(), numExp_h, 1, numImp_h); - } - else { - distributorActor_.doPostsAndWaits(distributorPlan, numExp_h, 1, numImp_h); - } + distributorActor_.doPostsAndWaits(distributorPlan, numExp_h, 1, numImp_h); if (verbose) { std::ostringstream os; @@ -1418,17 +1411,11 @@ namespace Tpetra { // MPI communication happens here. if (verbose) { std::ostringstream os; - os << *prefix << "Call do" - << (revOp == DoReverse ? "Reverse" : "") << "PostsAndWaits" + os << *prefix << "Call doPostsAndWaits" << endl; std::cerr << os.str (); } - if (revOp == DoReverse) { - distributorActor_.doPostsAndWaits(*distributorPlan.getReversePlan(), numExp_d, 1, numImp_d); - } - else { - distributorActor_.doPostsAndWaits(distributorPlan, numExp_d, 1, numImp_d); - } + distributorActor_.doPostsAndWaits(distributorPlan, numExp_d, 1, numImp_d); if (verbose) { std::ostringstream os; @@ -1481,49 +1468,28 @@ namespace Tpetra { std::ostringstream os; os << *prefix << "Comm on " << (commOnHost ? "host" : "device") - << "; call do" << (revOp == DoReverse ? "Reverse" : "") - << "PostsAndWaits" << endl; + << "; call doPosts" << endl; std::cerr << os.str (); } if (commOnHost) { this->imports_.modify_host (); - if (revOp == DoReverse) { - distributorActor_.doPosts - (*distributorPlan.getReversePlan(), - create_const_view (this->exports_.view_host ()), - numExportPacketsPerLID_av, - this->imports_.view_host (), - numImportPacketsPerLID_av); - } - else { - distributorActor_.doPosts - (distributorPlan, - create_const_view (this->exports_.view_host ()), - numExportPacketsPerLID_av, - this->imports_.view_host (), - numImportPacketsPerLID_av); - } + distributorActor_.doPosts + (distributorPlan, + create_const_view (this->exports_.view_host ()), + numExportPacketsPerLID_av, + this->imports_.view_host (), + numImportPacketsPerLID_av); } else { // pack on device Kokkos::fence(); // for UVM this->imports_.modify_device (); - if (revOp == DoReverse) { - distributorActor_.doPosts - (*distributorPlan.getReversePlan(), - create_const_view (this->exports_.view_device ()), - numExportPacketsPerLID_av, - this->imports_.view_device (), - numImportPacketsPerLID_av); - } - else { - distributorActor_.doPosts - (distributorPlan, - create_const_view (this->exports_.view_device ()), - numExportPacketsPerLID_av, - this->imports_.view_device (), - numImportPacketsPerLID_av); - } + distributorActor_.doPosts + (distributorPlan, + create_const_view (this->exports_.view_device ()), + numExportPacketsPerLID_av, + this->imports_.view_device (), + numImportPacketsPerLID_av); } } else { // constant number of packets per LID @@ -1549,56 +1515,29 @@ namespace Tpetra { std::ostringstream os; os << *prefix << "7.2. Comm on " << (commOnHost ? "host" : "device") - << "; call do" << (revOp == DoReverse ? "Reverse" : "") - << "PostsAndWaits" << endl; + << "; call doPosts" << endl; std::cerr << os.str (); } if (commOnHost) { this->imports_.modify_host (); - if (revOp == DoReverse) { - distributorActor_.doPosts - (*distributorPlan.getReversePlan(), - create_const_view (this->exports_.view_host ()), - constantNumPackets, - this->imports_.view_host ()); - } - else { - distributorActor_.doPosts - (distributorPlan, - create_const_view (this->exports_.view_host ()), - constantNumPackets, - this->imports_.view_host ()); - } + distributorActor_.doPosts + (distributorPlan, + create_const_view (this->exports_.view_host ()), + constantNumPackets, + this->imports_.view_host ()); } else { // pack on device Kokkos::fence(); // for UVM this->imports_.modify_device (); - if (revOp == DoReverse) { - distributorActor_.doPosts - (*distributorPlan.getReversePlan(), - create_const_view (this->exports_.view_device ()), - constantNumPackets, - this->imports_.view_device ()); - } - else { - distributorActor_.doPosts - (distributorPlan, - create_const_view (this->exports_.view_device ()), - constantNumPackets, - this->imports_.view_device ()); - } + distributorActor_.doPosts + (distributorPlan, + create_const_view (this->exports_.view_device ()), + constantNumPackets, + this->imports_.view_device ()); } // commOnHost } // constant or variable num packets per LID } - template - void - DistObject:: - doWaits(const Details::DistributorPlan& distributorPlan) - { - distributorActor_.doWaits(distributorPlan); - } - template void DistObject:: From 8d4b75c3d84552569d7725fc040439e12827f619 Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Mon, 23 Aug 2021 14:06:43 -0600 Subject: [PATCH 19/70] Tpetra: Add unit test for CrsGraphTransposer --- packages/tpetra/core/test/CMakeLists.txt | 1 + .../test/CrsGraphTransposer/CMakeLists.txt | 27 + .../CrsGraphTransposer_UnitTests.cpp | 128 + .../core/test/CrsGraphTransposer/L+LT.mtx | 10003 ++++++++++++++++ .../tpetra/core/test/CrsGraphTransposer/L.mtx | 5053 ++++++++ .../core/test/CrsGraphTransposer/LT.mtx | 5053 ++++++++ 6 files changed, 20265 insertions(+) create mode 100644 packages/tpetra/core/test/CrsGraphTransposer/CMakeLists.txt create mode 100644 packages/tpetra/core/test/CrsGraphTransposer/CrsGraphTransposer_UnitTests.cpp create mode 100644 packages/tpetra/core/test/CrsGraphTransposer/L+LT.mtx create mode 100644 packages/tpetra/core/test/CrsGraphTransposer/L.mtx create mode 100644 packages/tpetra/core/test/CrsGraphTransposer/LT.mtx diff --git a/packages/tpetra/core/test/CMakeLists.txt b/packages/tpetra/core/test/CMakeLists.txt index b44e99526514..3d4c2c29a944 100644 --- a/packages/tpetra/core/test/CMakeLists.txt +++ b/packages/tpetra/core/test/CMakeLists.txt @@ -11,6 +11,7 @@ ADD_SUBDIRECTORIES( Comm Core CrsGraph + CrsGraphTransposer CrsMatrix Directory Distributor diff --git a/packages/tpetra/core/test/CrsGraphTransposer/CMakeLists.txt b/packages/tpetra/core/test/CrsGraphTransposer/CMakeLists.txt new file mode 100644 index 000000000000..17e393afd8b4 --- /dev/null +++ b/packages/tpetra/core/test/CrsGraphTransposer/CMakeLists.txt @@ -0,0 +1,27 @@ + +# TRIBITS_ADD_EXECUTABLE_AND_TEST( +# RowMatrixTransposer_test +# SOURCES +# main.cpp +# ) + +TRIBITS_ADD_EXECUTABLE_AND_TEST( + CrsGraphTransposer_UnitTests + SOURCES + CrsGraphTransposer_UnitTests + ${TEUCHOS_STD_UNIT_TEST_MAIN} + COMM serial mpi + STANDARD_PASS_OUTPUT + ) + +# TRIBITS_ADD_EXECUTABLE_AND_TEST( +# CrsMatrix_transpose_sortedRows +# SOURCES sorted.cpp +# COMM serial mpi +# STANDARD_PASS_OUTPUT +# ) + +TRIBITS_COPY_FILES_TO_BINARY_DIR(TransposeGraphFileCopies + SOURCE_FILES L.mtx LT.mtx L+LT.mtx + EXEDEPS CrsGraphTransposer_UnitTests + ) diff --git a/packages/tpetra/core/test/CrsGraphTransposer/CrsGraphTransposer_UnitTests.cpp b/packages/tpetra/core/test/CrsGraphTransposer/CrsGraphTransposer_UnitTests.cpp new file mode 100644 index 000000000000..2393ae8aeff8 --- /dev/null +++ b/packages/tpetra/core/test/CrsGraphTransposer/CrsGraphTransposer_UnitTests.cpp @@ -0,0 +1,128 @@ +/* +// @HEADER +// *********************************************************************** +// +// Tpetra: Templated Linear Algebra Services Package +// Copyright (2008) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Michael A. Heroux (maherou@sandia.gov) +// +// ************************************************************************ +// @HEADER +*/ + +#include +#include +#include +#include +#include +#include + +namespace { + + using Teuchos::RCP; + using Teuchos::rcp; + using Teuchos::ScalarTraits; + + using Tpetra::CrsMatrix; + using Tpetra::CrsGraph; + using Tpetra::createUniformContigMapWithNode; + using Tpetra::MatrixMarket::Reader; + using Tpetra::CrsGraphTransposer; + + TEUCHOS_STATIC_SETUP() + { + Teuchos::CommandLineProcessor &clp = Teuchos::UnitTestRepository::getCLP(); + clp.addOutputSetupOptions(true); + clp.setOption( + "test-mpi", "test-serial", &Tpetra::TestingUtilities::testMpi, + "Test MPI (if available) or force test of serial. In a serial build," + " this option is ignored and a serial comm is always used." ); + } + + TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL( CrsGraphTransposer, RectangularTranspose, LO, GO, Node ) + { + typedef CrsMatrix<>::scalar_type Scalar; + typedef CrsMatrix MAT; + typedef CrsGraph GRAPH; + auto comm = Tpetra::getDefaultComm(); + + RCP matrix = Reader::readSparseFile("../RowMatrixTransposer/a.mtx", comm); + RCP matrixT = Reader::readSparseFile("../RowMatrixTransposer/atrans.mtx", comm); + + RCP graph = matrix->getCrsGraph(); + RCP graphT = matrixT->getCrsGraph(); + + CrsGraphTransposer gt (graph); + RCP calculatedT = gt.createTranspose(); + + TEUCHOS_ASSERT(calculatedT->isIdenticalTo(*graphT)); + } + + TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL( CrsGraphTransposer, Symmetrization, LO, GO, Node ) + { + typedef CrsMatrix<>::scalar_type Scalar; + typedef CrsMatrix MAT; + typedef CrsGraph GRAPH; + auto comm = Tpetra::getDefaultComm(); + + RCP matrix = Reader::readSparseFile("L.mtx", comm); + RCP matrixT = Reader::readSparseFile("LT.mtx", comm); + RCP matrixSym = Reader::readSparseFile("L+LT.mtx", comm); + + RCP graph = matrix->getCrsGraph(); + RCP graphT = matrixT->getCrsGraph(); + RCP graphSym = matrixSym->getCrsGraph(); + + CrsGraphTransposer gt (graph); + RCP calculatedT = gt.createTranspose(); + + TEUCHOS_ASSERT(calculatedT->isIdenticalTo(*graphT)); + + RCP calculatedSym = gt.symmetrize(); + + Tpetra::MatrixMarket::Writer::writeSparseGraphFile("test",*calculatedSym); + + TEUCHOS_ASSERT(calculatedSym->isIdenticalTo(*graphSym)); + } + + +#define UNIT_TEST_GROUP( LO, GO, NODE ) \ + TEUCHOS_UNIT_TEST_TEMPLATE_3_INSTANT( CrsGraphTransposer, RectangularTranspose, LO, GO, NODE ) \ + TEUCHOS_UNIT_TEST_TEMPLATE_3_INSTANT( CrsGraphTransposer, Symmetrization, LO, GO, NODE ) + + TPETRA_ETI_MANGLING_TYPEDEFS() + + TPETRA_INSTANTIATE_LGN( UNIT_TEST_GROUP ) + +} diff --git a/packages/tpetra/core/test/CrsGraphTransposer/L+LT.mtx b/packages/tpetra/core/test/CrsGraphTransposer/L+LT.mtx new file mode 100644 index 000000000000..8b87a0a18e85 --- /dev/null +++ b/packages/tpetra/core/test/CrsGraphTransposer/L+LT.mtx @@ -0,0 +1,10003 @@ +%%MatrixMarket matrix coordinate real general +% +100 100 10000 +1 1 2.000000000000000e+00 +1 2 2.000000000000000e+00 +1 3 3.000000000000000e+00 +1 4 4.000000000000000e+00 +1 5 5.000000000000000e+00 +1 6 6.000000000000000e+00 +1 7 7.000000000000000e+00 +1 8 8.000000000000000e+00 +1 9 9.000000000000000e+00 +1 10 1.000000000000000e+01 +1 11 1.100000000000000e+01 +1 12 1.200000000000000e+01 +1 13 1.300000000000000e+01 +1 14 1.400000000000000e+01 +1 15 1.500000000000000e+01 +1 16 1.600000000000000e+01 +1 17 1.700000000000000e+01 +1 18 1.800000000000000e+01 +1 19 1.900000000000000e+01 +1 20 2.000000000000000e+01 +1 21 2.100000000000000e+01 +1 22 2.200000000000000e+01 +1 23 2.300000000000000e+01 +1 24 2.400000000000000e+01 +1 25 2.500000000000000e+01 +1 26 2.600000000000000e+01 +1 27 2.700000000000000e+01 +1 28 2.800000000000000e+01 +1 29 2.900000000000000e+01 +1 30 3.000000000000000e+01 +1 31 3.100000000000000e+01 +1 32 3.200000000000000e+01 +1 33 3.300000000000000e+01 +1 34 3.400000000000000e+01 +1 35 3.500000000000000e+01 +1 36 3.600000000000000e+01 +1 37 3.700000000000000e+01 +1 38 3.800000000000000e+01 +1 39 3.900000000000000e+01 +1 40 4.000000000000000e+01 +1 41 4.100000000000000e+01 +1 42 4.200000000000000e+01 +1 43 4.300000000000000e+01 +1 44 4.400000000000000e+01 +1 45 4.500000000000000e+01 +1 46 4.600000000000000e+01 +1 47 4.700000000000000e+01 +1 48 4.800000000000000e+01 +1 49 4.900000000000000e+01 +1 50 5.000000000000000e+01 +1 51 5.100000000000000e+01 +1 52 5.200000000000000e+01 +1 53 5.300000000000000e+01 +1 54 5.400000000000000e+01 +1 55 5.500000000000000e+01 +1 56 5.600000000000000e+01 +1 57 5.700000000000000e+01 +1 58 5.800000000000000e+01 +1 59 5.900000000000000e+01 +1 60 6.000000000000000e+01 +1 61 6.100000000000000e+01 +1 62 6.200000000000000e+01 +1 63 6.300000000000000e+01 +1 64 6.400000000000000e+01 +1 65 6.500000000000000e+01 +1 66 6.600000000000000e+01 +1 67 6.700000000000000e+01 +1 68 6.800000000000000e+01 +1 69 6.900000000000000e+01 +1 70 7.000000000000000e+01 +1 71 7.100000000000000e+01 +1 72 7.200000000000000e+01 +1 73 7.300000000000000e+01 +1 74 7.400000000000000e+01 +1 75 7.500000000000000e+01 +1 76 7.600000000000000e+01 +1 77 7.700000000000000e+01 +1 78 7.800000000000000e+01 +1 79 7.900000000000000e+01 +1 80 8.000000000000000e+01 +1 81 8.100000000000000e+01 +1 82 8.200000000000000e+01 +1 83 8.300000000000000e+01 +1 84 8.400000000000000e+01 +1 85 8.500000000000000e+01 +1 86 8.600000000000000e+01 +1 87 8.700000000000000e+01 +1 88 8.800000000000000e+01 +1 89 8.900000000000000e+01 +1 90 9.000000000000000e+01 +1 91 9.100000000000000e+01 +1 92 9.200000000000000e+01 +1 93 9.300000000000000e+01 +1 94 9.400000000000000e+01 +1 95 9.500000000000000e+01 +1 96 9.600000000000000e+01 +1 97 9.700000000000000e+01 +1 98 9.800000000000000e+01 +1 99 9.900000000000000e+01 +1 100 1.000000000000000e+02 +2 1 2.000000000000000e+00 +2 2 2.000000000000000e+00 +2 3 2.000000000000000e+00 +2 4 3.000000000000000e+00 +2 5 4.000000000000000e+00 +2 6 5.000000000000000e+00 +2 7 6.000000000000000e+00 +2 8 7.000000000000000e+00 +2 9 8.000000000000000e+00 +2 10 9.000000000000000e+00 +2 11 1.000000000000000e+01 +2 12 1.100000000000000e+01 +2 13 1.200000000000000e+01 +2 14 1.300000000000000e+01 +2 15 1.400000000000000e+01 +2 16 1.500000000000000e+01 +2 17 1.600000000000000e+01 +2 18 1.700000000000000e+01 +2 19 1.800000000000000e+01 +2 20 1.900000000000000e+01 +2 21 2.000000000000000e+01 +2 22 2.100000000000000e+01 +2 23 2.200000000000000e+01 +2 24 2.300000000000000e+01 +2 25 2.400000000000000e+01 +2 26 2.500000000000000e+01 +2 27 2.600000000000000e+01 +2 28 2.700000000000000e+01 +2 29 2.800000000000000e+01 +2 30 2.900000000000000e+01 +2 31 3.000000000000000e+01 +2 32 3.100000000000000e+01 +2 33 3.200000000000000e+01 +2 34 3.300000000000000e+01 +2 35 3.400000000000000e+01 +2 36 3.500000000000000e+01 +2 37 3.600000000000000e+01 +2 38 3.700000000000000e+01 +2 39 3.800000000000000e+01 +2 40 3.900000000000000e+01 +2 41 4.000000000000000e+01 +2 42 4.100000000000000e+01 +2 43 4.200000000000000e+01 +2 44 4.300000000000000e+01 +2 45 4.400000000000000e+01 +2 46 4.500000000000000e+01 +2 47 4.600000000000000e+01 +2 48 4.700000000000000e+01 +2 49 4.800000000000000e+01 +2 50 4.900000000000000e+01 +2 51 5.000000000000000e+01 +2 52 5.100000000000000e+01 +2 53 5.200000000000000e+01 +2 54 5.300000000000000e+01 +2 55 5.400000000000000e+01 +2 56 5.500000000000000e+01 +2 57 5.600000000000000e+01 +2 58 5.700000000000000e+01 +2 59 5.800000000000000e+01 +2 60 5.900000000000000e+01 +2 61 6.000000000000000e+01 +2 62 6.100000000000000e+01 +2 63 6.200000000000000e+01 +2 64 6.300000000000000e+01 +2 65 6.400000000000000e+01 +2 66 6.500000000000000e+01 +2 67 6.600000000000000e+01 +2 68 6.700000000000000e+01 +2 69 6.800000000000000e+01 +2 70 6.900000000000000e+01 +2 71 7.000000000000000e+01 +2 72 7.100000000000000e+01 +2 73 7.200000000000000e+01 +2 74 7.300000000000000e+01 +2 75 7.400000000000000e+01 +2 76 7.500000000000000e+01 +2 77 7.600000000000000e+01 +2 78 7.700000000000000e+01 +2 79 7.800000000000000e+01 +2 80 7.900000000000000e+01 +2 81 8.000000000000000e+01 +2 82 8.100000000000000e+01 +2 83 8.200000000000000e+01 +2 84 8.300000000000000e+01 +2 85 8.400000000000000e+01 +2 86 8.500000000000000e+01 +2 87 8.600000000000000e+01 +2 88 8.700000000000000e+01 +2 89 8.800000000000000e+01 +2 90 8.900000000000000e+01 +2 91 9.000000000000000e+01 +2 92 9.100000000000000e+01 +2 93 9.200000000000000e+01 +2 94 9.300000000000000e+01 +2 95 9.400000000000000e+01 +2 96 9.500000000000000e+01 +2 97 9.600000000000000e+01 +2 98 9.700000000000000e+01 +2 99 9.800000000000000e+01 +2 100 9.900000000000000e+01 +3 1 3.000000000000000e+00 +3 2 2.000000000000000e+00 +3 3 2.000000000000000e+00 +3 4 2.000000000000000e+00 +3 5 3.000000000000000e+00 +3 6 4.000000000000000e+00 +3 7 5.000000000000000e+00 +3 8 6.000000000000000e+00 +3 9 7.000000000000000e+00 +3 10 8.000000000000000e+00 +3 11 9.000000000000000e+00 +3 12 1.000000000000000e+01 +3 13 1.100000000000000e+01 +3 14 1.200000000000000e+01 +3 15 1.300000000000000e+01 +3 16 1.400000000000000e+01 +3 17 1.500000000000000e+01 +3 18 1.600000000000000e+01 +3 19 1.700000000000000e+01 +3 20 1.800000000000000e+01 +3 21 1.900000000000000e+01 +3 22 2.000000000000000e+01 +3 23 2.100000000000000e+01 +3 24 2.200000000000000e+01 +3 25 2.300000000000000e+01 +3 26 2.400000000000000e+01 +3 27 2.500000000000000e+01 +3 28 2.600000000000000e+01 +3 29 2.700000000000000e+01 +3 30 2.800000000000000e+01 +3 31 2.900000000000000e+01 +3 32 3.000000000000000e+01 +3 33 3.100000000000000e+01 +3 34 3.200000000000000e+01 +3 35 3.300000000000000e+01 +3 36 3.400000000000000e+01 +3 37 3.500000000000000e+01 +3 38 3.600000000000000e+01 +3 39 3.700000000000000e+01 +3 40 3.800000000000000e+01 +3 41 3.900000000000000e+01 +3 42 4.000000000000000e+01 +3 43 4.100000000000000e+01 +3 44 4.200000000000000e+01 +3 45 4.300000000000000e+01 +3 46 4.400000000000000e+01 +3 47 4.500000000000000e+01 +3 48 4.600000000000000e+01 +3 49 4.700000000000000e+01 +3 50 4.800000000000000e+01 +3 51 4.900000000000000e+01 +3 52 5.000000000000000e+01 +3 53 5.100000000000000e+01 +3 54 5.200000000000000e+01 +3 55 5.300000000000000e+01 +3 56 5.400000000000000e+01 +3 57 5.500000000000000e+01 +3 58 5.600000000000000e+01 +3 59 5.700000000000000e+01 +3 60 5.800000000000000e+01 +3 61 5.900000000000000e+01 +3 62 6.000000000000000e+01 +3 63 6.100000000000000e+01 +3 64 6.200000000000000e+01 +3 65 6.300000000000000e+01 +3 66 6.400000000000000e+01 +3 67 6.500000000000000e+01 +3 68 6.600000000000000e+01 +3 69 6.700000000000000e+01 +3 70 6.800000000000000e+01 +3 71 6.900000000000000e+01 +3 72 7.000000000000000e+01 +3 73 7.100000000000000e+01 +3 74 7.200000000000000e+01 +3 75 7.300000000000000e+01 +3 76 7.400000000000000e+01 +3 77 7.500000000000000e+01 +3 78 7.600000000000000e+01 +3 79 7.700000000000000e+01 +3 80 7.800000000000000e+01 +3 81 7.900000000000000e+01 +3 82 8.000000000000000e+01 +3 83 8.100000000000000e+01 +3 84 8.200000000000000e+01 +3 85 8.300000000000000e+01 +3 86 8.400000000000000e+01 +3 87 8.500000000000000e+01 +3 88 8.600000000000000e+01 +3 89 8.700000000000000e+01 +3 90 8.800000000000000e+01 +3 91 8.900000000000000e+01 +3 92 9.000000000000000e+01 +3 93 9.100000000000000e+01 +3 94 9.200000000000000e+01 +3 95 9.300000000000000e+01 +3 96 9.400000000000000e+01 +3 97 9.500000000000000e+01 +3 98 9.600000000000000e+01 +3 99 9.700000000000000e+01 +3 100 9.800000000000000e+01 +4 1 4.000000000000000e+00 +4 2 3.000000000000000e+00 +4 3 2.000000000000000e+00 +4 4 2.000000000000000e+00 +4 5 2.000000000000000e+00 +4 6 3.000000000000000e+00 +4 7 4.000000000000000e+00 +4 8 5.000000000000000e+00 +4 9 6.000000000000000e+00 +4 10 7.000000000000000e+00 +4 11 8.000000000000000e+00 +4 12 9.000000000000000e+00 +4 13 1.000000000000000e+01 +4 14 1.100000000000000e+01 +4 15 1.200000000000000e+01 +4 16 1.300000000000000e+01 +4 17 1.400000000000000e+01 +4 18 1.500000000000000e+01 +4 19 1.600000000000000e+01 +4 20 1.700000000000000e+01 +4 21 1.800000000000000e+01 +4 22 1.900000000000000e+01 +4 23 2.000000000000000e+01 +4 24 2.100000000000000e+01 +4 25 2.200000000000000e+01 +4 26 2.300000000000000e+01 +4 27 2.400000000000000e+01 +4 28 2.500000000000000e+01 +4 29 2.600000000000000e+01 +4 30 2.700000000000000e+01 +4 31 2.800000000000000e+01 +4 32 2.900000000000000e+01 +4 33 3.000000000000000e+01 +4 34 3.100000000000000e+01 +4 35 3.200000000000000e+01 +4 36 3.300000000000000e+01 +4 37 3.400000000000000e+01 +4 38 3.500000000000000e+01 +4 39 3.600000000000000e+01 +4 40 3.700000000000000e+01 +4 41 3.800000000000000e+01 +4 42 3.900000000000000e+01 +4 43 4.000000000000000e+01 +4 44 4.100000000000000e+01 +4 45 4.200000000000000e+01 +4 46 4.300000000000000e+01 +4 47 4.400000000000000e+01 +4 48 4.500000000000000e+01 +4 49 4.600000000000000e+01 +4 50 4.700000000000000e+01 +4 51 4.800000000000000e+01 +4 52 4.900000000000000e+01 +4 53 5.000000000000000e+01 +4 54 5.100000000000000e+01 +4 55 5.200000000000000e+01 +4 56 5.300000000000000e+01 +4 57 5.400000000000000e+01 +4 58 5.500000000000000e+01 +4 59 5.600000000000000e+01 +4 60 5.700000000000000e+01 +4 61 5.800000000000000e+01 +4 62 5.900000000000000e+01 +4 63 6.000000000000000e+01 +4 64 6.100000000000000e+01 +4 65 6.200000000000000e+01 +4 66 6.300000000000000e+01 +4 67 6.400000000000000e+01 +4 68 6.500000000000000e+01 +4 69 6.600000000000000e+01 +4 70 6.700000000000000e+01 +4 71 6.800000000000000e+01 +4 72 6.900000000000000e+01 +4 73 7.000000000000000e+01 +4 74 7.100000000000000e+01 +4 75 7.200000000000000e+01 +4 76 7.300000000000000e+01 +4 77 7.400000000000000e+01 +4 78 7.500000000000000e+01 +4 79 7.600000000000000e+01 +4 80 7.700000000000000e+01 +4 81 7.800000000000000e+01 +4 82 7.900000000000000e+01 +4 83 8.000000000000000e+01 +4 84 8.100000000000000e+01 +4 85 8.200000000000000e+01 +4 86 8.300000000000000e+01 +4 87 8.400000000000000e+01 +4 88 8.500000000000000e+01 +4 89 8.600000000000000e+01 +4 90 8.700000000000000e+01 +4 91 8.800000000000000e+01 +4 92 8.900000000000000e+01 +4 93 9.000000000000000e+01 +4 94 9.100000000000000e+01 +4 95 9.200000000000000e+01 +4 96 9.300000000000000e+01 +4 97 9.400000000000000e+01 +4 98 9.500000000000000e+01 +4 99 9.600000000000000e+01 +4 100 9.700000000000000e+01 +5 1 5.000000000000000e+00 +5 2 4.000000000000000e+00 +5 3 3.000000000000000e+00 +5 4 2.000000000000000e+00 +5 5 2.000000000000000e+00 +5 6 2.000000000000000e+00 +5 7 3.000000000000000e+00 +5 8 4.000000000000000e+00 +5 9 5.000000000000000e+00 +5 10 6.000000000000000e+00 +5 11 7.000000000000000e+00 +5 12 8.000000000000000e+00 +5 13 9.000000000000000e+00 +5 14 1.000000000000000e+01 +5 15 1.100000000000000e+01 +5 16 1.200000000000000e+01 +5 17 1.300000000000000e+01 +5 18 1.400000000000000e+01 +5 19 1.500000000000000e+01 +5 20 1.600000000000000e+01 +5 21 1.700000000000000e+01 +5 22 1.800000000000000e+01 +5 23 1.900000000000000e+01 +5 24 2.000000000000000e+01 +5 25 2.100000000000000e+01 +5 26 2.200000000000000e+01 +5 27 2.300000000000000e+01 +5 28 2.400000000000000e+01 +5 29 2.500000000000000e+01 +5 30 2.600000000000000e+01 +5 31 2.700000000000000e+01 +5 32 2.800000000000000e+01 +5 33 2.900000000000000e+01 +5 34 3.000000000000000e+01 +5 35 3.100000000000000e+01 +5 36 3.200000000000000e+01 +5 37 3.300000000000000e+01 +5 38 3.400000000000000e+01 +5 39 3.500000000000000e+01 +5 40 3.600000000000000e+01 +5 41 3.700000000000000e+01 +5 42 3.800000000000000e+01 +5 43 3.900000000000000e+01 +5 44 4.000000000000000e+01 +5 45 4.100000000000000e+01 +5 46 4.200000000000000e+01 +5 47 4.300000000000000e+01 +5 48 4.400000000000000e+01 +5 49 4.500000000000000e+01 +5 50 4.600000000000000e+01 +5 51 4.700000000000000e+01 +5 52 4.800000000000000e+01 +5 53 4.900000000000000e+01 +5 54 5.000000000000000e+01 +5 55 5.100000000000000e+01 +5 56 5.200000000000000e+01 +5 57 5.300000000000000e+01 +5 58 5.400000000000000e+01 +5 59 5.500000000000000e+01 +5 60 5.600000000000000e+01 +5 61 5.700000000000000e+01 +5 62 5.800000000000000e+01 +5 63 5.900000000000000e+01 +5 64 6.000000000000000e+01 +5 65 6.100000000000000e+01 +5 66 6.200000000000000e+01 +5 67 6.300000000000000e+01 +5 68 6.400000000000000e+01 +5 69 6.500000000000000e+01 +5 70 6.600000000000000e+01 +5 71 6.700000000000000e+01 +5 72 6.800000000000000e+01 +5 73 6.900000000000000e+01 +5 74 7.000000000000000e+01 +5 75 7.100000000000000e+01 +5 76 7.200000000000000e+01 +5 77 7.300000000000000e+01 +5 78 7.400000000000000e+01 +5 79 7.500000000000000e+01 +5 80 7.600000000000000e+01 +5 81 7.700000000000000e+01 +5 82 7.800000000000000e+01 +5 83 7.900000000000000e+01 +5 84 8.000000000000000e+01 +5 85 8.100000000000000e+01 +5 86 8.200000000000000e+01 +5 87 8.300000000000000e+01 +5 88 8.400000000000000e+01 +5 89 8.500000000000000e+01 +5 90 8.600000000000000e+01 +5 91 8.700000000000000e+01 +5 92 8.800000000000000e+01 +5 93 8.900000000000000e+01 +5 94 9.000000000000000e+01 +5 95 9.100000000000000e+01 +5 96 9.200000000000000e+01 +5 97 9.300000000000000e+01 +5 98 9.400000000000000e+01 +5 99 9.500000000000000e+01 +5 100 9.600000000000000e+01 +6 1 6.000000000000000e+00 +6 2 5.000000000000000e+00 +6 3 4.000000000000000e+00 +6 4 3.000000000000000e+00 +6 5 2.000000000000000e+00 +6 6 2.000000000000000e+00 +6 7 2.000000000000000e+00 +6 8 3.000000000000000e+00 +6 9 4.000000000000000e+00 +6 10 5.000000000000000e+00 +6 11 6.000000000000000e+00 +6 12 7.000000000000000e+00 +6 13 8.000000000000000e+00 +6 14 9.000000000000000e+00 +6 15 1.000000000000000e+01 +6 16 1.100000000000000e+01 +6 17 1.200000000000000e+01 +6 18 1.300000000000000e+01 +6 19 1.400000000000000e+01 +6 20 1.500000000000000e+01 +6 21 1.600000000000000e+01 +6 22 1.700000000000000e+01 +6 23 1.800000000000000e+01 +6 24 1.900000000000000e+01 +6 25 2.000000000000000e+01 +6 26 2.100000000000000e+01 +6 27 2.200000000000000e+01 +6 28 2.300000000000000e+01 +6 29 2.400000000000000e+01 +6 30 2.500000000000000e+01 +6 31 2.600000000000000e+01 +6 32 2.700000000000000e+01 +6 33 2.800000000000000e+01 +6 34 2.900000000000000e+01 +6 35 3.000000000000000e+01 +6 36 3.100000000000000e+01 +6 37 3.200000000000000e+01 +6 38 3.300000000000000e+01 +6 39 3.400000000000000e+01 +6 40 3.500000000000000e+01 +6 41 3.600000000000000e+01 +6 42 3.700000000000000e+01 +6 43 3.800000000000000e+01 +6 44 3.900000000000000e+01 +6 45 4.000000000000000e+01 +6 46 4.100000000000000e+01 +6 47 4.200000000000000e+01 +6 48 4.300000000000000e+01 +6 49 4.400000000000000e+01 +6 50 4.500000000000000e+01 +6 51 4.600000000000000e+01 +6 52 4.700000000000000e+01 +6 53 4.800000000000000e+01 +6 54 4.900000000000000e+01 +6 55 5.000000000000000e+01 +6 56 5.100000000000000e+01 +6 57 5.200000000000000e+01 +6 58 5.300000000000000e+01 +6 59 5.400000000000000e+01 +6 60 5.500000000000000e+01 +6 61 5.600000000000000e+01 +6 62 5.700000000000000e+01 +6 63 5.800000000000000e+01 +6 64 5.900000000000000e+01 +6 65 6.000000000000000e+01 +6 66 6.100000000000000e+01 +6 67 6.200000000000000e+01 +6 68 6.300000000000000e+01 +6 69 6.400000000000000e+01 +6 70 6.500000000000000e+01 +6 71 6.600000000000000e+01 +6 72 6.700000000000000e+01 +6 73 6.800000000000000e+01 +6 74 6.900000000000000e+01 +6 75 7.000000000000000e+01 +6 76 7.100000000000000e+01 +6 77 7.200000000000000e+01 +6 78 7.300000000000000e+01 +6 79 7.400000000000000e+01 +6 80 7.500000000000000e+01 +6 81 7.600000000000000e+01 +6 82 7.700000000000000e+01 +6 83 7.800000000000000e+01 +6 84 7.900000000000000e+01 +6 85 8.000000000000000e+01 +6 86 8.100000000000000e+01 +6 87 8.200000000000000e+01 +6 88 8.300000000000000e+01 +6 89 8.400000000000000e+01 +6 90 8.500000000000000e+01 +6 91 8.600000000000000e+01 +6 92 8.700000000000000e+01 +6 93 8.800000000000000e+01 +6 94 8.900000000000000e+01 +6 95 9.000000000000000e+01 +6 96 9.100000000000000e+01 +6 97 9.200000000000000e+01 +6 98 9.300000000000000e+01 +6 99 9.400000000000000e+01 +6 100 9.500000000000000e+01 +7 1 7.000000000000000e+00 +7 2 6.000000000000000e+00 +7 3 5.000000000000000e+00 +7 4 4.000000000000000e+00 +7 5 3.000000000000000e+00 +7 6 2.000000000000000e+00 +7 7 2.000000000000000e+00 +7 8 2.000000000000000e+00 +7 9 3.000000000000000e+00 +7 10 4.000000000000000e+00 +7 11 5.000000000000000e+00 +7 12 6.000000000000000e+00 +7 13 7.000000000000000e+00 +7 14 8.000000000000000e+00 +7 15 9.000000000000000e+00 +7 16 1.000000000000000e+01 +7 17 1.100000000000000e+01 +7 18 1.200000000000000e+01 +7 19 1.300000000000000e+01 +7 20 1.400000000000000e+01 +7 21 1.500000000000000e+01 +7 22 1.600000000000000e+01 +7 23 1.700000000000000e+01 +7 24 1.800000000000000e+01 +7 25 1.900000000000000e+01 +7 26 2.000000000000000e+01 +7 27 2.100000000000000e+01 +7 28 2.200000000000000e+01 +7 29 2.300000000000000e+01 +7 30 2.400000000000000e+01 +7 31 2.500000000000000e+01 +7 32 2.600000000000000e+01 +7 33 2.700000000000000e+01 +7 34 2.800000000000000e+01 +7 35 2.900000000000000e+01 +7 36 3.000000000000000e+01 +7 37 3.100000000000000e+01 +7 38 3.200000000000000e+01 +7 39 3.300000000000000e+01 +7 40 3.400000000000000e+01 +7 41 3.500000000000000e+01 +7 42 3.600000000000000e+01 +7 43 3.700000000000000e+01 +7 44 3.800000000000000e+01 +7 45 3.900000000000000e+01 +7 46 4.000000000000000e+01 +7 47 4.100000000000000e+01 +7 48 4.200000000000000e+01 +7 49 4.300000000000000e+01 +7 50 4.400000000000000e+01 +7 51 4.500000000000000e+01 +7 52 4.600000000000000e+01 +7 53 4.700000000000000e+01 +7 54 4.800000000000000e+01 +7 55 4.900000000000000e+01 +7 56 5.000000000000000e+01 +7 57 5.100000000000000e+01 +7 58 5.200000000000000e+01 +7 59 5.300000000000000e+01 +7 60 5.400000000000000e+01 +7 61 5.500000000000000e+01 +7 62 5.600000000000000e+01 +7 63 5.700000000000000e+01 +7 64 5.800000000000000e+01 +7 65 5.900000000000000e+01 +7 66 6.000000000000000e+01 +7 67 6.100000000000000e+01 +7 68 6.200000000000000e+01 +7 69 6.300000000000000e+01 +7 70 6.400000000000000e+01 +7 71 6.500000000000000e+01 +7 72 6.600000000000000e+01 +7 73 6.700000000000000e+01 +7 74 6.800000000000000e+01 +7 75 6.900000000000000e+01 +7 76 7.000000000000000e+01 +7 77 7.100000000000000e+01 +7 78 7.200000000000000e+01 +7 79 7.300000000000000e+01 +7 80 7.400000000000000e+01 +7 81 7.500000000000000e+01 +7 82 7.600000000000000e+01 +7 83 7.700000000000000e+01 +7 84 7.800000000000000e+01 +7 85 7.900000000000000e+01 +7 86 8.000000000000000e+01 +7 87 8.100000000000000e+01 +7 88 8.200000000000000e+01 +7 89 8.300000000000000e+01 +7 90 8.400000000000000e+01 +7 91 8.500000000000000e+01 +7 92 8.600000000000000e+01 +7 93 8.700000000000000e+01 +7 94 8.800000000000000e+01 +7 95 8.900000000000000e+01 +7 96 9.000000000000000e+01 +7 97 9.100000000000000e+01 +7 98 9.200000000000000e+01 +7 99 9.300000000000000e+01 +7 100 9.400000000000000e+01 +8 1 8.000000000000000e+00 +8 2 7.000000000000000e+00 +8 3 6.000000000000000e+00 +8 4 5.000000000000000e+00 +8 5 4.000000000000000e+00 +8 6 3.000000000000000e+00 +8 7 2.000000000000000e+00 +8 8 2.000000000000000e+00 +8 9 2.000000000000000e+00 +8 10 3.000000000000000e+00 +8 11 4.000000000000000e+00 +8 12 5.000000000000000e+00 +8 13 6.000000000000000e+00 +8 14 7.000000000000000e+00 +8 15 8.000000000000000e+00 +8 16 9.000000000000000e+00 +8 17 1.000000000000000e+01 +8 18 1.100000000000000e+01 +8 19 1.200000000000000e+01 +8 20 1.300000000000000e+01 +8 21 1.400000000000000e+01 +8 22 1.500000000000000e+01 +8 23 1.600000000000000e+01 +8 24 1.700000000000000e+01 +8 25 1.800000000000000e+01 +8 26 1.900000000000000e+01 +8 27 2.000000000000000e+01 +8 28 2.100000000000000e+01 +8 29 2.200000000000000e+01 +8 30 2.300000000000000e+01 +8 31 2.400000000000000e+01 +8 32 2.500000000000000e+01 +8 33 2.600000000000000e+01 +8 34 2.700000000000000e+01 +8 35 2.800000000000000e+01 +8 36 2.900000000000000e+01 +8 37 3.000000000000000e+01 +8 38 3.100000000000000e+01 +8 39 3.200000000000000e+01 +8 40 3.300000000000000e+01 +8 41 3.400000000000000e+01 +8 42 3.500000000000000e+01 +8 43 3.600000000000000e+01 +8 44 3.700000000000000e+01 +8 45 3.800000000000000e+01 +8 46 3.900000000000000e+01 +8 47 4.000000000000000e+01 +8 48 4.100000000000000e+01 +8 49 4.200000000000000e+01 +8 50 4.300000000000000e+01 +8 51 4.400000000000000e+01 +8 52 4.500000000000000e+01 +8 53 4.600000000000000e+01 +8 54 4.700000000000000e+01 +8 55 4.800000000000000e+01 +8 56 4.900000000000000e+01 +8 57 5.000000000000000e+01 +8 58 5.100000000000000e+01 +8 59 5.200000000000000e+01 +8 60 5.300000000000000e+01 +8 61 5.400000000000000e+01 +8 62 5.500000000000000e+01 +8 63 5.600000000000000e+01 +8 64 5.700000000000000e+01 +8 65 5.800000000000000e+01 +8 66 5.900000000000000e+01 +8 67 6.000000000000000e+01 +8 68 6.100000000000000e+01 +8 69 6.200000000000000e+01 +8 70 6.300000000000000e+01 +8 71 6.400000000000000e+01 +8 72 6.500000000000000e+01 +8 73 6.600000000000000e+01 +8 74 6.700000000000000e+01 +8 75 6.800000000000000e+01 +8 76 6.900000000000000e+01 +8 77 7.000000000000000e+01 +8 78 7.100000000000000e+01 +8 79 7.200000000000000e+01 +8 80 7.300000000000000e+01 +8 81 7.400000000000000e+01 +8 82 7.500000000000000e+01 +8 83 7.600000000000000e+01 +8 84 7.700000000000000e+01 +8 85 7.800000000000000e+01 +8 86 7.900000000000000e+01 +8 87 8.000000000000000e+01 +8 88 8.100000000000000e+01 +8 89 8.200000000000000e+01 +8 90 8.300000000000000e+01 +8 91 8.400000000000000e+01 +8 92 8.500000000000000e+01 +8 93 8.600000000000000e+01 +8 94 8.700000000000000e+01 +8 95 8.800000000000000e+01 +8 96 8.900000000000000e+01 +8 97 9.000000000000000e+01 +8 98 9.100000000000000e+01 +8 99 9.200000000000000e+01 +8 100 9.300000000000000e+01 +9 1 9.000000000000000e+00 +9 2 8.000000000000000e+00 +9 3 7.000000000000000e+00 +9 4 6.000000000000000e+00 +9 5 5.000000000000000e+00 +9 6 4.000000000000000e+00 +9 7 3.000000000000000e+00 +9 8 2.000000000000000e+00 +9 9 2.000000000000000e+00 +9 10 2.000000000000000e+00 +9 11 3.000000000000000e+00 +9 12 4.000000000000000e+00 +9 13 5.000000000000000e+00 +9 14 6.000000000000000e+00 +9 15 7.000000000000000e+00 +9 16 8.000000000000000e+00 +9 17 9.000000000000000e+00 +9 18 1.000000000000000e+01 +9 19 1.100000000000000e+01 +9 20 1.200000000000000e+01 +9 21 1.300000000000000e+01 +9 22 1.400000000000000e+01 +9 23 1.500000000000000e+01 +9 24 1.600000000000000e+01 +9 25 1.700000000000000e+01 +9 26 1.800000000000000e+01 +9 27 1.900000000000000e+01 +9 28 2.000000000000000e+01 +9 29 2.100000000000000e+01 +9 30 2.200000000000000e+01 +9 31 2.300000000000000e+01 +9 32 2.400000000000000e+01 +9 33 2.500000000000000e+01 +9 34 2.600000000000000e+01 +9 35 2.700000000000000e+01 +9 36 2.800000000000000e+01 +9 37 2.900000000000000e+01 +9 38 3.000000000000000e+01 +9 39 3.100000000000000e+01 +9 40 3.200000000000000e+01 +9 41 3.300000000000000e+01 +9 42 3.400000000000000e+01 +9 43 3.500000000000000e+01 +9 44 3.600000000000000e+01 +9 45 3.700000000000000e+01 +9 46 3.800000000000000e+01 +9 47 3.900000000000000e+01 +9 48 4.000000000000000e+01 +9 49 4.100000000000000e+01 +9 50 4.200000000000000e+01 +9 51 4.300000000000000e+01 +9 52 4.400000000000000e+01 +9 53 4.500000000000000e+01 +9 54 4.600000000000000e+01 +9 55 4.700000000000000e+01 +9 56 4.800000000000000e+01 +9 57 4.900000000000000e+01 +9 58 5.000000000000000e+01 +9 59 5.100000000000000e+01 +9 60 5.200000000000000e+01 +9 61 5.300000000000000e+01 +9 62 5.400000000000000e+01 +9 63 5.500000000000000e+01 +9 64 5.600000000000000e+01 +9 65 5.700000000000000e+01 +9 66 5.800000000000000e+01 +9 67 5.900000000000000e+01 +9 68 6.000000000000000e+01 +9 69 6.100000000000000e+01 +9 70 6.200000000000000e+01 +9 71 6.300000000000000e+01 +9 72 6.400000000000000e+01 +9 73 6.500000000000000e+01 +9 74 6.600000000000000e+01 +9 75 6.700000000000000e+01 +9 76 6.800000000000000e+01 +9 77 6.900000000000000e+01 +9 78 7.000000000000000e+01 +9 79 7.100000000000000e+01 +9 80 7.200000000000000e+01 +9 81 7.300000000000000e+01 +9 82 7.400000000000000e+01 +9 83 7.500000000000000e+01 +9 84 7.600000000000000e+01 +9 85 7.700000000000000e+01 +9 86 7.800000000000000e+01 +9 87 7.900000000000000e+01 +9 88 8.000000000000000e+01 +9 89 8.100000000000000e+01 +9 90 8.200000000000000e+01 +9 91 8.300000000000000e+01 +9 92 8.400000000000000e+01 +9 93 8.500000000000000e+01 +9 94 8.600000000000000e+01 +9 95 8.700000000000000e+01 +9 96 8.800000000000000e+01 +9 97 8.900000000000000e+01 +9 98 9.000000000000000e+01 +9 99 9.100000000000000e+01 +9 100 9.200000000000000e+01 +10 1 1.000000000000000e+01 +10 2 9.000000000000000e+00 +10 3 8.000000000000000e+00 +10 4 7.000000000000000e+00 +10 5 6.000000000000000e+00 +10 6 5.000000000000000e+00 +10 7 4.000000000000000e+00 +10 8 3.000000000000000e+00 +10 9 2.000000000000000e+00 +10 10 2.000000000000000e+00 +10 11 2.000000000000000e+00 +10 12 3.000000000000000e+00 +10 13 4.000000000000000e+00 +10 14 5.000000000000000e+00 +10 15 6.000000000000000e+00 +10 16 7.000000000000000e+00 +10 17 8.000000000000000e+00 +10 18 9.000000000000000e+00 +10 19 1.000000000000000e+01 +10 20 1.100000000000000e+01 +10 21 1.200000000000000e+01 +10 22 1.300000000000000e+01 +10 23 1.400000000000000e+01 +10 24 1.500000000000000e+01 +10 25 1.600000000000000e+01 +10 26 1.700000000000000e+01 +10 27 1.800000000000000e+01 +10 28 1.900000000000000e+01 +10 29 2.000000000000000e+01 +10 30 2.100000000000000e+01 +10 31 2.200000000000000e+01 +10 32 2.300000000000000e+01 +10 33 2.400000000000000e+01 +10 34 2.500000000000000e+01 +10 35 2.600000000000000e+01 +10 36 2.700000000000000e+01 +10 37 2.800000000000000e+01 +10 38 2.900000000000000e+01 +10 39 3.000000000000000e+01 +10 40 3.100000000000000e+01 +10 41 3.200000000000000e+01 +10 42 3.300000000000000e+01 +10 43 3.400000000000000e+01 +10 44 3.500000000000000e+01 +10 45 3.600000000000000e+01 +10 46 3.700000000000000e+01 +10 47 3.800000000000000e+01 +10 48 3.900000000000000e+01 +10 49 4.000000000000000e+01 +10 50 4.100000000000000e+01 +10 51 4.200000000000000e+01 +10 52 4.300000000000000e+01 +10 53 4.400000000000000e+01 +10 54 4.500000000000000e+01 +10 55 4.600000000000000e+01 +10 56 4.700000000000000e+01 +10 57 4.800000000000000e+01 +10 58 4.900000000000000e+01 +10 59 5.000000000000000e+01 +10 60 5.100000000000000e+01 +10 61 5.200000000000000e+01 +10 62 5.300000000000000e+01 +10 63 5.400000000000000e+01 +10 64 5.500000000000000e+01 +10 65 5.600000000000000e+01 +10 66 5.700000000000000e+01 +10 67 5.800000000000000e+01 +10 68 5.900000000000000e+01 +10 69 6.000000000000000e+01 +10 70 6.100000000000000e+01 +10 71 6.200000000000000e+01 +10 72 6.300000000000000e+01 +10 73 6.400000000000000e+01 +10 74 6.500000000000000e+01 +10 75 6.600000000000000e+01 +10 76 6.700000000000000e+01 +10 77 6.800000000000000e+01 +10 78 6.900000000000000e+01 +10 79 7.000000000000000e+01 +10 80 7.100000000000000e+01 +10 81 7.200000000000000e+01 +10 82 7.300000000000000e+01 +10 83 7.400000000000000e+01 +10 84 7.500000000000000e+01 +10 85 7.600000000000000e+01 +10 86 7.700000000000000e+01 +10 87 7.800000000000000e+01 +10 88 7.900000000000000e+01 +10 89 8.000000000000000e+01 +10 90 8.100000000000000e+01 +10 91 8.200000000000000e+01 +10 92 8.300000000000000e+01 +10 93 8.400000000000000e+01 +10 94 8.500000000000000e+01 +10 95 8.600000000000000e+01 +10 96 8.700000000000000e+01 +10 97 8.800000000000000e+01 +10 98 8.900000000000000e+01 +10 99 9.000000000000000e+01 +10 100 9.100000000000000e+01 +11 1 1.100000000000000e+01 +11 2 1.000000000000000e+01 +11 3 9.000000000000000e+00 +11 4 8.000000000000000e+00 +11 5 7.000000000000000e+00 +11 6 6.000000000000000e+00 +11 7 5.000000000000000e+00 +11 8 4.000000000000000e+00 +11 9 3.000000000000000e+00 +11 10 2.000000000000000e+00 +11 11 2.000000000000000e+00 +11 12 2.000000000000000e+00 +11 13 3.000000000000000e+00 +11 14 4.000000000000000e+00 +11 15 5.000000000000000e+00 +11 16 6.000000000000000e+00 +11 17 7.000000000000000e+00 +11 18 8.000000000000000e+00 +11 19 9.000000000000000e+00 +11 20 1.000000000000000e+01 +11 21 1.100000000000000e+01 +11 22 1.200000000000000e+01 +11 23 1.300000000000000e+01 +11 24 1.400000000000000e+01 +11 25 1.500000000000000e+01 +11 26 1.600000000000000e+01 +11 27 1.700000000000000e+01 +11 28 1.800000000000000e+01 +11 29 1.900000000000000e+01 +11 30 2.000000000000000e+01 +11 31 2.100000000000000e+01 +11 32 2.200000000000000e+01 +11 33 2.300000000000000e+01 +11 34 2.400000000000000e+01 +11 35 2.500000000000000e+01 +11 36 2.600000000000000e+01 +11 37 2.700000000000000e+01 +11 38 2.800000000000000e+01 +11 39 2.900000000000000e+01 +11 40 3.000000000000000e+01 +11 41 3.100000000000000e+01 +11 42 3.200000000000000e+01 +11 43 3.300000000000000e+01 +11 44 3.400000000000000e+01 +11 45 3.500000000000000e+01 +11 46 3.600000000000000e+01 +11 47 3.700000000000000e+01 +11 48 3.800000000000000e+01 +11 49 3.900000000000000e+01 +11 50 4.000000000000000e+01 +11 51 4.100000000000000e+01 +11 52 4.200000000000000e+01 +11 53 4.300000000000000e+01 +11 54 4.400000000000000e+01 +11 55 4.500000000000000e+01 +11 56 4.600000000000000e+01 +11 57 4.700000000000000e+01 +11 58 4.800000000000000e+01 +11 59 4.900000000000000e+01 +11 60 5.000000000000000e+01 +11 61 5.100000000000000e+01 +11 62 5.200000000000000e+01 +11 63 5.300000000000000e+01 +11 64 5.400000000000000e+01 +11 65 5.500000000000000e+01 +11 66 5.600000000000000e+01 +11 67 5.700000000000000e+01 +11 68 5.800000000000000e+01 +11 69 5.900000000000000e+01 +11 70 6.000000000000000e+01 +11 71 6.100000000000000e+01 +11 72 6.200000000000000e+01 +11 73 6.300000000000000e+01 +11 74 6.400000000000000e+01 +11 75 6.500000000000000e+01 +11 76 6.600000000000000e+01 +11 77 6.700000000000000e+01 +11 78 6.800000000000000e+01 +11 79 6.900000000000000e+01 +11 80 7.000000000000000e+01 +11 81 7.100000000000000e+01 +11 82 7.200000000000000e+01 +11 83 7.300000000000000e+01 +11 84 7.400000000000000e+01 +11 85 7.500000000000000e+01 +11 86 7.600000000000000e+01 +11 87 7.700000000000000e+01 +11 88 7.800000000000000e+01 +11 89 7.900000000000000e+01 +11 90 8.000000000000000e+01 +11 91 8.100000000000000e+01 +11 92 8.200000000000000e+01 +11 93 8.300000000000000e+01 +11 94 8.400000000000000e+01 +11 95 8.500000000000000e+01 +11 96 8.600000000000000e+01 +11 97 8.700000000000000e+01 +11 98 8.800000000000000e+01 +11 99 8.900000000000000e+01 +11 100 9.000000000000000e+01 +12 1 1.200000000000000e+01 +12 2 1.100000000000000e+01 +12 3 1.000000000000000e+01 +12 4 9.000000000000000e+00 +12 5 8.000000000000000e+00 +12 6 7.000000000000000e+00 +12 7 6.000000000000000e+00 +12 8 5.000000000000000e+00 +12 9 4.000000000000000e+00 +12 10 3.000000000000000e+00 +12 11 2.000000000000000e+00 +12 12 2.000000000000000e+00 +12 13 2.000000000000000e+00 +12 14 3.000000000000000e+00 +12 15 4.000000000000000e+00 +12 16 5.000000000000000e+00 +12 17 6.000000000000000e+00 +12 18 7.000000000000000e+00 +12 19 8.000000000000000e+00 +12 20 9.000000000000000e+00 +12 21 1.000000000000000e+01 +12 22 1.100000000000000e+01 +12 23 1.200000000000000e+01 +12 24 1.300000000000000e+01 +12 25 1.400000000000000e+01 +12 26 1.500000000000000e+01 +12 27 1.600000000000000e+01 +12 28 1.700000000000000e+01 +12 29 1.800000000000000e+01 +12 30 1.900000000000000e+01 +12 31 2.000000000000000e+01 +12 32 2.100000000000000e+01 +12 33 2.200000000000000e+01 +12 34 2.300000000000000e+01 +12 35 2.400000000000000e+01 +12 36 2.500000000000000e+01 +12 37 2.600000000000000e+01 +12 38 2.700000000000000e+01 +12 39 2.800000000000000e+01 +12 40 2.900000000000000e+01 +12 41 3.000000000000000e+01 +12 42 3.100000000000000e+01 +12 43 3.200000000000000e+01 +12 44 3.300000000000000e+01 +12 45 3.400000000000000e+01 +12 46 3.500000000000000e+01 +12 47 3.600000000000000e+01 +12 48 3.700000000000000e+01 +12 49 3.800000000000000e+01 +12 50 3.900000000000000e+01 +12 51 4.000000000000000e+01 +12 52 4.100000000000000e+01 +12 53 4.200000000000000e+01 +12 54 4.300000000000000e+01 +12 55 4.400000000000000e+01 +12 56 4.500000000000000e+01 +12 57 4.600000000000000e+01 +12 58 4.700000000000000e+01 +12 59 4.800000000000000e+01 +12 60 4.900000000000000e+01 +12 61 5.000000000000000e+01 +12 62 5.100000000000000e+01 +12 63 5.200000000000000e+01 +12 64 5.300000000000000e+01 +12 65 5.400000000000000e+01 +12 66 5.500000000000000e+01 +12 67 5.600000000000000e+01 +12 68 5.700000000000000e+01 +12 69 5.800000000000000e+01 +12 70 5.900000000000000e+01 +12 71 6.000000000000000e+01 +12 72 6.100000000000000e+01 +12 73 6.200000000000000e+01 +12 74 6.300000000000000e+01 +12 75 6.400000000000000e+01 +12 76 6.500000000000000e+01 +12 77 6.600000000000000e+01 +12 78 6.700000000000000e+01 +12 79 6.800000000000000e+01 +12 80 6.900000000000000e+01 +12 81 7.000000000000000e+01 +12 82 7.100000000000000e+01 +12 83 7.200000000000000e+01 +12 84 7.300000000000000e+01 +12 85 7.400000000000000e+01 +12 86 7.500000000000000e+01 +12 87 7.600000000000000e+01 +12 88 7.700000000000000e+01 +12 89 7.800000000000000e+01 +12 90 7.900000000000000e+01 +12 91 8.000000000000000e+01 +12 92 8.100000000000000e+01 +12 93 8.200000000000000e+01 +12 94 8.300000000000000e+01 +12 95 8.400000000000000e+01 +12 96 8.500000000000000e+01 +12 97 8.600000000000000e+01 +12 98 8.700000000000000e+01 +12 99 8.800000000000000e+01 +12 100 8.900000000000000e+01 +13 1 1.300000000000000e+01 +13 2 1.200000000000000e+01 +13 3 1.100000000000000e+01 +13 4 1.000000000000000e+01 +13 5 9.000000000000000e+00 +13 6 8.000000000000000e+00 +13 7 7.000000000000000e+00 +13 8 6.000000000000000e+00 +13 9 5.000000000000000e+00 +13 10 4.000000000000000e+00 +13 11 3.000000000000000e+00 +13 12 2.000000000000000e+00 +13 13 2.000000000000000e+00 +13 14 2.000000000000000e+00 +13 15 3.000000000000000e+00 +13 16 4.000000000000000e+00 +13 17 5.000000000000000e+00 +13 18 6.000000000000000e+00 +13 19 7.000000000000000e+00 +13 20 8.000000000000000e+00 +13 21 9.000000000000000e+00 +13 22 1.000000000000000e+01 +13 23 1.100000000000000e+01 +13 24 1.200000000000000e+01 +13 25 1.300000000000000e+01 +13 26 1.400000000000000e+01 +13 27 1.500000000000000e+01 +13 28 1.600000000000000e+01 +13 29 1.700000000000000e+01 +13 30 1.800000000000000e+01 +13 31 1.900000000000000e+01 +13 32 2.000000000000000e+01 +13 33 2.100000000000000e+01 +13 34 2.200000000000000e+01 +13 35 2.300000000000000e+01 +13 36 2.400000000000000e+01 +13 37 2.500000000000000e+01 +13 38 2.600000000000000e+01 +13 39 2.700000000000000e+01 +13 40 2.800000000000000e+01 +13 41 2.900000000000000e+01 +13 42 3.000000000000000e+01 +13 43 3.100000000000000e+01 +13 44 3.200000000000000e+01 +13 45 3.300000000000000e+01 +13 46 3.400000000000000e+01 +13 47 3.500000000000000e+01 +13 48 3.600000000000000e+01 +13 49 3.700000000000000e+01 +13 50 3.800000000000000e+01 +13 51 3.900000000000000e+01 +13 52 4.000000000000000e+01 +13 53 4.100000000000000e+01 +13 54 4.200000000000000e+01 +13 55 4.300000000000000e+01 +13 56 4.400000000000000e+01 +13 57 4.500000000000000e+01 +13 58 4.600000000000000e+01 +13 59 4.700000000000000e+01 +13 60 4.800000000000000e+01 +13 61 4.900000000000000e+01 +13 62 5.000000000000000e+01 +13 63 5.100000000000000e+01 +13 64 5.200000000000000e+01 +13 65 5.300000000000000e+01 +13 66 5.400000000000000e+01 +13 67 5.500000000000000e+01 +13 68 5.600000000000000e+01 +13 69 5.700000000000000e+01 +13 70 5.800000000000000e+01 +13 71 5.900000000000000e+01 +13 72 6.000000000000000e+01 +13 73 6.100000000000000e+01 +13 74 6.200000000000000e+01 +13 75 6.300000000000000e+01 +13 76 6.400000000000000e+01 +13 77 6.500000000000000e+01 +13 78 6.600000000000000e+01 +13 79 6.700000000000000e+01 +13 80 6.800000000000000e+01 +13 81 6.900000000000000e+01 +13 82 7.000000000000000e+01 +13 83 7.100000000000000e+01 +13 84 7.200000000000000e+01 +13 85 7.300000000000000e+01 +13 86 7.400000000000000e+01 +13 87 7.500000000000000e+01 +13 88 7.600000000000000e+01 +13 89 7.700000000000000e+01 +13 90 7.800000000000000e+01 +13 91 7.900000000000000e+01 +13 92 8.000000000000000e+01 +13 93 8.100000000000000e+01 +13 94 8.200000000000000e+01 +13 95 8.300000000000000e+01 +13 96 8.400000000000000e+01 +13 97 8.500000000000000e+01 +13 98 8.600000000000000e+01 +13 99 8.700000000000000e+01 +13 100 8.800000000000000e+01 +14 1 1.400000000000000e+01 +14 2 1.300000000000000e+01 +14 3 1.200000000000000e+01 +14 4 1.100000000000000e+01 +14 5 1.000000000000000e+01 +14 6 9.000000000000000e+00 +14 7 8.000000000000000e+00 +14 8 7.000000000000000e+00 +14 9 6.000000000000000e+00 +14 10 5.000000000000000e+00 +14 11 4.000000000000000e+00 +14 12 3.000000000000000e+00 +14 13 2.000000000000000e+00 +14 14 2.000000000000000e+00 +14 15 2.000000000000000e+00 +14 16 3.000000000000000e+00 +14 17 4.000000000000000e+00 +14 18 5.000000000000000e+00 +14 19 6.000000000000000e+00 +14 20 7.000000000000000e+00 +14 21 8.000000000000000e+00 +14 22 9.000000000000000e+00 +14 23 1.000000000000000e+01 +14 24 1.100000000000000e+01 +14 25 1.200000000000000e+01 +14 26 1.300000000000000e+01 +14 27 1.400000000000000e+01 +14 28 1.500000000000000e+01 +14 29 1.600000000000000e+01 +14 30 1.700000000000000e+01 +14 31 1.800000000000000e+01 +14 32 1.900000000000000e+01 +14 33 2.000000000000000e+01 +14 34 2.100000000000000e+01 +14 35 2.200000000000000e+01 +14 36 2.300000000000000e+01 +14 37 2.400000000000000e+01 +14 38 2.500000000000000e+01 +14 39 2.600000000000000e+01 +14 40 2.700000000000000e+01 +14 41 2.800000000000000e+01 +14 42 2.900000000000000e+01 +14 43 3.000000000000000e+01 +14 44 3.100000000000000e+01 +14 45 3.200000000000000e+01 +14 46 3.300000000000000e+01 +14 47 3.400000000000000e+01 +14 48 3.500000000000000e+01 +14 49 3.600000000000000e+01 +14 50 3.700000000000000e+01 +14 51 3.800000000000000e+01 +14 52 3.900000000000000e+01 +14 53 4.000000000000000e+01 +14 54 4.100000000000000e+01 +14 55 4.200000000000000e+01 +14 56 4.300000000000000e+01 +14 57 4.400000000000000e+01 +14 58 4.500000000000000e+01 +14 59 4.600000000000000e+01 +14 60 4.700000000000000e+01 +14 61 4.800000000000000e+01 +14 62 4.900000000000000e+01 +14 63 5.000000000000000e+01 +14 64 5.100000000000000e+01 +14 65 5.200000000000000e+01 +14 66 5.300000000000000e+01 +14 67 5.400000000000000e+01 +14 68 5.500000000000000e+01 +14 69 5.600000000000000e+01 +14 70 5.700000000000000e+01 +14 71 5.800000000000000e+01 +14 72 5.900000000000000e+01 +14 73 6.000000000000000e+01 +14 74 6.100000000000000e+01 +14 75 6.200000000000000e+01 +14 76 6.300000000000000e+01 +14 77 6.400000000000000e+01 +14 78 6.500000000000000e+01 +14 79 6.600000000000000e+01 +14 80 6.700000000000000e+01 +14 81 6.800000000000000e+01 +14 82 6.900000000000000e+01 +14 83 7.000000000000000e+01 +14 84 7.100000000000000e+01 +14 85 7.200000000000000e+01 +14 86 7.300000000000000e+01 +14 87 7.400000000000000e+01 +14 88 7.500000000000000e+01 +14 89 7.600000000000000e+01 +14 90 7.700000000000000e+01 +14 91 7.800000000000000e+01 +14 92 7.900000000000000e+01 +14 93 8.000000000000000e+01 +14 94 8.100000000000000e+01 +14 95 8.200000000000000e+01 +14 96 8.300000000000000e+01 +14 97 8.400000000000000e+01 +14 98 8.500000000000000e+01 +14 99 8.600000000000000e+01 +14 100 8.700000000000000e+01 +15 1 1.500000000000000e+01 +15 2 1.400000000000000e+01 +15 3 1.300000000000000e+01 +15 4 1.200000000000000e+01 +15 5 1.100000000000000e+01 +15 6 1.000000000000000e+01 +15 7 9.000000000000000e+00 +15 8 8.000000000000000e+00 +15 9 7.000000000000000e+00 +15 10 6.000000000000000e+00 +15 11 5.000000000000000e+00 +15 12 4.000000000000000e+00 +15 13 3.000000000000000e+00 +15 14 2.000000000000000e+00 +15 15 2.000000000000000e+00 +15 16 2.000000000000000e+00 +15 17 3.000000000000000e+00 +15 18 4.000000000000000e+00 +15 19 5.000000000000000e+00 +15 20 6.000000000000000e+00 +15 21 7.000000000000000e+00 +15 22 8.000000000000000e+00 +15 23 9.000000000000000e+00 +15 24 1.000000000000000e+01 +15 25 1.100000000000000e+01 +15 26 1.200000000000000e+01 +15 27 1.300000000000000e+01 +15 28 1.400000000000000e+01 +15 29 1.500000000000000e+01 +15 30 1.600000000000000e+01 +15 31 1.700000000000000e+01 +15 32 1.800000000000000e+01 +15 33 1.900000000000000e+01 +15 34 2.000000000000000e+01 +15 35 2.100000000000000e+01 +15 36 2.200000000000000e+01 +15 37 2.300000000000000e+01 +15 38 2.400000000000000e+01 +15 39 2.500000000000000e+01 +15 40 2.600000000000000e+01 +15 41 2.700000000000000e+01 +15 42 2.800000000000000e+01 +15 43 2.900000000000000e+01 +15 44 3.000000000000000e+01 +15 45 3.100000000000000e+01 +15 46 3.200000000000000e+01 +15 47 3.300000000000000e+01 +15 48 3.400000000000000e+01 +15 49 3.500000000000000e+01 +15 50 3.600000000000000e+01 +15 51 3.700000000000000e+01 +15 52 3.800000000000000e+01 +15 53 3.900000000000000e+01 +15 54 4.000000000000000e+01 +15 55 4.100000000000000e+01 +15 56 4.200000000000000e+01 +15 57 4.300000000000000e+01 +15 58 4.400000000000000e+01 +15 59 4.500000000000000e+01 +15 60 4.600000000000000e+01 +15 61 4.700000000000000e+01 +15 62 4.800000000000000e+01 +15 63 4.900000000000000e+01 +15 64 5.000000000000000e+01 +15 65 5.100000000000000e+01 +15 66 5.200000000000000e+01 +15 67 5.300000000000000e+01 +15 68 5.400000000000000e+01 +15 69 5.500000000000000e+01 +15 70 5.600000000000000e+01 +15 71 5.700000000000000e+01 +15 72 5.800000000000000e+01 +15 73 5.900000000000000e+01 +15 74 6.000000000000000e+01 +15 75 6.100000000000000e+01 +15 76 6.200000000000000e+01 +15 77 6.300000000000000e+01 +15 78 6.400000000000000e+01 +15 79 6.500000000000000e+01 +15 80 6.600000000000000e+01 +15 81 6.700000000000000e+01 +15 82 6.800000000000000e+01 +15 83 6.900000000000000e+01 +15 84 7.000000000000000e+01 +15 85 7.100000000000000e+01 +15 86 7.200000000000000e+01 +15 87 7.300000000000000e+01 +15 88 7.400000000000000e+01 +15 89 7.500000000000000e+01 +15 90 7.600000000000000e+01 +15 91 7.700000000000000e+01 +15 92 7.800000000000000e+01 +15 93 7.900000000000000e+01 +15 94 8.000000000000000e+01 +15 95 8.100000000000000e+01 +15 96 8.200000000000000e+01 +15 97 8.300000000000000e+01 +15 98 8.400000000000000e+01 +15 99 8.500000000000000e+01 +15 100 8.600000000000000e+01 +16 1 1.600000000000000e+01 +16 2 1.500000000000000e+01 +16 3 1.400000000000000e+01 +16 4 1.300000000000000e+01 +16 5 1.200000000000000e+01 +16 6 1.100000000000000e+01 +16 7 1.000000000000000e+01 +16 8 9.000000000000000e+00 +16 9 8.000000000000000e+00 +16 10 7.000000000000000e+00 +16 11 6.000000000000000e+00 +16 12 5.000000000000000e+00 +16 13 4.000000000000000e+00 +16 14 3.000000000000000e+00 +16 15 2.000000000000000e+00 +16 16 2.000000000000000e+00 +16 17 2.000000000000000e+00 +16 18 3.000000000000000e+00 +16 19 4.000000000000000e+00 +16 20 5.000000000000000e+00 +16 21 6.000000000000000e+00 +16 22 7.000000000000000e+00 +16 23 8.000000000000000e+00 +16 24 9.000000000000000e+00 +16 25 1.000000000000000e+01 +16 26 1.100000000000000e+01 +16 27 1.200000000000000e+01 +16 28 1.300000000000000e+01 +16 29 1.400000000000000e+01 +16 30 1.500000000000000e+01 +16 31 1.600000000000000e+01 +16 32 1.700000000000000e+01 +16 33 1.800000000000000e+01 +16 34 1.900000000000000e+01 +16 35 2.000000000000000e+01 +16 36 2.100000000000000e+01 +16 37 2.200000000000000e+01 +16 38 2.300000000000000e+01 +16 39 2.400000000000000e+01 +16 40 2.500000000000000e+01 +16 41 2.600000000000000e+01 +16 42 2.700000000000000e+01 +16 43 2.800000000000000e+01 +16 44 2.900000000000000e+01 +16 45 3.000000000000000e+01 +16 46 3.100000000000000e+01 +16 47 3.200000000000000e+01 +16 48 3.300000000000000e+01 +16 49 3.400000000000000e+01 +16 50 3.500000000000000e+01 +16 51 3.600000000000000e+01 +16 52 3.700000000000000e+01 +16 53 3.800000000000000e+01 +16 54 3.900000000000000e+01 +16 55 4.000000000000000e+01 +16 56 4.100000000000000e+01 +16 57 4.200000000000000e+01 +16 58 4.300000000000000e+01 +16 59 4.400000000000000e+01 +16 60 4.500000000000000e+01 +16 61 4.600000000000000e+01 +16 62 4.700000000000000e+01 +16 63 4.800000000000000e+01 +16 64 4.900000000000000e+01 +16 65 5.000000000000000e+01 +16 66 5.100000000000000e+01 +16 67 5.200000000000000e+01 +16 68 5.300000000000000e+01 +16 69 5.400000000000000e+01 +16 70 5.500000000000000e+01 +16 71 5.600000000000000e+01 +16 72 5.700000000000000e+01 +16 73 5.800000000000000e+01 +16 74 5.900000000000000e+01 +16 75 6.000000000000000e+01 +16 76 6.100000000000000e+01 +16 77 6.200000000000000e+01 +16 78 6.300000000000000e+01 +16 79 6.400000000000000e+01 +16 80 6.500000000000000e+01 +16 81 6.600000000000000e+01 +16 82 6.700000000000000e+01 +16 83 6.800000000000000e+01 +16 84 6.900000000000000e+01 +16 85 7.000000000000000e+01 +16 86 7.100000000000000e+01 +16 87 7.200000000000000e+01 +16 88 7.300000000000000e+01 +16 89 7.400000000000000e+01 +16 90 7.500000000000000e+01 +16 91 7.600000000000000e+01 +16 92 7.700000000000000e+01 +16 93 7.800000000000000e+01 +16 94 7.900000000000000e+01 +16 95 8.000000000000000e+01 +16 96 8.100000000000000e+01 +16 97 8.200000000000000e+01 +16 98 8.300000000000000e+01 +16 99 8.400000000000000e+01 +16 100 8.500000000000000e+01 +17 1 1.700000000000000e+01 +17 2 1.600000000000000e+01 +17 3 1.500000000000000e+01 +17 4 1.400000000000000e+01 +17 5 1.300000000000000e+01 +17 6 1.200000000000000e+01 +17 7 1.100000000000000e+01 +17 8 1.000000000000000e+01 +17 9 9.000000000000000e+00 +17 10 8.000000000000000e+00 +17 11 7.000000000000000e+00 +17 12 6.000000000000000e+00 +17 13 5.000000000000000e+00 +17 14 4.000000000000000e+00 +17 15 3.000000000000000e+00 +17 16 2.000000000000000e+00 +17 17 2.000000000000000e+00 +17 18 2.000000000000000e+00 +17 19 3.000000000000000e+00 +17 20 4.000000000000000e+00 +17 21 5.000000000000000e+00 +17 22 6.000000000000000e+00 +17 23 7.000000000000000e+00 +17 24 8.000000000000000e+00 +17 25 9.000000000000000e+00 +17 26 1.000000000000000e+01 +17 27 1.100000000000000e+01 +17 28 1.200000000000000e+01 +17 29 1.300000000000000e+01 +17 30 1.400000000000000e+01 +17 31 1.500000000000000e+01 +17 32 1.600000000000000e+01 +17 33 1.700000000000000e+01 +17 34 1.800000000000000e+01 +17 35 1.900000000000000e+01 +17 36 2.000000000000000e+01 +17 37 2.100000000000000e+01 +17 38 2.200000000000000e+01 +17 39 2.300000000000000e+01 +17 40 2.400000000000000e+01 +17 41 2.500000000000000e+01 +17 42 2.600000000000000e+01 +17 43 2.700000000000000e+01 +17 44 2.800000000000000e+01 +17 45 2.900000000000000e+01 +17 46 3.000000000000000e+01 +17 47 3.100000000000000e+01 +17 48 3.200000000000000e+01 +17 49 3.300000000000000e+01 +17 50 3.400000000000000e+01 +17 51 3.500000000000000e+01 +17 52 3.600000000000000e+01 +17 53 3.700000000000000e+01 +17 54 3.800000000000000e+01 +17 55 3.900000000000000e+01 +17 56 4.000000000000000e+01 +17 57 4.100000000000000e+01 +17 58 4.200000000000000e+01 +17 59 4.300000000000000e+01 +17 60 4.400000000000000e+01 +17 61 4.500000000000000e+01 +17 62 4.600000000000000e+01 +17 63 4.700000000000000e+01 +17 64 4.800000000000000e+01 +17 65 4.900000000000000e+01 +17 66 5.000000000000000e+01 +17 67 5.100000000000000e+01 +17 68 5.200000000000000e+01 +17 69 5.300000000000000e+01 +17 70 5.400000000000000e+01 +17 71 5.500000000000000e+01 +17 72 5.600000000000000e+01 +17 73 5.700000000000000e+01 +17 74 5.800000000000000e+01 +17 75 5.900000000000000e+01 +17 76 6.000000000000000e+01 +17 77 6.100000000000000e+01 +17 78 6.200000000000000e+01 +17 79 6.300000000000000e+01 +17 80 6.400000000000000e+01 +17 81 6.500000000000000e+01 +17 82 6.600000000000000e+01 +17 83 6.700000000000000e+01 +17 84 6.800000000000000e+01 +17 85 6.900000000000000e+01 +17 86 7.000000000000000e+01 +17 87 7.100000000000000e+01 +17 88 7.200000000000000e+01 +17 89 7.300000000000000e+01 +17 90 7.400000000000000e+01 +17 91 7.500000000000000e+01 +17 92 7.600000000000000e+01 +17 93 7.700000000000000e+01 +17 94 7.800000000000000e+01 +17 95 7.900000000000000e+01 +17 96 8.000000000000000e+01 +17 97 8.100000000000000e+01 +17 98 8.200000000000000e+01 +17 99 8.300000000000000e+01 +17 100 8.400000000000000e+01 +18 1 1.800000000000000e+01 +18 2 1.700000000000000e+01 +18 3 1.600000000000000e+01 +18 4 1.500000000000000e+01 +18 5 1.400000000000000e+01 +18 6 1.300000000000000e+01 +18 7 1.200000000000000e+01 +18 8 1.100000000000000e+01 +18 9 1.000000000000000e+01 +18 10 9.000000000000000e+00 +18 11 8.000000000000000e+00 +18 12 7.000000000000000e+00 +18 13 6.000000000000000e+00 +18 14 5.000000000000000e+00 +18 15 4.000000000000000e+00 +18 16 3.000000000000000e+00 +18 17 2.000000000000000e+00 +18 18 2.000000000000000e+00 +18 19 2.000000000000000e+00 +18 20 3.000000000000000e+00 +18 21 4.000000000000000e+00 +18 22 5.000000000000000e+00 +18 23 6.000000000000000e+00 +18 24 7.000000000000000e+00 +18 25 8.000000000000000e+00 +18 26 9.000000000000000e+00 +18 27 1.000000000000000e+01 +18 28 1.100000000000000e+01 +18 29 1.200000000000000e+01 +18 30 1.300000000000000e+01 +18 31 1.400000000000000e+01 +18 32 1.500000000000000e+01 +18 33 1.600000000000000e+01 +18 34 1.700000000000000e+01 +18 35 1.800000000000000e+01 +18 36 1.900000000000000e+01 +18 37 2.000000000000000e+01 +18 38 2.100000000000000e+01 +18 39 2.200000000000000e+01 +18 40 2.300000000000000e+01 +18 41 2.400000000000000e+01 +18 42 2.500000000000000e+01 +18 43 2.600000000000000e+01 +18 44 2.700000000000000e+01 +18 45 2.800000000000000e+01 +18 46 2.900000000000000e+01 +18 47 3.000000000000000e+01 +18 48 3.100000000000000e+01 +18 49 3.200000000000000e+01 +18 50 3.300000000000000e+01 +18 51 3.400000000000000e+01 +18 52 3.500000000000000e+01 +18 53 3.600000000000000e+01 +18 54 3.700000000000000e+01 +18 55 3.800000000000000e+01 +18 56 3.900000000000000e+01 +18 57 4.000000000000000e+01 +18 58 4.100000000000000e+01 +18 59 4.200000000000000e+01 +18 60 4.300000000000000e+01 +18 61 4.400000000000000e+01 +18 62 4.500000000000000e+01 +18 63 4.600000000000000e+01 +18 64 4.700000000000000e+01 +18 65 4.800000000000000e+01 +18 66 4.900000000000000e+01 +18 67 5.000000000000000e+01 +18 68 5.100000000000000e+01 +18 69 5.200000000000000e+01 +18 70 5.300000000000000e+01 +18 71 5.400000000000000e+01 +18 72 5.500000000000000e+01 +18 73 5.600000000000000e+01 +18 74 5.700000000000000e+01 +18 75 5.800000000000000e+01 +18 76 5.900000000000000e+01 +18 77 6.000000000000000e+01 +18 78 6.100000000000000e+01 +18 79 6.200000000000000e+01 +18 80 6.300000000000000e+01 +18 81 6.400000000000000e+01 +18 82 6.500000000000000e+01 +18 83 6.600000000000000e+01 +18 84 6.700000000000000e+01 +18 85 6.800000000000000e+01 +18 86 6.900000000000000e+01 +18 87 7.000000000000000e+01 +18 88 7.100000000000000e+01 +18 89 7.200000000000000e+01 +18 90 7.300000000000000e+01 +18 91 7.400000000000000e+01 +18 92 7.500000000000000e+01 +18 93 7.600000000000000e+01 +18 94 7.700000000000000e+01 +18 95 7.800000000000000e+01 +18 96 7.900000000000000e+01 +18 97 8.000000000000000e+01 +18 98 8.100000000000000e+01 +18 99 8.200000000000000e+01 +18 100 8.300000000000000e+01 +19 1 1.900000000000000e+01 +19 2 1.800000000000000e+01 +19 3 1.700000000000000e+01 +19 4 1.600000000000000e+01 +19 5 1.500000000000000e+01 +19 6 1.400000000000000e+01 +19 7 1.300000000000000e+01 +19 8 1.200000000000000e+01 +19 9 1.100000000000000e+01 +19 10 1.000000000000000e+01 +19 11 9.000000000000000e+00 +19 12 8.000000000000000e+00 +19 13 7.000000000000000e+00 +19 14 6.000000000000000e+00 +19 15 5.000000000000000e+00 +19 16 4.000000000000000e+00 +19 17 3.000000000000000e+00 +19 18 2.000000000000000e+00 +19 19 2.000000000000000e+00 +19 20 2.000000000000000e+00 +19 21 3.000000000000000e+00 +19 22 4.000000000000000e+00 +19 23 5.000000000000000e+00 +19 24 6.000000000000000e+00 +19 25 7.000000000000000e+00 +19 26 8.000000000000000e+00 +19 27 9.000000000000000e+00 +19 28 1.000000000000000e+01 +19 29 1.100000000000000e+01 +19 30 1.200000000000000e+01 +19 31 1.300000000000000e+01 +19 32 1.400000000000000e+01 +19 33 1.500000000000000e+01 +19 34 1.600000000000000e+01 +19 35 1.700000000000000e+01 +19 36 1.800000000000000e+01 +19 37 1.900000000000000e+01 +19 38 2.000000000000000e+01 +19 39 2.100000000000000e+01 +19 40 2.200000000000000e+01 +19 41 2.300000000000000e+01 +19 42 2.400000000000000e+01 +19 43 2.500000000000000e+01 +19 44 2.600000000000000e+01 +19 45 2.700000000000000e+01 +19 46 2.800000000000000e+01 +19 47 2.900000000000000e+01 +19 48 3.000000000000000e+01 +19 49 3.100000000000000e+01 +19 50 3.200000000000000e+01 +19 51 3.300000000000000e+01 +19 52 3.400000000000000e+01 +19 53 3.500000000000000e+01 +19 54 3.600000000000000e+01 +19 55 3.700000000000000e+01 +19 56 3.800000000000000e+01 +19 57 3.900000000000000e+01 +19 58 4.000000000000000e+01 +19 59 4.100000000000000e+01 +19 60 4.200000000000000e+01 +19 61 4.300000000000000e+01 +19 62 4.400000000000000e+01 +19 63 4.500000000000000e+01 +19 64 4.600000000000000e+01 +19 65 4.700000000000000e+01 +19 66 4.800000000000000e+01 +19 67 4.900000000000000e+01 +19 68 5.000000000000000e+01 +19 69 5.100000000000000e+01 +19 70 5.200000000000000e+01 +19 71 5.300000000000000e+01 +19 72 5.400000000000000e+01 +19 73 5.500000000000000e+01 +19 74 5.600000000000000e+01 +19 75 5.700000000000000e+01 +19 76 5.800000000000000e+01 +19 77 5.900000000000000e+01 +19 78 6.000000000000000e+01 +19 79 6.100000000000000e+01 +19 80 6.200000000000000e+01 +19 81 6.300000000000000e+01 +19 82 6.400000000000000e+01 +19 83 6.500000000000000e+01 +19 84 6.600000000000000e+01 +19 85 6.700000000000000e+01 +19 86 6.800000000000000e+01 +19 87 6.900000000000000e+01 +19 88 7.000000000000000e+01 +19 89 7.100000000000000e+01 +19 90 7.200000000000000e+01 +19 91 7.300000000000000e+01 +19 92 7.400000000000000e+01 +19 93 7.500000000000000e+01 +19 94 7.600000000000000e+01 +19 95 7.700000000000000e+01 +19 96 7.800000000000000e+01 +19 97 7.900000000000000e+01 +19 98 8.000000000000000e+01 +19 99 8.100000000000000e+01 +19 100 8.200000000000000e+01 +20 1 2.000000000000000e+01 +20 2 1.900000000000000e+01 +20 3 1.800000000000000e+01 +20 4 1.700000000000000e+01 +20 5 1.600000000000000e+01 +20 6 1.500000000000000e+01 +20 7 1.400000000000000e+01 +20 8 1.300000000000000e+01 +20 9 1.200000000000000e+01 +20 10 1.100000000000000e+01 +20 11 1.000000000000000e+01 +20 12 9.000000000000000e+00 +20 13 8.000000000000000e+00 +20 14 7.000000000000000e+00 +20 15 6.000000000000000e+00 +20 16 5.000000000000000e+00 +20 17 4.000000000000000e+00 +20 18 3.000000000000000e+00 +20 19 2.000000000000000e+00 +20 20 2.000000000000000e+00 +20 21 2.000000000000000e+00 +20 22 3.000000000000000e+00 +20 23 4.000000000000000e+00 +20 24 5.000000000000000e+00 +20 25 6.000000000000000e+00 +20 26 7.000000000000000e+00 +20 27 8.000000000000000e+00 +20 28 9.000000000000000e+00 +20 29 1.000000000000000e+01 +20 30 1.100000000000000e+01 +20 31 1.200000000000000e+01 +20 32 1.300000000000000e+01 +20 33 1.400000000000000e+01 +20 34 1.500000000000000e+01 +20 35 1.600000000000000e+01 +20 36 1.700000000000000e+01 +20 37 1.800000000000000e+01 +20 38 1.900000000000000e+01 +20 39 2.000000000000000e+01 +20 40 2.100000000000000e+01 +20 41 2.200000000000000e+01 +20 42 2.300000000000000e+01 +20 43 2.400000000000000e+01 +20 44 2.500000000000000e+01 +20 45 2.600000000000000e+01 +20 46 2.700000000000000e+01 +20 47 2.800000000000000e+01 +20 48 2.900000000000000e+01 +20 49 3.000000000000000e+01 +20 50 3.100000000000000e+01 +20 51 3.200000000000000e+01 +20 52 3.300000000000000e+01 +20 53 3.400000000000000e+01 +20 54 3.500000000000000e+01 +20 55 3.600000000000000e+01 +20 56 3.700000000000000e+01 +20 57 3.800000000000000e+01 +20 58 3.900000000000000e+01 +20 59 4.000000000000000e+01 +20 60 4.100000000000000e+01 +20 61 4.200000000000000e+01 +20 62 4.300000000000000e+01 +20 63 4.400000000000000e+01 +20 64 4.500000000000000e+01 +20 65 4.600000000000000e+01 +20 66 4.700000000000000e+01 +20 67 4.800000000000000e+01 +20 68 4.900000000000000e+01 +20 69 5.000000000000000e+01 +20 70 5.100000000000000e+01 +20 71 5.200000000000000e+01 +20 72 5.300000000000000e+01 +20 73 5.400000000000000e+01 +20 74 5.500000000000000e+01 +20 75 5.600000000000000e+01 +20 76 5.700000000000000e+01 +20 77 5.800000000000000e+01 +20 78 5.900000000000000e+01 +20 79 6.000000000000000e+01 +20 80 6.100000000000000e+01 +20 81 6.200000000000000e+01 +20 82 6.300000000000000e+01 +20 83 6.400000000000000e+01 +20 84 6.500000000000000e+01 +20 85 6.600000000000000e+01 +20 86 6.700000000000000e+01 +20 87 6.800000000000000e+01 +20 88 6.900000000000000e+01 +20 89 7.000000000000000e+01 +20 90 7.100000000000000e+01 +20 91 7.200000000000000e+01 +20 92 7.300000000000000e+01 +20 93 7.400000000000000e+01 +20 94 7.500000000000000e+01 +20 95 7.600000000000000e+01 +20 96 7.700000000000000e+01 +20 97 7.800000000000000e+01 +20 98 7.900000000000000e+01 +20 99 8.000000000000000e+01 +20 100 8.100000000000000e+01 +21 1 2.100000000000000e+01 +21 2 2.000000000000000e+01 +21 3 1.900000000000000e+01 +21 4 1.800000000000000e+01 +21 5 1.700000000000000e+01 +21 6 1.600000000000000e+01 +21 7 1.500000000000000e+01 +21 8 1.400000000000000e+01 +21 9 1.300000000000000e+01 +21 10 1.200000000000000e+01 +21 11 1.100000000000000e+01 +21 12 1.000000000000000e+01 +21 13 9.000000000000000e+00 +21 14 8.000000000000000e+00 +21 15 7.000000000000000e+00 +21 16 6.000000000000000e+00 +21 17 5.000000000000000e+00 +21 18 4.000000000000000e+00 +21 19 3.000000000000000e+00 +21 20 2.000000000000000e+00 +21 21 2.000000000000000e+00 +21 22 2.000000000000000e+00 +21 23 3.000000000000000e+00 +21 24 4.000000000000000e+00 +21 25 5.000000000000000e+00 +21 26 6.000000000000000e+00 +21 27 7.000000000000000e+00 +21 28 8.000000000000000e+00 +21 29 9.000000000000000e+00 +21 30 1.000000000000000e+01 +21 31 1.100000000000000e+01 +21 32 1.200000000000000e+01 +21 33 1.300000000000000e+01 +21 34 1.400000000000000e+01 +21 35 1.500000000000000e+01 +21 36 1.600000000000000e+01 +21 37 1.700000000000000e+01 +21 38 1.800000000000000e+01 +21 39 1.900000000000000e+01 +21 40 2.000000000000000e+01 +21 41 2.100000000000000e+01 +21 42 2.200000000000000e+01 +21 43 2.300000000000000e+01 +21 44 2.400000000000000e+01 +21 45 2.500000000000000e+01 +21 46 2.600000000000000e+01 +21 47 2.700000000000000e+01 +21 48 2.800000000000000e+01 +21 49 2.900000000000000e+01 +21 50 3.000000000000000e+01 +21 51 3.100000000000000e+01 +21 52 3.200000000000000e+01 +21 53 3.300000000000000e+01 +21 54 3.400000000000000e+01 +21 55 3.500000000000000e+01 +21 56 3.600000000000000e+01 +21 57 3.700000000000000e+01 +21 58 3.800000000000000e+01 +21 59 3.900000000000000e+01 +21 60 4.000000000000000e+01 +21 61 4.100000000000000e+01 +21 62 4.200000000000000e+01 +21 63 4.300000000000000e+01 +21 64 4.400000000000000e+01 +21 65 4.500000000000000e+01 +21 66 4.600000000000000e+01 +21 67 4.700000000000000e+01 +21 68 4.800000000000000e+01 +21 69 4.900000000000000e+01 +21 70 5.000000000000000e+01 +21 71 5.100000000000000e+01 +21 72 5.200000000000000e+01 +21 73 5.300000000000000e+01 +21 74 5.400000000000000e+01 +21 75 5.500000000000000e+01 +21 76 5.600000000000000e+01 +21 77 5.700000000000000e+01 +21 78 5.800000000000000e+01 +21 79 5.900000000000000e+01 +21 80 6.000000000000000e+01 +21 81 6.100000000000000e+01 +21 82 6.200000000000000e+01 +21 83 6.300000000000000e+01 +21 84 6.400000000000000e+01 +21 85 6.500000000000000e+01 +21 86 6.600000000000000e+01 +21 87 6.700000000000000e+01 +21 88 6.800000000000000e+01 +21 89 6.900000000000000e+01 +21 90 7.000000000000000e+01 +21 91 7.100000000000000e+01 +21 92 7.200000000000000e+01 +21 93 7.300000000000000e+01 +21 94 7.400000000000000e+01 +21 95 7.500000000000000e+01 +21 96 7.600000000000000e+01 +21 97 7.700000000000000e+01 +21 98 7.800000000000000e+01 +21 99 7.900000000000000e+01 +21 100 8.000000000000000e+01 +22 1 2.200000000000000e+01 +22 2 2.100000000000000e+01 +22 3 2.000000000000000e+01 +22 4 1.900000000000000e+01 +22 5 1.800000000000000e+01 +22 6 1.700000000000000e+01 +22 7 1.600000000000000e+01 +22 8 1.500000000000000e+01 +22 9 1.400000000000000e+01 +22 10 1.300000000000000e+01 +22 11 1.200000000000000e+01 +22 12 1.100000000000000e+01 +22 13 1.000000000000000e+01 +22 14 9.000000000000000e+00 +22 15 8.000000000000000e+00 +22 16 7.000000000000000e+00 +22 17 6.000000000000000e+00 +22 18 5.000000000000000e+00 +22 19 4.000000000000000e+00 +22 20 3.000000000000000e+00 +22 21 2.000000000000000e+00 +22 22 2.000000000000000e+00 +22 23 2.000000000000000e+00 +22 24 3.000000000000000e+00 +22 25 4.000000000000000e+00 +22 26 5.000000000000000e+00 +22 27 6.000000000000000e+00 +22 28 7.000000000000000e+00 +22 29 8.000000000000000e+00 +22 30 9.000000000000000e+00 +22 31 1.000000000000000e+01 +22 32 1.100000000000000e+01 +22 33 1.200000000000000e+01 +22 34 1.300000000000000e+01 +22 35 1.400000000000000e+01 +22 36 1.500000000000000e+01 +22 37 1.600000000000000e+01 +22 38 1.700000000000000e+01 +22 39 1.800000000000000e+01 +22 40 1.900000000000000e+01 +22 41 2.000000000000000e+01 +22 42 2.100000000000000e+01 +22 43 2.200000000000000e+01 +22 44 2.300000000000000e+01 +22 45 2.400000000000000e+01 +22 46 2.500000000000000e+01 +22 47 2.600000000000000e+01 +22 48 2.700000000000000e+01 +22 49 2.800000000000000e+01 +22 50 2.900000000000000e+01 +22 51 3.000000000000000e+01 +22 52 3.100000000000000e+01 +22 53 3.200000000000000e+01 +22 54 3.300000000000000e+01 +22 55 3.400000000000000e+01 +22 56 3.500000000000000e+01 +22 57 3.600000000000000e+01 +22 58 3.700000000000000e+01 +22 59 3.800000000000000e+01 +22 60 3.900000000000000e+01 +22 61 4.000000000000000e+01 +22 62 4.100000000000000e+01 +22 63 4.200000000000000e+01 +22 64 4.300000000000000e+01 +22 65 4.400000000000000e+01 +22 66 4.500000000000000e+01 +22 67 4.600000000000000e+01 +22 68 4.700000000000000e+01 +22 69 4.800000000000000e+01 +22 70 4.900000000000000e+01 +22 71 5.000000000000000e+01 +22 72 5.100000000000000e+01 +22 73 5.200000000000000e+01 +22 74 5.300000000000000e+01 +22 75 5.400000000000000e+01 +22 76 5.500000000000000e+01 +22 77 5.600000000000000e+01 +22 78 5.700000000000000e+01 +22 79 5.800000000000000e+01 +22 80 5.900000000000000e+01 +22 81 6.000000000000000e+01 +22 82 6.100000000000000e+01 +22 83 6.200000000000000e+01 +22 84 6.300000000000000e+01 +22 85 6.400000000000000e+01 +22 86 6.500000000000000e+01 +22 87 6.600000000000000e+01 +22 88 6.700000000000000e+01 +22 89 6.800000000000000e+01 +22 90 6.900000000000000e+01 +22 91 7.000000000000000e+01 +22 92 7.100000000000000e+01 +22 93 7.200000000000000e+01 +22 94 7.300000000000000e+01 +22 95 7.400000000000000e+01 +22 96 7.500000000000000e+01 +22 97 7.600000000000000e+01 +22 98 7.700000000000000e+01 +22 99 7.800000000000000e+01 +22 100 7.900000000000000e+01 +23 1 2.300000000000000e+01 +23 2 2.200000000000000e+01 +23 3 2.100000000000000e+01 +23 4 2.000000000000000e+01 +23 5 1.900000000000000e+01 +23 6 1.800000000000000e+01 +23 7 1.700000000000000e+01 +23 8 1.600000000000000e+01 +23 9 1.500000000000000e+01 +23 10 1.400000000000000e+01 +23 11 1.300000000000000e+01 +23 12 1.200000000000000e+01 +23 13 1.100000000000000e+01 +23 14 1.000000000000000e+01 +23 15 9.000000000000000e+00 +23 16 8.000000000000000e+00 +23 17 7.000000000000000e+00 +23 18 6.000000000000000e+00 +23 19 5.000000000000000e+00 +23 20 4.000000000000000e+00 +23 21 3.000000000000000e+00 +23 22 2.000000000000000e+00 +23 23 2.000000000000000e+00 +23 24 2.000000000000000e+00 +23 25 3.000000000000000e+00 +23 26 4.000000000000000e+00 +23 27 5.000000000000000e+00 +23 28 6.000000000000000e+00 +23 29 7.000000000000000e+00 +23 30 8.000000000000000e+00 +23 31 9.000000000000000e+00 +23 32 1.000000000000000e+01 +23 33 1.100000000000000e+01 +23 34 1.200000000000000e+01 +23 35 1.300000000000000e+01 +23 36 1.400000000000000e+01 +23 37 1.500000000000000e+01 +23 38 1.600000000000000e+01 +23 39 1.700000000000000e+01 +23 40 1.800000000000000e+01 +23 41 1.900000000000000e+01 +23 42 2.000000000000000e+01 +23 43 2.100000000000000e+01 +23 44 2.200000000000000e+01 +23 45 2.300000000000000e+01 +23 46 2.400000000000000e+01 +23 47 2.500000000000000e+01 +23 48 2.600000000000000e+01 +23 49 2.700000000000000e+01 +23 50 2.800000000000000e+01 +23 51 2.900000000000000e+01 +23 52 3.000000000000000e+01 +23 53 3.100000000000000e+01 +23 54 3.200000000000000e+01 +23 55 3.300000000000000e+01 +23 56 3.400000000000000e+01 +23 57 3.500000000000000e+01 +23 58 3.600000000000000e+01 +23 59 3.700000000000000e+01 +23 60 3.800000000000000e+01 +23 61 3.900000000000000e+01 +23 62 4.000000000000000e+01 +23 63 4.100000000000000e+01 +23 64 4.200000000000000e+01 +23 65 4.300000000000000e+01 +23 66 4.400000000000000e+01 +23 67 4.500000000000000e+01 +23 68 4.600000000000000e+01 +23 69 4.700000000000000e+01 +23 70 4.800000000000000e+01 +23 71 4.900000000000000e+01 +23 72 5.000000000000000e+01 +23 73 5.100000000000000e+01 +23 74 5.200000000000000e+01 +23 75 5.300000000000000e+01 +23 76 5.400000000000000e+01 +23 77 5.500000000000000e+01 +23 78 5.600000000000000e+01 +23 79 5.700000000000000e+01 +23 80 5.800000000000000e+01 +23 81 5.900000000000000e+01 +23 82 6.000000000000000e+01 +23 83 6.100000000000000e+01 +23 84 6.200000000000000e+01 +23 85 6.300000000000000e+01 +23 86 6.400000000000000e+01 +23 87 6.500000000000000e+01 +23 88 6.600000000000000e+01 +23 89 6.700000000000000e+01 +23 90 6.800000000000000e+01 +23 91 6.900000000000000e+01 +23 92 7.000000000000000e+01 +23 93 7.100000000000000e+01 +23 94 7.200000000000000e+01 +23 95 7.300000000000000e+01 +23 96 7.400000000000000e+01 +23 97 7.500000000000000e+01 +23 98 7.600000000000000e+01 +23 99 7.700000000000000e+01 +23 100 7.800000000000000e+01 +24 1 2.400000000000000e+01 +24 2 2.300000000000000e+01 +24 3 2.200000000000000e+01 +24 4 2.100000000000000e+01 +24 5 2.000000000000000e+01 +24 6 1.900000000000000e+01 +24 7 1.800000000000000e+01 +24 8 1.700000000000000e+01 +24 9 1.600000000000000e+01 +24 10 1.500000000000000e+01 +24 11 1.400000000000000e+01 +24 12 1.300000000000000e+01 +24 13 1.200000000000000e+01 +24 14 1.100000000000000e+01 +24 15 1.000000000000000e+01 +24 16 9.000000000000000e+00 +24 17 8.000000000000000e+00 +24 18 7.000000000000000e+00 +24 19 6.000000000000000e+00 +24 20 5.000000000000000e+00 +24 21 4.000000000000000e+00 +24 22 3.000000000000000e+00 +24 23 2.000000000000000e+00 +24 24 2.000000000000000e+00 +24 25 2.000000000000000e+00 +24 26 3.000000000000000e+00 +24 27 4.000000000000000e+00 +24 28 5.000000000000000e+00 +24 29 6.000000000000000e+00 +24 30 7.000000000000000e+00 +24 31 8.000000000000000e+00 +24 32 9.000000000000000e+00 +24 33 1.000000000000000e+01 +24 34 1.100000000000000e+01 +24 35 1.200000000000000e+01 +24 36 1.300000000000000e+01 +24 37 1.400000000000000e+01 +24 38 1.500000000000000e+01 +24 39 1.600000000000000e+01 +24 40 1.700000000000000e+01 +24 41 1.800000000000000e+01 +24 42 1.900000000000000e+01 +24 43 2.000000000000000e+01 +24 44 2.100000000000000e+01 +24 45 2.200000000000000e+01 +24 46 2.300000000000000e+01 +24 47 2.400000000000000e+01 +24 48 2.500000000000000e+01 +24 49 2.600000000000000e+01 +24 50 2.700000000000000e+01 +24 51 2.800000000000000e+01 +24 52 2.900000000000000e+01 +24 53 3.000000000000000e+01 +24 54 3.100000000000000e+01 +24 55 3.200000000000000e+01 +24 56 3.300000000000000e+01 +24 57 3.400000000000000e+01 +24 58 3.500000000000000e+01 +24 59 3.600000000000000e+01 +24 60 3.700000000000000e+01 +24 61 3.800000000000000e+01 +24 62 3.900000000000000e+01 +24 63 4.000000000000000e+01 +24 64 4.100000000000000e+01 +24 65 4.200000000000000e+01 +24 66 4.300000000000000e+01 +24 67 4.400000000000000e+01 +24 68 4.500000000000000e+01 +24 69 4.600000000000000e+01 +24 70 4.700000000000000e+01 +24 71 4.800000000000000e+01 +24 72 4.900000000000000e+01 +24 73 5.000000000000000e+01 +24 74 5.100000000000000e+01 +24 75 5.200000000000000e+01 +24 76 5.300000000000000e+01 +24 77 5.400000000000000e+01 +24 78 5.500000000000000e+01 +24 79 5.600000000000000e+01 +24 80 5.700000000000000e+01 +24 81 5.800000000000000e+01 +24 82 5.900000000000000e+01 +24 83 6.000000000000000e+01 +24 84 6.100000000000000e+01 +24 85 6.200000000000000e+01 +24 86 6.300000000000000e+01 +24 87 6.400000000000000e+01 +24 88 6.500000000000000e+01 +24 89 6.600000000000000e+01 +24 90 6.700000000000000e+01 +24 91 6.800000000000000e+01 +24 92 6.900000000000000e+01 +24 93 7.000000000000000e+01 +24 94 7.100000000000000e+01 +24 95 7.200000000000000e+01 +24 96 7.300000000000000e+01 +24 97 7.400000000000000e+01 +24 98 7.500000000000000e+01 +24 99 7.600000000000000e+01 +24 100 7.700000000000000e+01 +25 1 2.500000000000000e+01 +25 2 2.400000000000000e+01 +25 3 2.300000000000000e+01 +25 4 2.200000000000000e+01 +25 5 2.100000000000000e+01 +25 6 2.000000000000000e+01 +25 7 1.900000000000000e+01 +25 8 1.800000000000000e+01 +25 9 1.700000000000000e+01 +25 10 1.600000000000000e+01 +25 11 1.500000000000000e+01 +25 12 1.400000000000000e+01 +25 13 1.300000000000000e+01 +25 14 1.200000000000000e+01 +25 15 1.100000000000000e+01 +25 16 1.000000000000000e+01 +25 17 9.000000000000000e+00 +25 18 8.000000000000000e+00 +25 19 7.000000000000000e+00 +25 20 6.000000000000000e+00 +25 21 5.000000000000000e+00 +25 22 4.000000000000000e+00 +25 23 3.000000000000000e+00 +25 24 2.000000000000000e+00 +25 25 2.000000000000000e+00 +25 26 2.000000000000000e+00 +25 27 3.000000000000000e+00 +25 28 4.000000000000000e+00 +25 29 5.000000000000000e+00 +25 30 6.000000000000000e+00 +25 31 7.000000000000000e+00 +25 32 8.000000000000000e+00 +25 33 9.000000000000000e+00 +25 34 1.000000000000000e+01 +25 35 1.100000000000000e+01 +25 36 1.200000000000000e+01 +25 37 1.300000000000000e+01 +25 38 1.400000000000000e+01 +25 39 1.500000000000000e+01 +25 40 1.600000000000000e+01 +25 41 1.700000000000000e+01 +25 42 1.800000000000000e+01 +25 43 1.900000000000000e+01 +25 44 2.000000000000000e+01 +25 45 2.100000000000000e+01 +25 46 2.200000000000000e+01 +25 47 2.300000000000000e+01 +25 48 2.400000000000000e+01 +25 49 2.500000000000000e+01 +25 50 2.600000000000000e+01 +25 51 2.700000000000000e+01 +25 52 2.800000000000000e+01 +25 53 2.900000000000000e+01 +25 54 3.000000000000000e+01 +25 55 3.100000000000000e+01 +25 56 3.200000000000000e+01 +25 57 3.300000000000000e+01 +25 58 3.400000000000000e+01 +25 59 3.500000000000000e+01 +25 60 3.600000000000000e+01 +25 61 3.700000000000000e+01 +25 62 3.800000000000000e+01 +25 63 3.900000000000000e+01 +25 64 4.000000000000000e+01 +25 65 4.100000000000000e+01 +25 66 4.200000000000000e+01 +25 67 4.300000000000000e+01 +25 68 4.400000000000000e+01 +25 69 4.500000000000000e+01 +25 70 4.600000000000000e+01 +25 71 4.700000000000000e+01 +25 72 4.800000000000000e+01 +25 73 4.900000000000000e+01 +25 74 5.000000000000000e+01 +25 75 5.100000000000000e+01 +25 76 5.200000000000000e+01 +25 77 5.300000000000000e+01 +25 78 5.400000000000000e+01 +25 79 5.500000000000000e+01 +25 80 5.600000000000000e+01 +25 81 5.700000000000000e+01 +25 82 5.800000000000000e+01 +25 83 5.900000000000000e+01 +25 84 6.000000000000000e+01 +25 85 6.100000000000000e+01 +25 86 6.200000000000000e+01 +25 87 6.300000000000000e+01 +25 88 6.400000000000000e+01 +25 89 6.500000000000000e+01 +25 90 6.600000000000000e+01 +25 91 6.700000000000000e+01 +25 92 6.800000000000000e+01 +25 93 6.900000000000000e+01 +25 94 7.000000000000000e+01 +25 95 7.100000000000000e+01 +25 96 7.200000000000000e+01 +25 97 7.300000000000000e+01 +25 98 7.400000000000000e+01 +25 99 7.500000000000000e+01 +25 100 7.600000000000000e+01 +26 1 2.600000000000000e+01 +26 2 2.500000000000000e+01 +26 3 2.400000000000000e+01 +26 4 2.300000000000000e+01 +26 5 2.200000000000000e+01 +26 6 2.100000000000000e+01 +26 7 2.000000000000000e+01 +26 8 1.900000000000000e+01 +26 9 1.800000000000000e+01 +26 10 1.700000000000000e+01 +26 11 1.600000000000000e+01 +26 12 1.500000000000000e+01 +26 13 1.400000000000000e+01 +26 14 1.300000000000000e+01 +26 15 1.200000000000000e+01 +26 16 1.100000000000000e+01 +26 17 1.000000000000000e+01 +26 18 9.000000000000000e+00 +26 19 8.000000000000000e+00 +26 20 7.000000000000000e+00 +26 21 6.000000000000000e+00 +26 22 5.000000000000000e+00 +26 23 4.000000000000000e+00 +26 24 3.000000000000000e+00 +26 25 2.000000000000000e+00 +26 26 2.000000000000000e+00 +26 27 2.000000000000000e+00 +26 28 3.000000000000000e+00 +26 29 4.000000000000000e+00 +26 30 5.000000000000000e+00 +26 31 6.000000000000000e+00 +26 32 7.000000000000000e+00 +26 33 8.000000000000000e+00 +26 34 9.000000000000000e+00 +26 35 1.000000000000000e+01 +26 36 1.100000000000000e+01 +26 37 1.200000000000000e+01 +26 38 1.300000000000000e+01 +26 39 1.400000000000000e+01 +26 40 1.500000000000000e+01 +26 41 1.600000000000000e+01 +26 42 1.700000000000000e+01 +26 43 1.800000000000000e+01 +26 44 1.900000000000000e+01 +26 45 2.000000000000000e+01 +26 46 2.100000000000000e+01 +26 47 2.200000000000000e+01 +26 48 2.300000000000000e+01 +26 49 2.400000000000000e+01 +26 50 2.500000000000000e+01 +26 51 2.600000000000000e+01 +26 52 2.700000000000000e+01 +26 53 2.800000000000000e+01 +26 54 2.900000000000000e+01 +26 55 3.000000000000000e+01 +26 56 3.100000000000000e+01 +26 57 3.200000000000000e+01 +26 58 3.300000000000000e+01 +26 59 3.400000000000000e+01 +26 60 3.500000000000000e+01 +26 61 3.600000000000000e+01 +26 62 3.700000000000000e+01 +26 63 3.800000000000000e+01 +26 64 3.900000000000000e+01 +26 65 4.000000000000000e+01 +26 66 4.100000000000000e+01 +26 67 4.200000000000000e+01 +26 68 4.300000000000000e+01 +26 69 4.400000000000000e+01 +26 70 4.500000000000000e+01 +26 71 4.600000000000000e+01 +26 72 4.700000000000000e+01 +26 73 4.800000000000000e+01 +26 74 4.900000000000000e+01 +26 75 5.000000000000000e+01 +26 76 5.100000000000000e+01 +26 77 5.200000000000000e+01 +26 78 5.300000000000000e+01 +26 79 5.400000000000000e+01 +26 80 5.500000000000000e+01 +26 81 5.600000000000000e+01 +26 82 5.700000000000000e+01 +26 83 5.800000000000000e+01 +26 84 5.900000000000000e+01 +26 85 6.000000000000000e+01 +26 86 6.100000000000000e+01 +26 87 6.200000000000000e+01 +26 88 6.300000000000000e+01 +26 89 6.400000000000000e+01 +26 90 6.500000000000000e+01 +26 91 6.600000000000000e+01 +26 92 6.700000000000000e+01 +26 93 6.800000000000000e+01 +26 94 6.900000000000000e+01 +26 95 7.000000000000000e+01 +26 96 7.100000000000000e+01 +26 97 7.200000000000000e+01 +26 98 7.300000000000000e+01 +26 99 7.400000000000000e+01 +26 100 7.500000000000000e+01 +27 1 2.700000000000000e+01 +27 2 2.600000000000000e+01 +27 3 2.500000000000000e+01 +27 4 2.400000000000000e+01 +27 5 2.300000000000000e+01 +27 6 2.200000000000000e+01 +27 7 2.100000000000000e+01 +27 8 2.000000000000000e+01 +27 9 1.900000000000000e+01 +27 10 1.800000000000000e+01 +27 11 1.700000000000000e+01 +27 12 1.600000000000000e+01 +27 13 1.500000000000000e+01 +27 14 1.400000000000000e+01 +27 15 1.300000000000000e+01 +27 16 1.200000000000000e+01 +27 17 1.100000000000000e+01 +27 18 1.000000000000000e+01 +27 19 9.000000000000000e+00 +27 20 8.000000000000000e+00 +27 21 7.000000000000000e+00 +27 22 6.000000000000000e+00 +27 23 5.000000000000000e+00 +27 24 4.000000000000000e+00 +27 25 3.000000000000000e+00 +27 26 2.000000000000000e+00 +27 27 2.000000000000000e+00 +27 28 2.000000000000000e+00 +27 29 3.000000000000000e+00 +27 30 4.000000000000000e+00 +27 31 5.000000000000000e+00 +27 32 6.000000000000000e+00 +27 33 7.000000000000000e+00 +27 34 8.000000000000000e+00 +27 35 9.000000000000000e+00 +27 36 1.000000000000000e+01 +27 37 1.100000000000000e+01 +27 38 1.200000000000000e+01 +27 39 1.300000000000000e+01 +27 40 1.400000000000000e+01 +27 41 1.500000000000000e+01 +27 42 1.600000000000000e+01 +27 43 1.700000000000000e+01 +27 44 1.800000000000000e+01 +27 45 1.900000000000000e+01 +27 46 2.000000000000000e+01 +27 47 2.100000000000000e+01 +27 48 2.200000000000000e+01 +27 49 2.300000000000000e+01 +27 50 2.400000000000000e+01 +27 51 2.500000000000000e+01 +27 52 2.600000000000000e+01 +27 53 2.700000000000000e+01 +27 54 2.800000000000000e+01 +27 55 2.900000000000000e+01 +27 56 3.000000000000000e+01 +27 57 3.100000000000000e+01 +27 58 3.200000000000000e+01 +27 59 3.300000000000000e+01 +27 60 3.400000000000000e+01 +27 61 3.500000000000000e+01 +27 62 3.600000000000000e+01 +27 63 3.700000000000000e+01 +27 64 3.800000000000000e+01 +27 65 3.900000000000000e+01 +27 66 4.000000000000000e+01 +27 67 4.100000000000000e+01 +27 68 4.200000000000000e+01 +27 69 4.300000000000000e+01 +27 70 4.400000000000000e+01 +27 71 4.500000000000000e+01 +27 72 4.600000000000000e+01 +27 73 4.700000000000000e+01 +27 74 4.800000000000000e+01 +27 75 4.900000000000000e+01 +27 76 5.000000000000000e+01 +27 77 5.100000000000000e+01 +27 78 5.200000000000000e+01 +27 79 5.300000000000000e+01 +27 80 5.400000000000000e+01 +27 81 5.500000000000000e+01 +27 82 5.600000000000000e+01 +27 83 5.700000000000000e+01 +27 84 5.800000000000000e+01 +27 85 5.900000000000000e+01 +27 86 6.000000000000000e+01 +27 87 6.100000000000000e+01 +27 88 6.200000000000000e+01 +27 89 6.300000000000000e+01 +27 90 6.400000000000000e+01 +27 91 6.500000000000000e+01 +27 92 6.600000000000000e+01 +27 93 6.700000000000000e+01 +27 94 6.800000000000000e+01 +27 95 6.900000000000000e+01 +27 96 7.000000000000000e+01 +27 97 7.100000000000000e+01 +27 98 7.200000000000000e+01 +27 99 7.300000000000000e+01 +27 100 7.400000000000000e+01 +28 1 2.800000000000000e+01 +28 2 2.700000000000000e+01 +28 3 2.600000000000000e+01 +28 4 2.500000000000000e+01 +28 5 2.400000000000000e+01 +28 6 2.300000000000000e+01 +28 7 2.200000000000000e+01 +28 8 2.100000000000000e+01 +28 9 2.000000000000000e+01 +28 10 1.900000000000000e+01 +28 11 1.800000000000000e+01 +28 12 1.700000000000000e+01 +28 13 1.600000000000000e+01 +28 14 1.500000000000000e+01 +28 15 1.400000000000000e+01 +28 16 1.300000000000000e+01 +28 17 1.200000000000000e+01 +28 18 1.100000000000000e+01 +28 19 1.000000000000000e+01 +28 20 9.000000000000000e+00 +28 21 8.000000000000000e+00 +28 22 7.000000000000000e+00 +28 23 6.000000000000000e+00 +28 24 5.000000000000000e+00 +28 25 4.000000000000000e+00 +28 26 3.000000000000000e+00 +28 27 2.000000000000000e+00 +28 28 2.000000000000000e+00 +28 29 2.000000000000000e+00 +28 30 3.000000000000000e+00 +28 31 4.000000000000000e+00 +28 32 5.000000000000000e+00 +28 33 6.000000000000000e+00 +28 34 7.000000000000000e+00 +28 35 8.000000000000000e+00 +28 36 9.000000000000000e+00 +28 37 1.000000000000000e+01 +28 38 1.100000000000000e+01 +28 39 1.200000000000000e+01 +28 40 1.300000000000000e+01 +28 41 1.400000000000000e+01 +28 42 1.500000000000000e+01 +28 43 1.600000000000000e+01 +28 44 1.700000000000000e+01 +28 45 1.800000000000000e+01 +28 46 1.900000000000000e+01 +28 47 2.000000000000000e+01 +28 48 2.100000000000000e+01 +28 49 2.200000000000000e+01 +28 50 2.300000000000000e+01 +28 51 2.400000000000000e+01 +28 52 2.500000000000000e+01 +28 53 2.600000000000000e+01 +28 54 2.700000000000000e+01 +28 55 2.800000000000000e+01 +28 56 2.900000000000000e+01 +28 57 3.000000000000000e+01 +28 58 3.100000000000000e+01 +28 59 3.200000000000000e+01 +28 60 3.300000000000000e+01 +28 61 3.400000000000000e+01 +28 62 3.500000000000000e+01 +28 63 3.600000000000000e+01 +28 64 3.700000000000000e+01 +28 65 3.800000000000000e+01 +28 66 3.900000000000000e+01 +28 67 4.000000000000000e+01 +28 68 4.100000000000000e+01 +28 69 4.200000000000000e+01 +28 70 4.300000000000000e+01 +28 71 4.400000000000000e+01 +28 72 4.500000000000000e+01 +28 73 4.600000000000000e+01 +28 74 4.700000000000000e+01 +28 75 4.800000000000000e+01 +28 76 4.900000000000000e+01 +28 77 5.000000000000000e+01 +28 78 5.100000000000000e+01 +28 79 5.200000000000000e+01 +28 80 5.300000000000000e+01 +28 81 5.400000000000000e+01 +28 82 5.500000000000000e+01 +28 83 5.600000000000000e+01 +28 84 5.700000000000000e+01 +28 85 5.800000000000000e+01 +28 86 5.900000000000000e+01 +28 87 6.000000000000000e+01 +28 88 6.100000000000000e+01 +28 89 6.200000000000000e+01 +28 90 6.300000000000000e+01 +28 91 6.400000000000000e+01 +28 92 6.500000000000000e+01 +28 93 6.600000000000000e+01 +28 94 6.700000000000000e+01 +28 95 6.800000000000000e+01 +28 96 6.900000000000000e+01 +28 97 7.000000000000000e+01 +28 98 7.100000000000000e+01 +28 99 7.200000000000000e+01 +28 100 7.300000000000000e+01 +29 1 2.900000000000000e+01 +29 2 2.800000000000000e+01 +29 3 2.700000000000000e+01 +29 4 2.600000000000000e+01 +29 5 2.500000000000000e+01 +29 6 2.400000000000000e+01 +29 7 2.300000000000000e+01 +29 8 2.200000000000000e+01 +29 9 2.100000000000000e+01 +29 10 2.000000000000000e+01 +29 11 1.900000000000000e+01 +29 12 1.800000000000000e+01 +29 13 1.700000000000000e+01 +29 14 1.600000000000000e+01 +29 15 1.500000000000000e+01 +29 16 1.400000000000000e+01 +29 17 1.300000000000000e+01 +29 18 1.200000000000000e+01 +29 19 1.100000000000000e+01 +29 20 1.000000000000000e+01 +29 21 9.000000000000000e+00 +29 22 8.000000000000000e+00 +29 23 7.000000000000000e+00 +29 24 6.000000000000000e+00 +29 25 5.000000000000000e+00 +29 26 4.000000000000000e+00 +29 27 3.000000000000000e+00 +29 28 2.000000000000000e+00 +29 29 2.000000000000000e+00 +29 30 2.000000000000000e+00 +29 31 3.000000000000000e+00 +29 32 4.000000000000000e+00 +29 33 5.000000000000000e+00 +29 34 6.000000000000000e+00 +29 35 7.000000000000000e+00 +29 36 8.000000000000000e+00 +29 37 9.000000000000000e+00 +29 38 1.000000000000000e+01 +29 39 1.100000000000000e+01 +29 40 1.200000000000000e+01 +29 41 1.300000000000000e+01 +29 42 1.400000000000000e+01 +29 43 1.500000000000000e+01 +29 44 1.600000000000000e+01 +29 45 1.700000000000000e+01 +29 46 1.800000000000000e+01 +29 47 1.900000000000000e+01 +29 48 2.000000000000000e+01 +29 49 2.100000000000000e+01 +29 50 2.200000000000000e+01 +29 51 2.300000000000000e+01 +29 52 2.400000000000000e+01 +29 53 2.500000000000000e+01 +29 54 2.600000000000000e+01 +29 55 2.700000000000000e+01 +29 56 2.800000000000000e+01 +29 57 2.900000000000000e+01 +29 58 3.000000000000000e+01 +29 59 3.100000000000000e+01 +29 60 3.200000000000000e+01 +29 61 3.300000000000000e+01 +29 62 3.400000000000000e+01 +29 63 3.500000000000000e+01 +29 64 3.600000000000000e+01 +29 65 3.700000000000000e+01 +29 66 3.800000000000000e+01 +29 67 3.900000000000000e+01 +29 68 4.000000000000000e+01 +29 69 4.100000000000000e+01 +29 70 4.200000000000000e+01 +29 71 4.300000000000000e+01 +29 72 4.400000000000000e+01 +29 73 4.500000000000000e+01 +29 74 4.600000000000000e+01 +29 75 4.700000000000000e+01 +29 76 4.800000000000000e+01 +29 77 4.900000000000000e+01 +29 78 5.000000000000000e+01 +29 79 5.100000000000000e+01 +29 80 5.200000000000000e+01 +29 81 5.300000000000000e+01 +29 82 5.400000000000000e+01 +29 83 5.500000000000000e+01 +29 84 5.600000000000000e+01 +29 85 5.700000000000000e+01 +29 86 5.800000000000000e+01 +29 87 5.900000000000000e+01 +29 88 6.000000000000000e+01 +29 89 6.100000000000000e+01 +29 90 6.200000000000000e+01 +29 91 6.300000000000000e+01 +29 92 6.400000000000000e+01 +29 93 6.500000000000000e+01 +29 94 6.600000000000000e+01 +29 95 6.700000000000000e+01 +29 96 6.800000000000000e+01 +29 97 6.900000000000000e+01 +29 98 7.000000000000000e+01 +29 99 7.100000000000000e+01 +29 100 7.200000000000000e+01 +30 1 3.000000000000000e+01 +30 2 2.900000000000000e+01 +30 3 2.800000000000000e+01 +30 4 2.700000000000000e+01 +30 5 2.600000000000000e+01 +30 6 2.500000000000000e+01 +30 7 2.400000000000000e+01 +30 8 2.300000000000000e+01 +30 9 2.200000000000000e+01 +30 10 2.100000000000000e+01 +30 11 2.000000000000000e+01 +30 12 1.900000000000000e+01 +30 13 1.800000000000000e+01 +30 14 1.700000000000000e+01 +30 15 1.600000000000000e+01 +30 16 1.500000000000000e+01 +30 17 1.400000000000000e+01 +30 18 1.300000000000000e+01 +30 19 1.200000000000000e+01 +30 20 1.100000000000000e+01 +30 21 1.000000000000000e+01 +30 22 9.000000000000000e+00 +30 23 8.000000000000000e+00 +30 24 7.000000000000000e+00 +30 25 6.000000000000000e+00 +30 26 5.000000000000000e+00 +30 27 4.000000000000000e+00 +30 28 3.000000000000000e+00 +30 29 2.000000000000000e+00 +30 30 2.000000000000000e+00 +30 31 2.000000000000000e+00 +30 32 3.000000000000000e+00 +30 33 4.000000000000000e+00 +30 34 5.000000000000000e+00 +30 35 6.000000000000000e+00 +30 36 7.000000000000000e+00 +30 37 8.000000000000000e+00 +30 38 9.000000000000000e+00 +30 39 1.000000000000000e+01 +30 40 1.100000000000000e+01 +30 41 1.200000000000000e+01 +30 42 1.300000000000000e+01 +30 43 1.400000000000000e+01 +30 44 1.500000000000000e+01 +30 45 1.600000000000000e+01 +30 46 1.700000000000000e+01 +30 47 1.800000000000000e+01 +30 48 1.900000000000000e+01 +30 49 2.000000000000000e+01 +30 50 2.100000000000000e+01 +30 51 2.200000000000000e+01 +30 52 2.300000000000000e+01 +30 53 2.400000000000000e+01 +30 54 2.500000000000000e+01 +30 55 2.600000000000000e+01 +30 56 2.700000000000000e+01 +30 57 2.800000000000000e+01 +30 58 2.900000000000000e+01 +30 59 3.000000000000000e+01 +30 60 3.100000000000000e+01 +30 61 3.200000000000000e+01 +30 62 3.300000000000000e+01 +30 63 3.400000000000000e+01 +30 64 3.500000000000000e+01 +30 65 3.600000000000000e+01 +30 66 3.700000000000000e+01 +30 67 3.800000000000000e+01 +30 68 3.900000000000000e+01 +30 69 4.000000000000000e+01 +30 70 4.100000000000000e+01 +30 71 4.200000000000000e+01 +30 72 4.300000000000000e+01 +30 73 4.400000000000000e+01 +30 74 4.500000000000000e+01 +30 75 4.600000000000000e+01 +30 76 4.700000000000000e+01 +30 77 4.800000000000000e+01 +30 78 4.900000000000000e+01 +30 79 5.000000000000000e+01 +30 80 5.100000000000000e+01 +30 81 5.200000000000000e+01 +30 82 5.300000000000000e+01 +30 83 5.400000000000000e+01 +30 84 5.500000000000000e+01 +30 85 5.600000000000000e+01 +30 86 5.700000000000000e+01 +30 87 5.800000000000000e+01 +30 88 5.900000000000000e+01 +30 89 6.000000000000000e+01 +30 90 6.100000000000000e+01 +30 91 6.200000000000000e+01 +30 92 6.300000000000000e+01 +30 93 6.400000000000000e+01 +30 94 6.500000000000000e+01 +30 95 6.600000000000000e+01 +30 96 6.700000000000000e+01 +30 97 6.800000000000000e+01 +30 98 6.900000000000000e+01 +30 99 7.000000000000000e+01 +30 100 7.100000000000000e+01 +31 1 3.100000000000000e+01 +31 2 3.000000000000000e+01 +31 3 2.900000000000000e+01 +31 4 2.800000000000000e+01 +31 5 2.700000000000000e+01 +31 6 2.600000000000000e+01 +31 7 2.500000000000000e+01 +31 8 2.400000000000000e+01 +31 9 2.300000000000000e+01 +31 10 2.200000000000000e+01 +31 11 2.100000000000000e+01 +31 12 2.000000000000000e+01 +31 13 1.900000000000000e+01 +31 14 1.800000000000000e+01 +31 15 1.700000000000000e+01 +31 16 1.600000000000000e+01 +31 17 1.500000000000000e+01 +31 18 1.400000000000000e+01 +31 19 1.300000000000000e+01 +31 20 1.200000000000000e+01 +31 21 1.100000000000000e+01 +31 22 1.000000000000000e+01 +31 23 9.000000000000000e+00 +31 24 8.000000000000000e+00 +31 25 7.000000000000000e+00 +31 26 6.000000000000000e+00 +31 27 5.000000000000000e+00 +31 28 4.000000000000000e+00 +31 29 3.000000000000000e+00 +31 30 2.000000000000000e+00 +31 31 2.000000000000000e+00 +31 32 2.000000000000000e+00 +31 33 3.000000000000000e+00 +31 34 4.000000000000000e+00 +31 35 5.000000000000000e+00 +31 36 6.000000000000000e+00 +31 37 7.000000000000000e+00 +31 38 8.000000000000000e+00 +31 39 9.000000000000000e+00 +31 40 1.000000000000000e+01 +31 41 1.100000000000000e+01 +31 42 1.200000000000000e+01 +31 43 1.300000000000000e+01 +31 44 1.400000000000000e+01 +31 45 1.500000000000000e+01 +31 46 1.600000000000000e+01 +31 47 1.700000000000000e+01 +31 48 1.800000000000000e+01 +31 49 1.900000000000000e+01 +31 50 2.000000000000000e+01 +31 51 2.100000000000000e+01 +31 52 2.200000000000000e+01 +31 53 2.300000000000000e+01 +31 54 2.400000000000000e+01 +31 55 2.500000000000000e+01 +31 56 2.600000000000000e+01 +31 57 2.700000000000000e+01 +31 58 2.800000000000000e+01 +31 59 2.900000000000000e+01 +31 60 3.000000000000000e+01 +31 61 3.100000000000000e+01 +31 62 3.200000000000000e+01 +31 63 3.300000000000000e+01 +31 64 3.400000000000000e+01 +31 65 3.500000000000000e+01 +31 66 3.600000000000000e+01 +31 67 3.700000000000000e+01 +31 68 3.800000000000000e+01 +31 69 3.900000000000000e+01 +31 70 4.000000000000000e+01 +31 71 4.100000000000000e+01 +31 72 4.200000000000000e+01 +31 73 4.300000000000000e+01 +31 74 4.400000000000000e+01 +31 75 4.500000000000000e+01 +31 76 4.600000000000000e+01 +31 77 4.700000000000000e+01 +31 78 4.800000000000000e+01 +31 79 4.900000000000000e+01 +31 80 5.000000000000000e+01 +31 81 5.100000000000000e+01 +31 82 5.200000000000000e+01 +31 83 5.300000000000000e+01 +31 84 5.400000000000000e+01 +31 85 5.500000000000000e+01 +31 86 5.600000000000000e+01 +31 87 5.700000000000000e+01 +31 88 5.800000000000000e+01 +31 89 5.900000000000000e+01 +31 90 6.000000000000000e+01 +31 91 6.100000000000000e+01 +31 92 6.200000000000000e+01 +31 93 6.300000000000000e+01 +31 94 6.400000000000000e+01 +31 95 6.500000000000000e+01 +31 96 6.600000000000000e+01 +31 97 6.700000000000000e+01 +31 98 6.800000000000000e+01 +31 99 6.900000000000000e+01 +31 100 7.000000000000000e+01 +32 1 3.200000000000000e+01 +32 2 3.100000000000000e+01 +32 3 3.000000000000000e+01 +32 4 2.900000000000000e+01 +32 5 2.800000000000000e+01 +32 6 2.700000000000000e+01 +32 7 2.600000000000000e+01 +32 8 2.500000000000000e+01 +32 9 2.400000000000000e+01 +32 10 2.300000000000000e+01 +32 11 2.200000000000000e+01 +32 12 2.100000000000000e+01 +32 13 2.000000000000000e+01 +32 14 1.900000000000000e+01 +32 15 1.800000000000000e+01 +32 16 1.700000000000000e+01 +32 17 1.600000000000000e+01 +32 18 1.500000000000000e+01 +32 19 1.400000000000000e+01 +32 20 1.300000000000000e+01 +32 21 1.200000000000000e+01 +32 22 1.100000000000000e+01 +32 23 1.000000000000000e+01 +32 24 9.000000000000000e+00 +32 25 8.000000000000000e+00 +32 26 7.000000000000000e+00 +32 27 6.000000000000000e+00 +32 28 5.000000000000000e+00 +32 29 4.000000000000000e+00 +32 30 3.000000000000000e+00 +32 31 2.000000000000000e+00 +32 32 2.000000000000000e+00 +32 33 2.000000000000000e+00 +32 34 3.000000000000000e+00 +32 35 4.000000000000000e+00 +32 36 5.000000000000000e+00 +32 37 6.000000000000000e+00 +32 38 7.000000000000000e+00 +32 39 8.000000000000000e+00 +32 40 9.000000000000000e+00 +32 41 1.000000000000000e+01 +32 42 1.100000000000000e+01 +32 43 1.200000000000000e+01 +32 44 1.300000000000000e+01 +32 45 1.400000000000000e+01 +32 46 1.500000000000000e+01 +32 47 1.600000000000000e+01 +32 48 1.700000000000000e+01 +32 49 1.800000000000000e+01 +32 50 1.900000000000000e+01 +32 51 2.000000000000000e+01 +32 52 2.100000000000000e+01 +32 53 2.200000000000000e+01 +32 54 2.300000000000000e+01 +32 55 2.400000000000000e+01 +32 56 2.500000000000000e+01 +32 57 2.600000000000000e+01 +32 58 2.700000000000000e+01 +32 59 2.800000000000000e+01 +32 60 2.900000000000000e+01 +32 61 3.000000000000000e+01 +32 62 3.100000000000000e+01 +32 63 3.200000000000000e+01 +32 64 3.300000000000000e+01 +32 65 3.400000000000000e+01 +32 66 3.500000000000000e+01 +32 67 3.600000000000000e+01 +32 68 3.700000000000000e+01 +32 69 3.800000000000000e+01 +32 70 3.900000000000000e+01 +32 71 4.000000000000000e+01 +32 72 4.100000000000000e+01 +32 73 4.200000000000000e+01 +32 74 4.300000000000000e+01 +32 75 4.400000000000000e+01 +32 76 4.500000000000000e+01 +32 77 4.600000000000000e+01 +32 78 4.700000000000000e+01 +32 79 4.800000000000000e+01 +32 80 4.900000000000000e+01 +32 81 5.000000000000000e+01 +32 82 5.100000000000000e+01 +32 83 5.200000000000000e+01 +32 84 5.300000000000000e+01 +32 85 5.400000000000000e+01 +32 86 5.500000000000000e+01 +32 87 5.600000000000000e+01 +32 88 5.700000000000000e+01 +32 89 5.800000000000000e+01 +32 90 5.900000000000000e+01 +32 91 6.000000000000000e+01 +32 92 6.100000000000000e+01 +32 93 6.200000000000000e+01 +32 94 6.300000000000000e+01 +32 95 6.400000000000000e+01 +32 96 6.500000000000000e+01 +32 97 6.600000000000000e+01 +32 98 6.700000000000000e+01 +32 99 6.800000000000000e+01 +32 100 6.900000000000000e+01 +33 1 3.300000000000000e+01 +33 2 3.200000000000000e+01 +33 3 3.100000000000000e+01 +33 4 3.000000000000000e+01 +33 5 2.900000000000000e+01 +33 6 2.800000000000000e+01 +33 7 2.700000000000000e+01 +33 8 2.600000000000000e+01 +33 9 2.500000000000000e+01 +33 10 2.400000000000000e+01 +33 11 2.300000000000000e+01 +33 12 2.200000000000000e+01 +33 13 2.100000000000000e+01 +33 14 2.000000000000000e+01 +33 15 1.900000000000000e+01 +33 16 1.800000000000000e+01 +33 17 1.700000000000000e+01 +33 18 1.600000000000000e+01 +33 19 1.500000000000000e+01 +33 20 1.400000000000000e+01 +33 21 1.300000000000000e+01 +33 22 1.200000000000000e+01 +33 23 1.100000000000000e+01 +33 24 1.000000000000000e+01 +33 25 9.000000000000000e+00 +33 26 8.000000000000000e+00 +33 27 7.000000000000000e+00 +33 28 6.000000000000000e+00 +33 29 5.000000000000000e+00 +33 30 4.000000000000000e+00 +33 31 3.000000000000000e+00 +33 32 2.000000000000000e+00 +33 33 2.000000000000000e+00 +33 34 2.000000000000000e+00 +33 35 3.000000000000000e+00 +33 36 4.000000000000000e+00 +33 37 5.000000000000000e+00 +33 38 6.000000000000000e+00 +33 39 7.000000000000000e+00 +33 40 8.000000000000000e+00 +33 41 9.000000000000000e+00 +33 42 1.000000000000000e+01 +33 43 1.100000000000000e+01 +33 44 1.200000000000000e+01 +33 45 1.300000000000000e+01 +33 46 1.400000000000000e+01 +33 47 1.500000000000000e+01 +33 48 1.600000000000000e+01 +33 49 1.700000000000000e+01 +33 50 1.800000000000000e+01 +33 51 1.900000000000000e+01 +33 52 2.000000000000000e+01 +33 53 2.100000000000000e+01 +33 54 2.200000000000000e+01 +33 55 2.300000000000000e+01 +33 56 2.400000000000000e+01 +33 57 2.500000000000000e+01 +33 58 2.600000000000000e+01 +33 59 2.700000000000000e+01 +33 60 2.800000000000000e+01 +33 61 2.900000000000000e+01 +33 62 3.000000000000000e+01 +33 63 3.100000000000000e+01 +33 64 3.200000000000000e+01 +33 65 3.300000000000000e+01 +33 66 3.400000000000000e+01 +33 67 3.500000000000000e+01 +33 68 3.600000000000000e+01 +33 69 3.700000000000000e+01 +33 70 3.800000000000000e+01 +33 71 3.900000000000000e+01 +33 72 4.000000000000000e+01 +33 73 4.100000000000000e+01 +33 74 4.200000000000000e+01 +33 75 4.300000000000000e+01 +33 76 4.400000000000000e+01 +33 77 4.500000000000000e+01 +33 78 4.600000000000000e+01 +33 79 4.700000000000000e+01 +33 80 4.800000000000000e+01 +33 81 4.900000000000000e+01 +33 82 5.000000000000000e+01 +33 83 5.100000000000000e+01 +33 84 5.200000000000000e+01 +33 85 5.300000000000000e+01 +33 86 5.400000000000000e+01 +33 87 5.500000000000000e+01 +33 88 5.600000000000000e+01 +33 89 5.700000000000000e+01 +33 90 5.800000000000000e+01 +33 91 5.900000000000000e+01 +33 92 6.000000000000000e+01 +33 93 6.100000000000000e+01 +33 94 6.200000000000000e+01 +33 95 6.300000000000000e+01 +33 96 6.400000000000000e+01 +33 97 6.500000000000000e+01 +33 98 6.600000000000000e+01 +33 99 6.700000000000000e+01 +33 100 6.800000000000000e+01 +34 1 3.400000000000000e+01 +34 2 3.300000000000000e+01 +34 3 3.200000000000000e+01 +34 4 3.100000000000000e+01 +34 5 3.000000000000000e+01 +34 6 2.900000000000000e+01 +34 7 2.800000000000000e+01 +34 8 2.700000000000000e+01 +34 9 2.600000000000000e+01 +34 10 2.500000000000000e+01 +34 11 2.400000000000000e+01 +34 12 2.300000000000000e+01 +34 13 2.200000000000000e+01 +34 14 2.100000000000000e+01 +34 15 2.000000000000000e+01 +34 16 1.900000000000000e+01 +34 17 1.800000000000000e+01 +34 18 1.700000000000000e+01 +34 19 1.600000000000000e+01 +34 20 1.500000000000000e+01 +34 21 1.400000000000000e+01 +34 22 1.300000000000000e+01 +34 23 1.200000000000000e+01 +34 24 1.100000000000000e+01 +34 25 1.000000000000000e+01 +34 26 9.000000000000000e+00 +34 27 8.000000000000000e+00 +34 28 7.000000000000000e+00 +34 29 6.000000000000000e+00 +34 30 5.000000000000000e+00 +34 31 4.000000000000000e+00 +34 32 3.000000000000000e+00 +34 33 2.000000000000000e+00 +34 34 2.000000000000000e+00 +34 35 2.000000000000000e+00 +34 36 3.000000000000000e+00 +34 37 4.000000000000000e+00 +34 38 5.000000000000000e+00 +34 39 6.000000000000000e+00 +34 40 7.000000000000000e+00 +34 41 8.000000000000000e+00 +34 42 9.000000000000000e+00 +34 43 1.000000000000000e+01 +34 44 1.100000000000000e+01 +34 45 1.200000000000000e+01 +34 46 1.300000000000000e+01 +34 47 1.400000000000000e+01 +34 48 1.500000000000000e+01 +34 49 1.600000000000000e+01 +34 50 1.700000000000000e+01 +34 51 1.800000000000000e+01 +34 52 1.900000000000000e+01 +34 53 2.000000000000000e+01 +34 54 2.100000000000000e+01 +34 55 2.200000000000000e+01 +34 56 2.300000000000000e+01 +34 57 2.400000000000000e+01 +34 58 2.500000000000000e+01 +34 59 2.600000000000000e+01 +34 60 2.700000000000000e+01 +34 61 2.800000000000000e+01 +34 62 2.900000000000000e+01 +34 63 3.000000000000000e+01 +34 64 3.100000000000000e+01 +34 65 3.200000000000000e+01 +34 66 3.300000000000000e+01 +34 67 3.400000000000000e+01 +34 68 3.500000000000000e+01 +34 69 3.600000000000000e+01 +34 70 3.700000000000000e+01 +34 71 3.800000000000000e+01 +34 72 3.900000000000000e+01 +34 73 4.000000000000000e+01 +34 74 4.100000000000000e+01 +34 75 4.200000000000000e+01 +34 76 4.300000000000000e+01 +34 77 4.400000000000000e+01 +34 78 4.500000000000000e+01 +34 79 4.600000000000000e+01 +34 80 4.700000000000000e+01 +34 81 4.800000000000000e+01 +34 82 4.900000000000000e+01 +34 83 5.000000000000000e+01 +34 84 5.100000000000000e+01 +34 85 5.200000000000000e+01 +34 86 5.300000000000000e+01 +34 87 5.400000000000000e+01 +34 88 5.500000000000000e+01 +34 89 5.600000000000000e+01 +34 90 5.700000000000000e+01 +34 91 5.800000000000000e+01 +34 92 5.900000000000000e+01 +34 93 6.000000000000000e+01 +34 94 6.100000000000000e+01 +34 95 6.200000000000000e+01 +34 96 6.300000000000000e+01 +34 97 6.400000000000000e+01 +34 98 6.500000000000000e+01 +34 99 6.600000000000000e+01 +34 100 6.700000000000000e+01 +35 1 3.500000000000000e+01 +35 2 3.400000000000000e+01 +35 3 3.300000000000000e+01 +35 4 3.200000000000000e+01 +35 5 3.100000000000000e+01 +35 6 3.000000000000000e+01 +35 7 2.900000000000000e+01 +35 8 2.800000000000000e+01 +35 9 2.700000000000000e+01 +35 10 2.600000000000000e+01 +35 11 2.500000000000000e+01 +35 12 2.400000000000000e+01 +35 13 2.300000000000000e+01 +35 14 2.200000000000000e+01 +35 15 2.100000000000000e+01 +35 16 2.000000000000000e+01 +35 17 1.900000000000000e+01 +35 18 1.800000000000000e+01 +35 19 1.700000000000000e+01 +35 20 1.600000000000000e+01 +35 21 1.500000000000000e+01 +35 22 1.400000000000000e+01 +35 23 1.300000000000000e+01 +35 24 1.200000000000000e+01 +35 25 1.100000000000000e+01 +35 26 1.000000000000000e+01 +35 27 9.000000000000000e+00 +35 28 8.000000000000000e+00 +35 29 7.000000000000000e+00 +35 30 6.000000000000000e+00 +35 31 5.000000000000000e+00 +35 32 4.000000000000000e+00 +35 33 3.000000000000000e+00 +35 34 2.000000000000000e+00 +35 35 2.000000000000000e+00 +35 36 2.000000000000000e+00 +35 37 3.000000000000000e+00 +35 38 4.000000000000000e+00 +35 39 5.000000000000000e+00 +35 40 6.000000000000000e+00 +35 41 7.000000000000000e+00 +35 42 8.000000000000000e+00 +35 43 9.000000000000000e+00 +35 44 1.000000000000000e+01 +35 45 1.100000000000000e+01 +35 46 1.200000000000000e+01 +35 47 1.300000000000000e+01 +35 48 1.400000000000000e+01 +35 49 1.500000000000000e+01 +35 50 1.600000000000000e+01 +35 51 1.700000000000000e+01 +35 52 1.800000000000000e+01 +35 53 1.900000000000000e+01 +35 54 2.000000000000000e+01 +35 55 2.100000000000000e+01 +35 56 2.200000000000000e+01 +35 57 2.300000000000000e+01 +35 58 2.400000000000000e+01 +35 59 2.500000000000000e+01 +35 60 2.600000000000000e+01 +35 61 2.700000000000000e+01 +35 62 2.800000000000000e+01 +35 63 2.900000000000000e+01 +35 64 3.000000000000000e+01 +35 65 3.100000000000000e+01 +35 66 3.200000000000000e+01 +35 67 3.300000000000000e+01 +35 68 3.400000000000000e+01 +35 69 3.500000000000000e+01 +35 70 3.600000000000000e+01 +35 71 3.700000000000000e+01 +35 72 3.800000000000000e+01 +35 73 3.900000000000000e+01 +35 74 4.000000000000000e+01 +35 75 4.100000000000000e+01 +35 76 4.200000000000000e+01 +35 77 4.300000000000000e+01 +35 78 4.400000000000000e+01 +35 79 4.500000000000000e+01 +35 80 4.600000000000000e+01 +35 81 4.700000000000000e+01 +35 82 4.800000000000000e+01 +35 83 4.900000000000000e+01 +35 84 5.000000000000000e+01 +35 85 5.100000000000000e+01 +35 86 5.200000000000000e+01 +35 87 5.300000000000000e+01 +35 88 5.400000000000000e+01 +35 89 5.500000000000000e+01 +35 90 5.600000000000000e+01 +35 91 5.700000000000000e+01 +35 92 5.800000000000000e+01 +35 93 5.900000000000000e+01 +35 94 6.000000000000000e+01 +35 95 6.100000000000000e+01 +35 96 6.200000000000000e+01 +35 97 6.300000000000000e+01 +35 98 6.400000000000000e+01 +35 99 6.500000000000000e+01 +35 100 6.600000000000000e+01 +36 1 3.600000000000000e+01 +36 2 3.500000000000000e+01 +36 3 3.400000000000000e+01 +36 4 3.300000000000000e+01 +36 5 3.200000000000000e+01 +36 6 3.100000000000000e+01 +36 7 3.000000000000000e+01 +36 8 2.900000000000000e+01 +36 9 2.800000000000000e+01 +36 10 2.700000000000000e+01 +36 11 2.600000000000000e+01 +36 12 2.500000000000000e+01 +36 13 2.400000000000000e+01 +36 14 2.300000000000000e+01 +36 15 2.200000000000000e+01 +36 16 2.100000000000000e+01 +36 17 2.000000000000000e+01 +36 18 1.900000000000000e+01 +36 19 1.800000000000000e+01 +36 20 1.700000000000000e+01 +36 21 1.600000000000000e+01 +36 22 1.500000000000000e+01 +36 23 1.400000000000000e+01 +36 24 1.300000000000000e+01 +36 25 1.200000000000000e+01 +36 26 1.100000000000000e+01 +36 27 1.000000000000000e+01 +36 28 9.000000000000000e+00 +36 29 8.000000000000000e+00 +36 30 7.000000000000000e+00 +36 31 6.000000000000000e+00 +36 32 5.000000000000000e+00 +36 33 4.000000000000000e+00 +36 34 3.000000000000000e+00 +36 35 2.000000000000000e+00 +36 36 2.000000000000000e+00 +36 37 2.000000000000000e+00 +36 38 3.000000000000000e+00 +36 39 4.000000000000000e+00 +36 40 5.000000000000000e+00 +36 41 6.000000000000000e+00 +36 42 7.000000000000000e+00 +36 43 8.000000000000000e+00 +36 44 9.000000000000000e+00 +36 45 1.000000000000000e+01 +36 46 1.100000000000000e+01 +36 47 1.200000000000000e+01 +36 48 1.300000000000000e+01 +36 49 1.400000000000000e+01 +36 50 1.500000000000000e+01 +36 51 1.600000000000000e+01 +36 52 1.700000000000000e+01 +36 53 1.800000000000000e+01 +36 54 1.900000000000000e+01 +36 55 2.000000000000000e+01 +36 56 2.100000000000000e+01 +36 57 2.200000000000000e+01 +36 58 2.300000000000000e+01 +36 59 2.400000000000000e+01 +36 60 2.500000000000000e+01 +36 61 2.600000000000000e+01 +36 62 2.700000000000000e+01 +36 63 2.800000000000000e+01 +36 64 2.900000000000000e+01 +36 65 3.000000000000000e+01 +36 66 3.100000000000000e+01 +36 67 3.200000000000000e+01 +36 68 3.300000000000000e+01 +36 69 3.400000000000000e+01 +36 70 3.500000000000000e+01 +36 71 3.600000000000000e+01 +36 72 3.700000000000000e+01 +36 73 3.800000000000000e+01 +36 74 3.900000000000000e+01 +36 75 4.000000000000000e+01 +36 76 4.100000000000000e+01 +36 77 4.200000000000000e+01 +36 78 4.300000000000000e+01 +36 79 4.400000000000000e+01 +36 80 4.500000000000000e+01 +36 81 4.600000000000000e+01 +36 82 4.700000000000000e+01 +36 83 4.800000000000000e+01 +36 84 4.900000000000000e+01 +36 85 5.000000000000000e+01 +36 86 5.100000000000000e+01 +36 87 5.200000000000000e+01 +36 88 5.300000000000000e+01 +36 89 5.400000000000000e+01 +36 90 5.500000000000000e+01 +36 91 5.600000000000000e+01 +36 92 5.700000000000000e+01 +36 93 5.800000000000000e+01 +36 94 5.900000000000000e+01 +36 95 6.000000000000000e+01 +36 96 6.100000000000000e+01 +36 97 6.200000000000000e+01 +36 98 6.300000000000000e+01 +36 99 6.400000000000000e+01 +36 100 6.500000000000000e+01 +37 1 3.700000000000000e+01 +37 2 3.600000000000000e+01 +37 3 3.500000000000000e+01 +37 4 3.400000000000000e+01 +37 5 3.300000000000000e+01 +37 6 3.200000000000000e+01 +37 7 3.100000000000000e+01 +37 8 3.000000000000000e+01 +37 9 2.900000000000000e+01 +37 10 2.800000000000000e+01 +37 11 2.700000000000000e+01 +37 12 2.600000000000000e+01 +37 13 2.500000000000000e+01 +37 14 2.400000000000000e+01 +37 15 2.300000000000000e+01 +37 16 2.200000000000000e+01 +37 17 2.100000000000000e+01 +37 18 2.000000000000000e+01 +37 19 1.900000000000000e+01 +37 20 1.800000000000000e+01 +37 21 1.700000000000000e+01 +37 22 1.600000000000000e+01 +37 23 1.500000000000000e+01 +37 24 1.400000000000000e+01 +37 25 1.300000000000000e+01 +37 26 1.200000000000000e+01 +37 27 1.100000000000000e+01 +37 28 1.000000000000000e+01 +37 29 9.000000000000000e+00 +37 30 8.000000000000000e+00 +37 31 7.000000000000000e+00 +37 32 6.000000000000000e+00 +37 33 5.000000000000000e+00 +37 34 4.000000000000000e+00 +37 35 3.000000000000000e+00 +37 36 2.000000000000000e+00 +37 37 2.000000000000000e+00 +37 38 2.000000000000000e+00 +37 39 3.000000000000000e+00 +37 40 4.000000000000000e+00 +37 41 5.000000000000000e+00 +37 42 6.000000000000000e+00 +37 43 7.000000000000000e+00 +37 44 8.000000000000000e+00 +37 45 9.000000000000000e+00 +37 46 1.000000000000000e+01 +37 47 1.100000000000000e+01 +37 48 1.200000000000000e+01 +37 49 1.300000000000000e+01 +37 50 1.400000000000000e+01 +37 51 1.500000000000000e+01 +37 52 1.600000000000000e+01 +37 53 1.700000000000000e+01 +37 54 1.800000000000000e+01 +37 55 1.900000000000000e+01 +37 56 2.000000000000000e+01 +37 57 2.100000000000000e+01 +37 58 2.200000000000000e+01 +37 59 2.300000000000000e+01 +37 60 2.400000000000000e+01 +37 61 2.500000000000000e+01 +37 62 2.600000000000000e+01 +37 63 2.700000000000000e+01 +37 64 2.800000000000000e+01 +37 65 2.900000000000000e+01 +37 66 3.000000000000000e+01 +37 67 3.100000000000000e+01 +37 68 3.200000000000000e+01 +37 69 3.300000000000000e+01 +37 70 3.400000000000000e+01 +37 71 3.500000000000000e+01 +37 72 3.600000000000000e+01 +37 73 3.700000000000000e+01 +37 74 3.800000000000000e+01 +37 75 3.900000000000000e+01 +37 76 4.000000000000000e+01 +37 77 4.100000000000000e+01 +37 78 4.200000000000000e+01 +37 79 4.300000000000000e+01 +37 80 4.400000000000000e+01 +37 81 4.500000000000000e+01 +37 82 4.600000000000000e+01 +37 83 4.700000000000000e+01 +37 84 4.800000000000000e+01 +37 85 4.900000000000000e+01 +37 86 5.000000000000000e+01 +37 87 5.100000000000000e+01 +37 88 5.200000000000000e+01 +37 89 5.300000000000000e+01 +37 90 5.400000000000000e+01 +37 91 5.500000000000000e+01 +37 92 5.600000000000000e+01 +37 93 5.700000000000000e+01 +37 94 5.800000000000000e+01 +37 95 5.900000000000000e+01 +37 96 6.000000000000000e+01 +37 97 6.100000000000000e+01 +37 98 6.200000000000000e+01 +37 99 6.300000000000000e+01 +37 100 6.400000000000000e+01 +38 1 3.800000000000000e+01 +38 2 3.700000000000000e+01 +38 3 3.600000000000000e+01 +38 4 3.500000000000000e+01 +38 5 3.400000000000000e+01 +38 6 3.300000000000000e+01 +38 7 3.200000000000000e+01 +38 8 3.100000000000000e+01 +38 9 3.000000000000000e+01 +38 10 2.900000000000000e+01 +38 11 2.800000000000000e+01 +38 12 2.700000000000000e+01 +38 13 2.600000000000000e+01 +38 14 2.500000000000000e+01 +38 15 2.400000000000000e+01 +38 16 2.300000000000000e+01 +38 17 2.200000000000000e+01 +38 18 2.100000000000000e+01 +38 19 2.000000000000000e+01 +38 20 1.900000000000000e+01 +38 21 1.800000000000000e+01 +38 22 1.700000000000000e+01 +38 23 1.600000000000000e+01 +38 24 1.500000000000000e+01 +38 25 1.400000000000000e+01 +38 26 1.300000000000000e+01 +38 27 1.200000000000000e+01 +38 28 1.100000000000000e+01 +38 29 1.000000000000000e+01 +38 30 9.000000000000000e+00 +38 31 8.000000000000000e+00 +38 32 7.000000000000000e+00 +38 33 6.000000000000000e+00 +38 34 5.000000000000000e+00 +38 35 4.000000000000000e+00 +38 36 3.000000000000000e+00 +38 37 2.000000000000000e+00 +38 38 2.000000000000000e+00 +38 39 2.000000000000000e+00 +38 40 3.000000000000000e+00 +38 41 4.000000000000000e+00 +38 42 5.000000000000000e+00 +38 43 6.000000000000000e+00 +38 44 7.000000000000000e+00 +38 45 8.000000000000000e+00 +38 46 9.000000000000000e+00 +38 47 1.000000000000000e+01 +38 48 1.100000000000000e+01 +38 49 1.200000000000000e+01 +38 50 1.300000000000000e+01 +38 51 1.400000000000000e+01 +38 52 1.500000000000000e+01 +38 53 1.600000000000000e+01 +38 54 1.700000000000000e+01 +38 55 1.800000000000000e+01 +38 56 1.900000000000000e+01 +38 57 2.000000000000000e+01 +38 58 2.100000000000000e+01 +38 59 2.200000000000000e+01 +38 60 2.300000000000000e+01 +38 61 2.400000000000000e+01 +38 62 2.500000000000000e+01 +38 63 2.600000000000000e+01 +38 64 2.700000000000000e+01 +38 65 2.800000000000000e+01 +38 66 2.900000000000000e+01 +38 67 3.000000000000000e+01 +38 68 3.100000000000000e+01 +38 69 3.200000000000000e+01 +38 70 3.300000000000000e+01 +38 71 3.400000000000000e+01 +38 72 3.500000000000000e+01 +38 73 3.600000000000000e+01 +38 74 3.700000000000000e+01 +38 75 3.800000000000000e+01 +38 76 3.900000000000000e+01 +38 77 4.000000000000000e+01 +38 78 4.100000000000000e+01 +38 79 4.200000000000000e+01 +38 80 4.300000000000000e+01 +38 81 4.400000000000000e+01 +38 82 4.500000000000000e+01 +38 83 4.600000000000000e+01 +38 84 4.700000000000000e+01 +38 85 4.800000000000000e+01 +38 86 4.900000000000000e+01 +38 87 5.000000000000000e+01 +38 88 5.100000000000000e+01 +38 89 5.200000000000000e+01 +38 90 5.300000000000000e+01 +38 91 5.400000000000000e+01 +38 92 5.500000000000000e+01 +38 93 5.600000000000000e+01 +38 94 5.700000000000000e+01 +38 95 5.800000000000000e+01 +38 96 5.900000000000000e+01 +38 97 6.000000000000000e+01 +38 98 6.100000000000000e+01 +38 99 6.200000000000000e+01 +38 100 6.300000000000000e+01 +39 1 3.900000000000000e+01 +39 2 3.800000000000000e+01 +39 3 3.700000000000000e+01 +39 4 3.600000000000000e+01 +39 5 3.500000000000000e+01 +39 6 3.400000000000000e+01 +39 7 3.300000000000000e+01 +39 8 3.200000000000000e+01 +39 9 3.100000000000000e+01 +39 10 3.000000000000000e+01 +39 11 2.900000000000000e+01 +39 12 2.800000000000000e+01 +39 13 2.700000000000000e+01 +39 14 2.600000000000000e+01 +39 15 2.500000000000000e+01 +39 16 2.400000000000000e+01 +39 17 2.300000000000000e+01 +39 18 2.200000000000000e+01 +39 19 2.100000000000000e+01 +39 20 2.000000000000000e+01 +39 21 1.900000000000000e+01 +39 22 1.800000000000000e+01 +39 23 1.700000000000000e+01 +39 24 1.600000000000000e+01 +39 25 1.500000000000000e+01 +39 26 1.400000000000000e+01 +39 27 1.300000000000000e+01 +39 28 1.200000000000000e+01 +39 29 1.100000000000000e+01 +39 30 1.000000000000000e+01 +39 31 9.000000000000000e+00 +39 32 8.000000000000000e+00 +39 33 7.000000000000000e+00 +39 34 6.000000000000000e+00 +39 35 5.000000000000000e+00 +39 36 4.000000000000000e+00 +39 37 3.000000000000000e+00 +39 38 2.000000000000000e+00 +39 39 2.000000000000000e+00 +39 40 2.000000000000000e+00 +39 41 3.000000000000000e+00 +39 42 4.000000000000000e+00 +39 43 5.000000000000000e+00 +39 44 6.000000000000000e+00 +39 45 7.000000000000000e+00 +39 46 8.000000000000000e+00 +39 47 9.000000000000000e+00 +39 48 1.000000000000000e+01 +39 49 1.100000000000000e+01 +39 50 1.200000000000000e+01 +39 51 1.300000000000000e+01 +39 52 1.400000000000000e+01 +39 53 1.500000000000000e+01 +39 54 1.600000000000000e+01 +39 55 1.700000000000000e+01 +39 56 1.800000000000000e+01 +39 57 1.900000000000000e+01 +39 58 2.000000000000000e+01 +39 59 2.100000000000000e+01 +39 60 2.200000000000000e+01 +39 61 2.300000000000000e+01 +39 62 2.400000000000000e+01 +39 63 2.500000000000000e+01 +39 64 2.600000000000000e+01 +39 65 2.700000000000000e+01 +39 66 2.800000000000000e+01 +39 67 2.900000000000000e+01 +39 68 3.000000000000000e+01 +39 69 3.100000000000000e+01 +39 70 3.200000000000000e+01 +39 71 3.300000000000000e+01 +39 72 3.400000000000000e+01 +39 73 3.500000000000000e+01 +39 74 3.600000000000000e+01 +39 75 3.700000000000000e+01 +39 76 3.800000000000000e+01 +39 77 3.900000000000000e+01 +39 78 4.000000000000000e+01 +39 79 4.100000000000000e+01 +39 80 4.200000000000000e+01 +39 81 4.300000000000000e+01 +39 82 4.400000000000000e+01 +39 83 4.500000000000000e+01 +39 84 4.600000000000000e+01 +39 85 4.700000000000000e+01 +39 86 4.800000000000000e+01 +39 87 4.900000000000000e+01 +39 88 5.000000000000000e+01 +39 89 5.100000000000000e+01 +39 90 5.200000000000000e+01 +39 91 5.300000000000000e+01 +39 92 5.400000000000000e+01 +39 93 5.500000000000000e+01 +39 94 5.600000000000000e+01 +39 95 5.700000000000000e+01 +39 96 5.800000000000000e+01 +39 97 5.900000000000000e+01 +39 98 6.000000000000000e+01 +39 99 6.100000000000000e+01 +39 100 6.200000000000000e+01 +40 1 4.000000000000000e+01 +40 2 3.900000000000000e+01 +40 3 3.800000000000000e+01 +40 4 3.700000000000000e+01 +40 5 3.600000000000000e+01 +40 6 3.500000000000000e+01 +40 7 3.400000000000000e+01 +40 8 3.300000000000000e+01 +40 9 3.200000000000000e+01 +40 10 3.100000000000000e+01 +40 11 3.000000000000000e+01 +40 12 2.900000000000000e+01 +40 13 2.800000000000000e+01 +40 14 2.700000000000000e+01 +40 15 2.600000000000000e+01 +40 16 2.500000000000000e+01 +40 17 2.400000000000000e+01 +40 18 2.300000000000000e+01 +40 19 2.200000000000000e+01 +40 20 2.100000000000000e+01 +40 21 2.000000000000000e+01 +40 22 1.900000000000000e+01 +40 23 1.800000000000000e+01 +40 24 1.700000000000000e+01 +40 25 1.600000000000000e+01 +40 26 1.500000000000000e+01 +40 27 1.400000000000000e+01 +40 28 1.300000000000000e+01 +40 29 1.200000000000000e+01 +40 30 1.100000000000000e+01 +40 31 1.000000000000000e+01 +40 32 9.000000000000000e+00 +40 33 8.000000000000000e+00 +40 34 7.000000000000000e+00 +40 35 6.000000000000000e+00 +40 36 5.000000000000000e+00 +40 37 4.000000000000000e+00 +40 38 3.000000000000000e+00 +40 39 2.000000000000000e+00 +40 40 2.000000000000000e+00 +40 41 2.000000000000000e+00 +40 42 3.000000000000000e+00 +40 43 4.000000000000000e+00 +40 44 5.000000000000000e+00 +40 45 6.000000000000000e+00 +40 46 7.000000000000000e+00 +40 47 8.000000000000000e+00 +40 48 9.000000000000000e+00 +40 49 1.000000000000000e+01 +40 50 1.100000000000000e+01 +40 51 1.200000000000000e+01 +40 52 1.300000000000000e+01 +40 53 1.400000000000000e+01 +40 54 1.500000000000000e+01 +40 55 1.600000000000000e+01 +40 56 1.700000000000000e+01 +40 57 1.800000000000000e+01 +40 58 1.900000000000000e+01 +40 59 2.000000000000000e+01 +40 60 2.100000000000000e+01 +40 61 2.200000000000000e+01 +40 62 2.300000000000000e+01 +40 63 2.400000000000000e+01 +40 64 2.500000000000000e+01 +40 65 2.600000000000000e+01 +40 66 2.700000000000000e+01 +40 67 2.800000000000000e+01 +40 68 2.900000000000000e+01 +40 69 3.000000000000000e+01 +40 70 3.100000000000000e+01 +40 71 3.200000000000000e+01 +40 72 3.300000000000000e+01 +40 73 3.400000000000000e+01 +40 74 3.500000000000000e+01 +40 75 3.600000000000000e+01 +40 76 3.700000000000000e+01 +40 77 3.800000000000000e+01 +40 78 3.900000000000000e+01 +40 79 4.000000000000000e+01 +40 80 4.100000000000000e+01 +40 81 4.200000000000000e+01 +40 82 4.300000000000000e+01 +40 83 4.400000000000000e+01 +40 84 4.500000000000000e+01 +40 85 4.600000000000000e+01 +40 86 4.700000000000000e+01 +40 87 4.800000000000000e+01 +40 88 4.900000000000000e+01 +40 89 5.000000000000000e+01 +40 90 5.100000000000000e+01 +40 91 5.200000000000000e+01 +40 92 5.300000000000000e+01 +40 93 5.400000000000000e+01 +40 94 5.500000000000000e+01 +40 95 5.600000000000000e+01 +40 96 5.700000000000000e+01 +40 97 5.800000000000000e+01 +40 98 5.900000000000000e+01 +40 99 6.000000000000000e+01 +40 100 6.100000000000000e+01 +41 1 4.100000000000000e+01 +41 2 4.000000000000000e+01 +41 3 3.900000000000000e+01 +41 4 3.800000000000000e+01 +41 5 3.700000000000000e+01 +41 6 3.600000000000000e+01 +41 7 3.500000000000000e+01 +41 8 3.400000000000000e+01 +41 9 3.300000000000000e+01 +41 10 3.200000000000000e+01 +41 11 3.100000000000000e+01 +41 12 3.000000000000000e+01 +41 13 2.900000000000000e+01 +41 14 2.800000000000000e+01 +41 15 2.700000000000000e+01 +41 16 2.600000000000000e+01 +41 17 2.500000000000000e+01 +41 18 2.400000000000000e+01 +41 19 2.300000000000000e+01 +41 20 2.200000000000000e+01 +41 21 2.100000000000000e+01 +41 22 2.000000000000000e+01 +41 23 1.900000000000000e+01 +41 24 1.800000000000000e+01 +41 25 1.700000000000000e+01 +41 26 1.600000000000000e+01 +41 27 1.500000000000000e+01 +41 28 1.400000000000000e+01 +41 29 1.300000000000000e+01 +41 30 1.200000000000000e+01 +41 31 1.100000000000000e+01 +41 32 1.000000000000000e+01 +41 33 9.000000000000000e+00 +41 34 8.000000000000000e+00 +41 35 7.000000000000000e+00 +41 36 6.000000000000000e+00 +41 37 5.000000000000000e+00 +41 38 4.000000000000000e+00 +41 39 3.000000000000000e+00 +41 40 2.000000000000000e+00 +41 41 2.000000000000000e+00 +41 42 2.000000000000000e+00 +41 43 3.000000000000000e+00 +41 44 4.000000000000000e+00 +41 45 5.000000000000000e+00 +41 46 6.000000000000000e+00 +41 47 7.000000000000000e+00 +41 48 8.000000000000000e+00 +41 49 9.000000000000000e+00 +41 50 1.000000000000000e+01 +41 51 1.100000000000000e+01 +41 52 1.200000000000000e+01 +41 53 1.300000000000000e+01 +41 54 1.400000000000000e+01 +41 55 1.500000000000000e+01 +41 56 1.600000000000000e+01 +41 57 1.700000000000000e+01 +41 58 1.800000000000000e+01 +41 59 1.900000000000000e+01 +41 60 2.000000000000000e+01 +41 61 2.100000000000000e+01 +41 62 2.200000000000000e+01 +41 63 2.300000000000000e+01 +41 64 2.400000000000000e+01 +41 65 2.500000000000000e+01 +41 66 2.600000000000000e+01 +41 67 2.700000000000000e+01 +41 68 2.800000000000000e+01 +41 69 2.900000000000000e+01 +41 70 3.000000000000000e+01 +41 71 3.100000000000000e+01 +41 72 3.200000000000000e+01 +41 73 3.300000000000000e+01 +41 74 3.400000000000000e+01 +41 75 3.500000000000000e+01 +41 76 3.600000000000000e+01 +41 77 3.700000000000000e+01 +41 78 3.800000000000000e+01 +41 79 3.900000000000000e+01 +41 80 4.000000000000000e+01 +41 81 4.100000000000000e+01 +41 82 4.200000000000000e+01 +41 83 4.300000000000000e+01 +41 84 4.400000000000000e+01 +41 85 4.500000000000000e+01 +41 86 4.600000000000000e+01 +41 87 4.700000000000000e+01 +41 88 4.800000000000000e+01 +41 89 4.900000000000000e+01 +41 90 5.000000000000000e+01 +41 91 5.100000000000000e+01 +41 92 5.200000000000000e+01 +41 93 5.300000000000000e+01 +41 94 5.400000000000000e+01 +41 95 5.500000000000000e+01 +41 96 5.600000000000000e+01 +41 97 5.700000000000000e+01 +41 98 5.800000000000000e+01 +41 99 5.900000000000000e+01 +41 100 6.000000000000000e+01 +42 1 4.200000000000000e+01 +42 2 4.100000000000000e+01 +42 3 4.000000000000000e+01 +42 4 3.900000000000000e+01 +42 5 3.800000000000000e+01 +42 6 3.700000000000000e+01 +42 7 3.600000000000000e+01 +42 8 3.500000000000000e+01 +42 9 3.400000000000000e+01 +42 10 3.300000000000000e+01 +42 11 3.200000000000000e+01 +42 12 3.100000000000000e+01 +42 13 3.000000000000000e+01 +42 14 2.900000000000000e+01 +42 15 2.800000000000000e+01 +42 16 2.700000000000000e+01 +42 17 2.600000000000000e+01 +42 18 2.500000000000000e+01 +42 19 2.400000000000000e+01 +42 20 2.300000000000000e+01 +42 21 2.200000000000000e+01 +42 22 2.100000000000000e+01 +42 23 2.000000000000000e+01 +42 24 1.900000000000000e+01 +42 25 1.800000000000000e+01 +42 26 1.700000000000000e+01 +42 27 1.600000000000000e+01 +42 28 1.500000000000000e+01 +42 29 1.400000000000000e+01 +42 30 1.300000000000000e+01 +42 31 1.200000000000000e+01 +42 32 1.100000000000000e+01 +42 33 1.000000000000000e+01 +42 34 9.000000000000000e+00 +42 35 8.000000000000000e+00 +42 36 7.000000000000000e+00 +42 37 6.000000000000000e+00 +42 38 5.000000000000000e+00 +42 39 4.000000000000000e+00 +42 40 3.000000000000000e+00 +42 41 2.000000000000000e+00 +42 42 2.000000000000000e+00 +42 43 2.000000000000000e+00 +42 44 3.000000000000000e+00 +42 45 4.000000000000000e+00 +42 46 5.000000000000000e+00 +42 47 6.000000000000000e+00 +42 48 7.000000000000000e+00 +42 49 8.000000000000000e+00 +42 50 9.000000000000000e+00 +42 51 1.000000000000000e+01 +42 52 1.100000000000000e+01 +42 53 1.200000000000000e+01 +42 54 1.300000000000000e+01 +42 55 1.400000000000000e+01 +42 56 1.500000000000000e+01 +42 57 1.600000000000000e+01 +42 58 1.700000000000000e+01 +42 59 1.800000000000000e+01 +42 60 1.900000000000000e+01 +42 61 2.000000000000000e+01 +42 62 2.100000000000000e+01 +42 63 2.200000000000000e+01 +42 64 2.300000000000000e+01 +42 65 2.400000000000000e+01 +42 66 2.500000000000000e+01 +42 67 2.600000000000000e+01 +42 68 2.700000000000000e+01 +42 69 2.800000000000000e+01 +42 70 2.900000000000000e+01 +42 71 3.000000000000000e+01 +42 72 3.100000000000000e+01 +42 73 3.200000000000000e+01 +42 74 3.300000000000000e+01 +42 75 3.400000000000000e+01 +42 76 3.500000000000000e+01 +42 77 3.600000000000000e+01 +42 78 3.700000000000000e+01 +42 79 3.800000000000000e+01 +42 80 3.900000000000000e+01 +42 81 4.000000000000000e+01 +42 82 4.100000000000000e+01 +42 83 4.200000000000000e+01 +42 84 4.300000000000000e+01 +42 85 4.400000000000000e+01 +42 86 4.500000000000000e+01 +42 87 4.600000000000000e+01 +42 88 4.700000000000000e+01 +42 89 4.800000000000000e+01 +42 90 4.900000000000000e+01 +42 91 5.000000000000000e+01 +42 92 5.100000000000000e+01 +42 93 5.200000000000000e+01 +42 94 5.300000000000000e+01 +42 95 5.400000000000000e+01 +42 96 5.500000000000000e+01 +42 97 5.600000000000000e+01 +42 98 5.700000000000000e+01 +42 99 5.800000000000000e+01 +42 100 5.900000000000000e+01 +43 1 4.300000000000000e+01 +43 2 4.200000000000000e+01 +43 3 4.100000000000000e+01 +43 4 4.000000000000000e+01 +43 5 3.900000000000000e+01 +43 6 3.800000000000000e+01 +43 7 3.700000000000000e+01 +43 8 3.600000000000000e+01 +43 9 3.500000000000000e+01 +43 10 3.400000000000000e+01 +43 11 3.300000000000000e+01 +43 12 3.200000000000000e+01 +43 13 3.100000000000000e+01 +43 14 3.000000000000000e+01 +43 15 2.900000000000000e+01 +43 16 2.800000000000000e+01 +43 17 2.700000000000000e+01 +43 18 2.600000000000000e+01 +43 19 2.500000000000000e+01 +43 20 2.400000000000000e+01 +43 21 2.300000000000000e+01 +43 22 2.200000000000000e+01 +43 23 2.100000000000000e+01 +43 24 2.000000000000000e+01 +43 25 1.900000000000000e+01 +43 26 1.800000000000000e+01 +43 27 1.700000000000000e+01 +43 28 1.600000000000000e+01 +43 29 1.500000000000000e+01 +43 30 1.400000000000000e+01 +43 31 1.300000000000000e+01 +43 32 1.200000000000000e+01 +43 33 1.100000000000000e+01 +43 34 1.000000000000000e+01 +43 35 9.000000000000000e+00 +43 36 8.000000000000000e+00 +43 37 7.000000000000000e+00 +43 38 6.000000000000000e+00 +43 39 5.000000000000000e+00 +43 40 4.000000000000000e+00 +43 41 3.000000000000000e+00 +43 42 2.000000000000000e+00 +43 43 2.000000000000000e+00 +43 44 2.000000000000000e+00 +43 45 3.000000000000000e+00 +43 46 4.000000000000000e+00 +43 47 5.000000000000000e+00 +43 48 6.000000000000000e+00 +43 49 7.000000000000000e+00 +43 50 8.000000000000000e+00 +43 51 9.000000000000000e+00 +43 52 1.000000000000000e+01 +43 53 1.100000000000000e+01 +43 54 1.200000000000000e+01 +43 55 1.300000000000000e+01 +43 56 1.400000000000000e+01 +43 57 1.500000000000000e+01 +43 58 1.600000000000000e+01 +43 59 1.700000000000000e+01 +43 60 1.800000000000000e+01 +43 61 1.900000000000000e+01 +43 62 2.000000000000000e+01 +43 63 2.100000000000000e+01 +43 64 2.200000000000000e+01 +43 65 2.300000000000000e+01 +43 66 2.400000000000000e+01 +43 67 2.500000000000000e+01 +43 68 2.600000000000000e+01 +43 69 2.700000000000000e+01 +43 70 2.800000000000000e+01 +43 71 2.900000000000000e+01 +43 72 3.000000000000000e+01 +43 73 3.100000000000000e+01 +43 74 3.200000000000000e+01 +43 75 3.300000000000000e+01 +43 76 3.400000000000000e+01 +43 77 3.500000000000000e+01 +43 78 3.600000000000000e+01 +43 79 3.700000000000000e+01 +43 80 3.800000000000000e+01 +43 81 3.900000000000000e+01 +43 82 4.000000000000000e+01 +43 83 4.100000000000000e+01 +43 84 4.200000000000000e+01 +43 85 4.300000000000000e+01 +43 86 4.400000000000000e+01 +43 87 4.500000000000000e+01 +43 88 4.600000000000000e+01 +43 89 4.700000000000000e+01 +43 90 4.800000000000000e+01 +43 91 4.900000000000000e+01 +43 92 5.000000000000000e+01 +43 93 5.100000000000000e+01 +43 94 5.200000000000000e+01 +43 95 5.300000000000000e+01 +43 96 5.400000000000000e+01 +43 97 5.500000000000000e+01 +43 98 5.600000000000000e+01 +43 99 5.700000000000000e+01 +43 100 5.800000000000000e+01 +44 1 4.400000000000000e+01 +44 2 4.300000000000000e+01 +44 3 4.200000000000000e+01 +44 4 4.100000000000000e+01 +44 5 4.000000000000000e+01 +44 6 3.900000000000000e+01 +44 7 3.800000000000000e+01 +44 8 3.700000000000000e+01 +44 9 3.600000000000000e+01 +44 10 3.500000000000000e+01 +44 11 3.400000000000000e+01 +44 12 3.300000000000000e+01 +44 13 3.200000000000000e+01 +44 14 3.100000000000000e+01 +44 15 3.000000000000000e+01 +44 16 2.900000000000000e+01 +44 17 2.800000000000000e+01 +44 18 2.700000000000000e+01 +44 19 2.600000000000000e+01 +44 20 2.500000000000000e+01 +44 21 2.400000000000000e+01 +44 22 2.300000000000000e+01 +44 23 2.200000000000000e+01 +44 24 2.100000000000000e+01 +44 25 2.000000000000000e+01 +44 26 1.900000000000000e+01 +44 27 1.800000000000000e+01 +44 28 1.700000000000000e+01 +44 29 1.600000000000000e+01 +44 30 1.500000000000000e+01 +44 31 1.400000000000000e+01 +44 32 1.300000000000000e+01 +44 33 1.200000000000000e+01 +44 34 1.100000000000000e+01 +44 35 1.000000000000000e+01 +44 36 9.000000000000000e+00 +44 37 8.000000000000000e+00 +44 38 7.000000000000000e+00 +44 39 6.000000000000000e+00 +44 40 5.000000000000000e+00 +44 41 4.000000000000000e+00 +44 42 3.000000000000000e+00 +44 43 2.000000000000000e+00 +44 44 2.000000000000000e+00 +44 45 2.000000000000000e+00 +44 46 3.000000000000000e+00 +44 47 4.000000000000000e+00 +44 48 5.000000000000000e+00 +44 49 6.000000000000000e+00 +44 50 7.000000000000000e+00 +44 51 8.000000000000000e+00 +44 52 9.000000000000000e+00 +44 53 1.000000000000000e+01 +44 54 1.100000000000000e+01 +44 55 1.200000000000000e+01 +44 56 1.300000000000000e+01 +44 57 1.400000000000000e+01 +44 58 1.500000000000000e+01 +44 59 1.600000000000000e+01 +44 60 1.700000000000000e+01 +44 61 1.800000000000000e+01 +44 62 1.900000000000000e+01 +44 63 2.000000000000000e+01 +44 64 2.100000000000000e+01 +44 65 2.200000000000000e+01 +44 66 2.300000000000000e+01 +44 67 2.400000000000000e+01 +44 68 2.500000000000000e+01 +44 69 2.600000000000000e+01 +44 70 2.700000000000000e+01 +44 71 2.800000000000000e+01 +44 72 2.900000000000000e+01 +44 73 3.000000000000000e+01 +44 74 3.100000000000000e+01 +44 75 3.200000000000000e+01 +44 76 3.300000000000000e+01 +44 77 3.400000000000000e+01 +44 78 3.500000000000000e+01 +44 79 3.600000000000000e+01 +44 80 3.700000000000000e+01 +44 81 3.800000000000000e+01 +44 82 3.900000000000000e+01 +44 83 4.000000000000000e+01 +44 84 4.100000000000000e+01 +44 85 4.200000000000000e+01 +44 86 4.300000000000000e+01 +44 87 4.400000000000000e+01 +44 88 4.500000000000000e+01 +44 89 4.600000000000000e+01 +44 90 4.700000000000000e+01 +44 91 4.800000000000000e+01 +44 92 4.900000000000000e+01 +44 93 5.000000000000000e+01 +44 94 5.100000000000000e+01 +44 95 5.200000000000000e+01 +44 96 5.300000000000000e+01 +44 97 5.400000000000000e+01 +44 98 5.500000000000000e+01 +44 99 5.600000000000000e+01 +44 100 5.700000000000000e+01 +45 1 4.500000000000000e+01 +45 2 4.400000000000000e+01 +45 3 4.300000000000000e+01 +45 4 4.200000000000000e+01 +45 5 4.100000000000000e+01 +45 6 4.000000000000000e+01 +45 7 3.900000000000000e+01 +45 8 3.800000000000000e+01 +45 9 3.700000000000000e+01 +45 10 3.600000000000000e+01 +45 11 3.500000000000000e+01 +45 12 3.400000000000000e+01 +45 13 3.300000000000000e+01 +45 14 3.200000000000000e+01 +45 15 3.100000000000000e+01 +45 16 3.000000000000000e+01 +45 17 2.900000000000000e+01 +45 18 2.800000000000000e+01 +45 19 2.700000000000000e+01 +45 20 2.600000000000000e+01 +45 21 2.500000000000000e+01 +45 22 2.400000000000000e+01 +45 23 2.300000000000000e+01 +45 24 2.200000000000000e+01 +45 25 2.100000000000000e+01 +45 26 2.000000000000000e+01 +45 27 1.900000000000000e+01 +45 28 1.800000000000000e+01 +45 29 1.700000000000000e+01 +45 30 1.600000000000000e+01 +45 31 1.500000000000000e+01 +45 32 1.400000000000000e+01 +45 33 1.300000000000000e+01 +45 34 1.200000000000000e+01 +45 35 1.100000000000000e+01 +45 36 1.000000000000000e+01 +45 37 9.000000000000000e+00 +45 38 8.000000000000000e+00 +45 39 7.000000000000000e+00 +45 40 6.000000000000000e+00 +45 41 5.000000000000000e+00 +45 42 4.000000000000000e+00 +45 43 3.000000000000000e+00 +45 44 2.000000000000000e+00 +45 45 2.000000000000000e+00 +45 46 2.000000000000000e+00 +45 47 3.000000000000000e+00 +45 48 4.000000000000000e+00 +45 49 5.000000000000000e+00 +45 50 6.000000000000000e+00 +45 51 7.000000000000000e+00 +45 52 8.000000000000000e+00 +45 53 9.000000000000000e+00 +45 54 1.000000000000000e+01 +45 55 1.100000000000000e+01 +45 56 1.200000000000000e+01 +45 57 1.300000000000000e+01 +45 58 1.400000000000000e+01 +45 59 1.500000000000000e+01 +45 60 1.600000000000000e+01 +45 61 1.700000000000000e+01 +45 62 1.800000000000000e+01 +45 63 1.900000000000000e+01 +45 64 2.000000000000000e+01 +45 65 2.100000000000000e+01 +45 66 2.200000000000000e+01 +45 67 2.300000000000000e+01 +45 68 2.400000000000000e+01 +45 69 2.500000000000000e+01 +45 70 2.600000000000000e+01 +45 71 2.700000000000000e+01 +45 72 2.800000000000000e+01 +45 73 2.900000000000000e+01 +45 74 3.000000000000000e+01 +45 75 3.100000000000000e+01 +45 76 3.200000000000000e+01 +45 77 3.300000000000000e+01 +45 78 3.400000000000000e+01 +45 79 3.500000000000000e+01 +45 80 3.600000000000000e+01 +45 81 3.700000000000000e+01 +45 82 3.800000000000000e+01 +45 83 3.900000000000000e+01 +45 84 4.000000000000000e+01 +45 85 4.100000000000000e+01 +45 86 4.200000000000000e+01 +45 87 4.300000000000000e+01 +45 88 4.400000000000000e+01 +45 89 4.500000000000000e+01 +45 90 4.600000000000000e+01 +45 91 4.700000000000000e+01 +45 92 4.800000000000000e+01 +45 93 4.900000000000000e+01 +45 94 5.000000000000000e+01 +45 95 5.100000000000000e+01 +45 96 5.200000000000000e+01 +45 97 5.300000000000000e+01 +45 98 5.400000000000000e+01 +45 99 5.500000000000000e+01 +45 100 5.600000000000000e+01 +46 1 4.600000000000000e+01 +46 2 4.500000000000000e+01 +46 3 4.400000000000000e+01 +46 4 4.300000000000000e+01 +46 5 4.200000000000000e+01 +46 6 4.100000000000000e+01 +46 7 4.000000000000000e+01 +46 8 3.900000000000000e+01 +46 9 3.800000000000000e+01 +46 10 3.700000000000000e+01 +46 11 3.600000000000000e+01 +46 12 3.500000000000000e+01 +46 13 3.400000000000000e+01 +46 14 3.300000000000000e+01 +46 15 3.200000000000000e+01 +46 16 3.100000000000000e+01 +46 17 3.000000000000000e+01 +46 18 2.900000000000000e+01 +46 19 2.800000000000000e+01 +46 20 2.700000000000000e+01 +46 21 2.600000000000000e+01 +46 22 2.500000000000000e+01 +46 23 2.400000000000000e+01 +46 24 2.300000000000000e+01 +46 25 2.200000000000000e+01 +46 26 2.100000000000000e+01 +46 27 2.000000000000000e+01 +46 28 1.900000000000000e+01 +46 29 1.800000000000000e+01 +46 30 1.700000000000000e+01 +46 31 1.600000000000000e+01 +46 32 1.500000000000000e+01 +46 33 1.400000000000000e+01 +46 34 1.300000000000000e+01 +46 35 1.200000000000000e+01 +46 36 1.100000000000000e+01 +46 37 1.000000000000000e+01 +46 38 9.000000000000000e+00 +46 39 8.000000000000000e+00 +46 40 7.000000000000000e+00 +46 41 6.000000000000000e+00 +46 42 5.000000000000000e+00 +46 43 4.000000000000000e+00 +46 44 3.000000000000000e+00 +46 45 2.000000000000000e+00 +46 46 2.000000000000000e+00 +46 47 2.000000000000000e+00 +46 48 3.000000000000000e+00 +46 49 4.000000000000000e+00 +46 50 5.000000000000000e+00 +46 51 6.000000000000000e+00 +46 52 7.000000000000000e+00 +46 53 8.000000000000000e+00 +46 54 9.000000000000000e+00 +46 55 1.000000000000000e+01 +46 56 1.100000000000000e+01 +46 57 1.200000000000000e+01 +46 58 1.300000000000000e+01 +46 59 1.400000000000000e+01 +46 60 1.500000000000000e+01 +46 61 1.600000000000000e+01 +46 62 1.700000000000000e+01 +46 63 1.800000000000000e+01 +46 64 1.900000000000000e+01 +46 65 2.000000000000000e+01 +46 66 2.100000000000000e+01 +46 67 2.200000000000000e+01 +46 68 2.300000000000000e+01 +46 69 2.400000000000000e+01 +46 70 2.500000000000000e+01 +46 71 2.600000000000000e+01 +46 72 2.700000000000000e+01 +46 73 2.800000000000000e+01 +46 74 2.900000000000000e+01 +46 75 3.000000000000000e+01 +46 76 3.100000000000000e+01 +46 77 3.200000000000000e+01 +46 78 3.300000000000000e+01 +46 79 3.400000000000000e+01 +46 80 3.500000000000000e+01 +46 81 3.600000000000000e+01 +46 82 3.700000000000000e+01 +46 83 3.800000000000000e+01 +46 84 3.900000000000000e+01 +46 85 4.000000000000000e+01 +46 86 4.100000000000000e+01 +46 87 4.200000000000000e+01 +46 88 4.300000000000000e+01 +46 89 4.400000000000000e+01 +46 90 4.500000000000000e+01 +46 91 4.600000000000000e+01 +46 92 4.700000000000000e+01 +46 93 4.800000000000000e+01 +46 94 4.900000000000000e+01 +46 95 5.000000000000000e+01 +46 96 5.100000000000000e+01 +46 97 5.200000000000000e+01 +46 98 5.300000000000000e+01 +46 99 5.400000000000000e+01 +46 100 5.500000000000000e+01 +47 1 4.700000000000000e+01 +47 2 4.600000000000000e+01 +47 3 4.500000000000000e+01 +47 4 4.400000000000000e+01 +47 5 4.300000000000000e+01 +47 6 4.200000000000000e+01 +47 7 4.100000000000000e+01 +47 8 4.000000000000000e+01 +47 9 3.900000000000000e+01 +47 10 3.800000000000000e+01 +47 11 3.700000000000000e+01 +47 12 3.600000000000000e+01 +47 13 3.500000000000000e+01 +47 14 3.400000000000000e+01 +47 15 3.300000000000000e+01 +47 16 3.200000000000000e+01 +47 17 3.100000000000000e+01 +47 18 3.000000000000000e+01 +47 19 2.900000000000000e+01 +47 20 2.800000000000000e+01 +47 21 2.700000000000000e+01 +47 22 2.600000000000000e+01 +47 23 2.500000000000000e+01 +47 24 2.400000000000000e+01 +47 25 2.300000000000000e+01 +47 26 2.200000000000000e+01 +47 27 2.100000000000000e+01 +47 28 2.000000000000000e+01 +47 29 1.900000000000000e+01 +47 30 1.800000000000000e+01 +47 31 1.700000000000000e+01 +47 32 1.600000000000000e+01 +47 33 1.500000000000000e+01 +47 34 1.400000000000000e+01 +47 35 1.300000000000000e+01 +47 36 1.200000000000000e+01 +47 37 1.100000000000000e+01 +47 38 1.000000000000000e+01 +47 39 9.000000000000000e+00 +47 40 8.000000000000000e+00 +47 41 7.000000000000000e+00 +47 42 6.000000000000000e+00 +47 43 5.000000000000000e+00 +47 44 4.000000000000000e+00 +47 45 3.000000000000000e+00 +47 46 2.000000000000000e+00 +47 47 2.000000000000000e+00 +47 48 2.000000000000000e+00 +47 49 3.000000000000000e+00 +47 50 4.000000000000000e+00 +47 51 5.000000000000000e+00 +47 52 6.000000000000000e+00 +47 53 7.000000000000000e+00 +47 54 8.000000000000000e+00 +47 55 9.000000000000000e+00 +47 56 1.000000000000000e+01 +47 57 1.100000000000000e+01 +47 58 1.200000000000000e+01 +47 59 1.300000000000000e+01 +47 60 1.400000000000000e+01 +47 61 1.500000000000000e+01 +47 62 1.600000000000000e+01 +47 63 1.700000000000000e+01 +47 64 1.800000000000000e+01 +47 65 1.900000000000000e+01 +47 66 2.000000000000000e+01 +47 67 2.100000000000000e+01 +47 68 2.200000000000000e+01 +47 69 2.300000000000000e+01 +47 70 2.400000000000000e+01 +47 71 2.500000000000000e+01 +47 72 2.600000000000000e+01 +47 73 2.700000000000000e+01 +47 74 2.800000000000000e+01 +47 75 2.900000000000000e+01 +47 76 3.000000000000000e+01 +47 77 3.100000000000000e+01 +47 78 3.200000000000000e+01 +47 79 3.300000000000000e+01 +47 80 3.400000000000000e+01 +47 81 3.500000000000000e+01 +47 82 3.600000000000000e+01 +47 83 3.700000000000000e+01 +47 84 3.800000000000000e+01 +47 85 3.900000000000000e+01 +47 86 4.000000000000000e+01 +47 87 4.100000000000000e+01 +47 88 4.200000000000000e+01 +47 89 4.300000000000000e+01 +47 90 4.400000000000000e+01 +47 91 4.500000000000000e+01 +47 92 4.600000000000000e+01 +47 93 4.700000000000000e+01 +47 94 4.800000000000000e+01 +47 95 4.900000000000000e+01 +47 96 5.000000000000000e+01 +47 97 5.100000000000000e+01 +47 98 5.200000000000000e+01 +47 99 5.300000000000000e+01 +47 100 5.400000000000000e+01 +48 1 4.800000000000000e+01 +48 2 4.700000000000000e+01 +48 3 4.600000000000000e+01 +48 4 4.500000000000000e+01 +48 5 4.400000000000000e+01 +48 6 4.300000000000000e+01 +48 7 4.200000000000000e+01 +48 8 4.100000000000000e+01 +48 9 4.000000000000000e+01 +48 10 3.900000000000000e+01 +48 11 3.800000000000000e+01 +48 12 3.700000000000000e+01 +48 13 3.600000000000000e+01 +48 14 3.500000000000000e+01 +48 15 3.400000000000000e+01 +48 16 3.300000000000000e+01 +48 17 3.200000000000000e+01 +48 18 3.100000000000000e+01 +48 19 3.000000000000000e+01 +48 20 2.900000000000000e+01 +48 21 2.800000000000000e+01 +48 22 2.700000000000000e+01 +48 23 2.600000000000000e+01 +48 24 2.500000000000000e+01 +48 25 2.400000000000000e+01 +48 26 2.300000000000000e+01 +48 27 2.200000000000000e+01 +48 28 2.100000000000000e+01 +48 29 2.000000000000000e+01 +48 30 1.900000000000000e+01 +48 31 1.800000000000000e+01 +48 32 1.700000000000000e+01 +48 33 1.600000000000000e+01 +48 34 1.500000000000000e+01 +48 35 1.400000000000000e+01 +48 36 1.300000000000000e+01 +48 37 1.200000000000000e+01 +48 38 1.100000000000000e+01 +48 39 1.000000000000000e+01 +48 40 9.000000000000000e+00 +48 41 8.000000000000000e+00 +48 42 7.000000000000000e+00 +48 43 6.000000000000000e+00 +48 44 5.000000000000000e+00 +48 45 4.000000000000000e+00 +48 46 3.000000000000000e+00 +48 47 2.000000000000000e+00 +48 48 2.000000000000000e+00 +48 49 2.000000000000000e+00 +48 50 3.000000000000000e+00 +48 51 4.000000000000000e+00 +48 52 5.000000000000000e+00 +48 53 6.000000000000000e+00 +48 54 7.000000000000000e+00 +48 55 8.000000000000000e+00 +48 56 9.000000000000000e+00 +48 57 1.000000000000000e+01 +48 58 1.100000000000000e+01 +48 59 1.200000000000000e+01 +48 60 1.300000000000000e+01 +48 61 1.400000000000000e+01 +48 62 1.500000000000000e+01 +48 63 1.600000000000000e+01 +48 64 1.700000000000000e+01 +48 65 1.800000000000000e+01 +48 66 1.900000000000000e+01 +48 67 2.000000000000000e+01 +48 68 2.100000000000000e+01 +48 69 2.200000000000000e+01 +48 70 2.300000000000000e+01 +48 71 2.400000000000000e+01 +48 72 2.500000000000000e+01 +48 73 2.600000000000000e+01 +48 74 2.700000000000000e+01 +48 75 2.800000000000000e+01 +48 76 2.900000000000000e+01 +48 77 3.000000000000000e+01 +48 78 3.100000000000000e+01 +48 79 3.200000000000000e+01 +48 80 3.300000000000000e+01 +48 81 3.400000000000000e+01 +48 82 3.500000000000000e+01 +48 83 3.600000000000000e+01 +48 84 3.700000000000000e+01 +48 85 3.800000000000000e+01 +48 86 3.900000000000000e+01 +48 87 4.000000000000000e+01 +48 88 4.100000000000000e+01 +48 89 4.200000000000000e+01 +48 90 4.300000000000000e+01 +48 91 4.400000000000000e+01 +48 92 4.500000000000000e+01 +48 93 4.600000000000000e+01 +48 94 4.700000000000000e+01 +48 95 4.800000000000000e+01 +48 96 4.900000000000000e+01 +48 97 5.000000000000000e+01 +48 98 5.100000000000000e+01 +48 99 5.200000000000000e+01 +48 100 5.300000000000000e+01 +49 1 4.900000000000000e+01 +49 2 4.800000000000000e+01 +49 3 4.700000000000000e+01 +49 4 4.600000000000000e+01 +49 5 4.500000000000000e+01 +49 6 4.400000000000000e+01 +49 7 4.300000000000000e+01 +49 8 4.200000000000000e+01 +49 9 4.100000000000000e+01 +49 10 4.000000000000000e+01 +49 11 3.900000000000000e+01 +49 12 3.800000000000000e+01 +49 13 3.700000000000000e+01 +49 14 3.600000000000000e+01 +49 15 3.500000000000000e+01 +49 16 3.400000000000000e+01 +49 17 3.300000000000000e+01 +49 18 3.200000000000000e+01 +49 19 3.100000000000000e+01 +49 20 3.000000000000000e+01 +49 21 2.900000000000000e+01 +49 22 2.800000000000000e+01 +49 23 2.700000000000000e+01 +49 24 2.600000000000000e+01 +49 25 2.500000000000000e+01 +49 26 2.400000000000000e+01 +49 27 2.300000000000000e+01 +49 28 2.200000000000000e+01 +49 29 2.100000000000000e+01 +49 30 2.000000000000000e+01 +49 31 1.900000000000000e+01 +49 32 1.800000000000000e+01 +49 33 1.700000000000000e+01 +49 34 1.600000000000000e+01 +49 35 1.500000000000000e+01 +49 36 1.400000000000000e+01 +49 37 1.300000000000000e+01 +49 38 1.200000000000000e+01 +49 39 1.100000000000000e+01 +49 40 1.000000000000000e+01 +49 41 9.000000000000000e+00 +49 42 8.000000000000000e+00 +49 43 7.000000000000000e+00 +49 44 6.000000000000000e+00 +49 45 5.000000000000000e+00 +49 46 4.000000000000000e+00 +49 47 3.000000000000000e+00 +49 48 2.000000000000000e+00 +49 49 2.000000000000000e+00 +49 50 2.000000000000000e+00 +49 51 3.000000000000000e+00 +49 52 4.000000000000000e+00 +49 53 5.000000000000000e+00 +49 54 6.000000000000000e+00 +49 55 7.000000000000000e+00 +49 56 8.000000000000000e+00 +49 57 9.000000000000000e+00 +49 58 1.000000000000000e+01 +49 59 1.100000000000000e+01 +49 60 1.200000000000000e+01 +49 61 1.300000000000000e+01 +49 62 1.400000000000000e+01 +49 63 1.500000000000000e+01 +49 64 1.600000000000000e+01 +49 65 1.700000000000000e+01 +49 66 1.800000000000000e+01 +49 67 1.900000000000000e+01 +49 68 2.000000000000000e+01 +49 69 2.100000000000000e+01 +49 70 2.200000000000000e+01 +49 71 2.300000000000000e+01 +49 72 2.400000000000000e+01 +49 73 2.500000000000000e+01 +49 74 2.600000000000000e+01 +49 75 2.700000000000000e+01 +49 76 2.800000000000000e+01 +49 77 2.900000000000000e+01 +49 78 3.000000000000000e+01 +49 79 3.100000000000000e+01 +49 80 3.200000000000000e+01 +49 81 3.300000000000000e+01 +49 82 3.400000000000000e+01 +49 83 3.500000000000000e+01 +49 84 3.600000000000000e+01 +49 85 3.700000000000000e+01 +49 86 3.800000000000000e+01 +49 87 3.900000000000000e+01 +49 88 4.000000000000000e+01 +49 89 4.100000000000000e+01 +49 90 4.200000000000000e+01 +49 91 4.300000000000000e+01 +49 92 4.400000000000000e+01 +49 93 4.500000000000000e+01 +49 94 4.600000000000000e+01 +49 95 4.700000000000000e+01 +49 96 4.800000000000000e+01 +49 97 4.900000000000000e+01 +49 98 5.000000000000000e+01 +49 99 5.100000000000000e+01 +49 100 5.200000000000000e+01 +50 1 5.000000000000000e+01 +50 2 4.900000000000000e+01 +50 3 4.800000000000000e+01 +50 4 4.700000000000000e+01 +50 5 4.600000000000000e+01 +50 6 4.500000000000000e+01 +50 7 4.400000000000000e+01 +50 8 4.300000000000000e+01 +50 9 4.200000000000000e+01 +50 10 4.100000000000000e+01 +50 11 4.000000000000000e+01 +50 12 3.900000000000000e+01 +50 13 3.800000000000000e+01 +50 14 3.700000000000000e+01 +50 15 3.600000000000000e+01 +50 16 3.500000000000000e+01 +50 17 3.400000000000000e+01 +50 18 3.300000000000000e+01 +50 19 3.200000000000000e+01 +50 20 3.100000000000000e+01 +50 21 3.000000000000000e+01 +50 22 2.900000000000000e+01 +50 23 2.800000000000000e+01 +50 24 2.700000000000000e+01 +50 25 2.600000000000000e+01 +50 26 2.500000000000000e+01 +50 27 2.400000000000000e+01 +50 28 2.300000000000000e+01 +50 29 2.200000000000000e+01 +50 30 2.100000000000000e+01 +50 31 2.000000000000000e+01 +50 32 1.900000000000000e+01 +50 33 1.800000000000000e+01 +50 34 1.700000000000000e+01 +50 35 1.600000000000000e+01 +50 36 1.500000000000000e+01 +50 37 1.400000000000000e+01 +50 38 1.300000000000000e+01 +50 39 1.200000000000000e+01 +50 40 1.100000000000000e+01 +50 41 1.000000000000000e+01 +50 42 9.000000000000000e+00 +50 43 8.000000000000000e+00 +50 44 7.000000000000000e+00 +50 45 6.000000000000000e+00 +50 46 5.000000000000000e+00 +50 47 4.000000000000000e+00 +50 48 3.000000000000000e+00 +50 49 2.000000000000000e+00 +50 50 2.000000000000000e+00 +50 51 2.000000000000000e+00 +50 52 3.000000000000000e+00 +50 53 4.000000000000000e+00 +50 54 5.000000000000000e+00 +50 55 6.000000000000000e+00 +50 56 7.000000000000000e+00 +50 57 8.000000000000000e+00 +50 58 9.000000000000000e+00 +50 59 1.000000000000000e+01 +50 60 1.100000000000000e+01 +50 61 1.200000000000000e+01 +50 62 1.300000000000000e+01 +50 63 1.400000000000000e+01 +50 64 1.500000000000000e+01 +50 65 1.600000000000000e+01 +50 66 1.700000000000000e+01 +50 67 1.800000000000000e+01 +50 68 1.900000000000000e+01 +50 69 2.000000000000000e+01 +50 70 2.100000000000000e+01 +50 71 2.200000000000000e+01 +50 72 2.300000000000000e+01 +50 73 2.400000000000000e+01 +50 74 2.500000000000000e+01 +50 75 2.600000000000000e+01 +50 76 2.700000000000000e+01 +50 77 2.800000000000000e+01 +50 78 2.900000000000000e+01 +50 79 3.000000000000000e+01 +50 80 3.100000000000000e+01 +50 81 3.200000000000000e+01 +50 82 3.300000000000000e+01 +50 83 3.400000000000000e+01 +50 84 3.500000000000000e+01 +50 85 3.600000000000000e+01 +50 86 3.700000000000000e+01 +50 87 3.800000000000000e+01 +50 88 3.900000000000000e+01 +50 89 4.000000000000000e+01 +50 90 4.100000000000000e+01 +50 91 4.200000000000000e+01 +50 92 4.300000000000000e+01 +50 93 4.400000000000000e+01 +50 94 4.500000000000000e+01 +50 95 4.600000000000000e+01 +50 96 4.700000000000000e+01 +50 97 4.800000000000000e+01 +50 98 4.900000000000000e+01 +50 99 5.000000000000000e+01 +50 100 5.100000000000000e+01 +51 1 5.100000000000000e+01 +51 2 5.000000000000000e+01 +51 3 4.900000000000000e+01 +51 4 4.800000000000000e+01 +51 5 4.700000000000000e+01 +51 6 4.600000000000000e+01 +51 7 4.500000000000000e+01 +51 8 4.400000000000000e+01 +51 9 4.300000000000000e+01 +51 10 4.200000000000000e+01 +51 11 4.100000000000000e+01 +51 12 4.000000000000000e+01 +51 13 3.900000000000000e+01 +51 14 3.800000000000000e+01 +51 15 3.700000000000000e+01 +51 16 3.600000000000000e+01 +51 17 3.500000000000000e+01 +51 18 3.400000000000000e+01 +51 19 3.300000000000000e+01 +51 20 3.200000000000000e+01 +51 21 3.100000000000000e+01 +51 22 3.000000000000000e+01 +51 23 2.900000000000000e+01 +51 24 2.800000000000000e+01 +51 25 2.700000000000000e+01 +51 26 2.600000000000000e+01 +51 27 2.500000000000000e+01 +51 28 2.400000000000000e+01 +51 29 2.300000000000000e+01 +51 30 2.200000000000000e+01 +51 31 2.100000000000000e+01 +51 32 2.000000000000000e+01 +51 33 1.900000000000000e+01 +51 34 1.800000000000000e+01 +51 35 1.700000000000000e+01 +51 36 1.600000000000000e+01 +51 37 1.500000000000000e+01 +51 38 1.400000000000000e+01 +51 39 1.300000000000000e+01 +51 40 1.200000000000000e+01 +51 41 1.100000000000000e+01 +51 42 1.000000000000000e+01 +51 43 9.000000000000000e+00 +51 44 8.000000000000000e+00 +51 45 7.000000000000000e+00 +51 46 6.000000000000000e+00 +51 47 5.000000000000000e+00 +51 48 4.000000000000000e+00 +51 49 3.000000000000000e+00 +51 50 2.000000000000000e+00 +51 51 2.000000000000000e+00 +51 52 2.000000000000000e+00 +51 53 3.000000000000000e+00 +51 54 4.000000000000000e+00 +51 55 5.000000000000000e+00 +51 56 6.000000000000000e+00 +51 57 7.000000000000000e+00 +51 58 8.000000000000000e+00 +51 59 9.000000000000000e+00 +51 60 1.000000000000000e+01 +51 61 1.100000000000000e+01 +51 62 1.200000000000000e+01 +51 63 1.300000000000000e+01 +51 64 1.400000000000000e+01 +51 65 1.500000000000000e+01 +51 66 1.600000000000000e+01 +51 67 1.700000000000000e+01 +51 68 1.800000000000000e+01 +51 69 1.900000000000000e+01 +51 70 2.000000000000000e+01 +51 71 2.100000000000000e+01 +51 72 2.200000000000000e+01 +51 73 2.300000000000000e+01 +51 74 2.400000000000000e+01 +51 75 2.500000000000000e+01 +51 76 2.600000000000000e+01 +51 77 2.700000000000000e+01 +51 78 2.800000000000000e+01 +51 79 2.900000000000000e+01 +51 80 3.000000000000000e+01 +51 81 3.100000000000000e+01 +51 82 3.200000000000000e+01 +51 83 3.300000000000000e+01 +51 84 3.400000000000000e+01 +51 85 3.500000000000000e+01 +51 86 3.600000000000000e+01 +51 87 3.700000000000000e+01 +51 88 3.800000000000000e+01 +51 89 3.900000000000000e+01 +51 90 4.000000000000000e+01 +51 91 4.100000000000000e+01 +51 92 4.200000000000000e+01 +51 93 4.300000000000000e+01 +51 94 4.400000000000000e+01 +51 95 4.500000000000000e+01 +51 96 4.600000000000000e+01 +51 97 4.700000000000000e+01 +51 98 4.800000000000000e+01 +51 99 4.900000000000000e+01 +51 100 5.000000000000000e+01 +52 1 5.200000000000000e+01 +52 2 5.100000000000000e+01 +52 3 5.000000000000000e+01 +52 4 4.900000000000000e+01 +52 5 4.800000000000000e+01 +52 6 4.700000000000000e+01 +52 7 4.600000000000000e+01 +52 8 4.500000000000000e+01 +52 9 4.400000000000000e+01 +52 10 4.300000000000000e+01 +52 11 4.200000000000000e+01 +52 12 4.100000000000000e+01 +52 13 4.000000000000000e+01 +52 14 3.900000000000000e+01 +52 15 3.800000000000000e+01 +52 16 3.700000000000000e+01 +52 17 3.600000000000000e+01 +52 18 3.500000000000000e+01 +52 19 3.400000000000000e+01 +52 20 3.300000000000000e+01 +52 21 3.200000000000000e+01 +52 22 3.100000000000000e+01 +52 23 3.000000000000000e+01 +52 24 2.900000000000000e+01 +52 25 2.800000000000000e+01 +52 26 2.700000000000000e+01 +52 27 2.600000000000000e+01 +52 28 2.500000000000000e+01 +52 29 2.400000000000000e+01 +52 30 2.300000000000000e+01 +52 31 2.200000000000000e+01 +52 32 2.100000000000000e+01 +52 33 2.000000000000000e+01 +52 34 1.900000000000000e+01 +52 35 1.800000000000000e+01 +52 36 1.700000000000000e+01 +52 37 1.600000000000000e+01 +52 38 1.500000000000000e+01 +52 39 1.400000000000000e+01 +52 40 1.300000000000000e+01 +52 41 1.200000000000000e+01 +52 42 1.100000000000000e+01 +52 43 1.000000000000000e+01 +52 44 9.000000000000000e+00 +52 45 8.000000000000000e+00 +52 46 7.000000000000000e+00 +52 47 6.000000000000000e+00 +52 48 5.000000000000000e+00 +52 49 4.000000000000000e+00 +52 50 3.000000000000000e+00 +52 51 2.000000000000000e+00 +52 52 2.000000000000000e+00 +52 53 2.000000000000000e+00 +52 54 3.000000000000000e+00 +52 55 4.000000000000000e+00 +52 56 5.000000000000000e+00 +52 57 6.000000000000000e+00 +52 58 7.000000000000000e+00 +52 59 8.000000000000000e+00 +52 60 9.000000000000000e+00 +52 61 1.000000000000000e+01 +52 62 1.100000000000000e+01 +52 63 1.200000000000000e+01 +52 64 1.300000000000000e+01 +52 65 1.400000000000000e+01 +52 66 1.500000000000000e+01 +52 67 1.600000000000000e+01 +52 68 1.700000000000000e+01 +52 69 1.800000000000000e+01 +52 70 1.900000000000000e+01 +52 71 2.000000000000000e+01 +52 72 2.100000000000000e+01 +52 73 2.200000000000000e+01 +52 74 2.300000000000000e+01 +52 75 2.400000000000000e+01 +52 76 2.500000000000000e+01 +52 77 2.600000000000000e+01 +52 78 2.700000000000000e+01 +52 79 2.800000000000000e+01 +52 80 2.900000000000000e+01 +52 81 3.000000000000000e+01 +52 82 3.100000000000000e+01 +52 83 3.200000000000000e+01 +52 84 3.300000000000000e+01 +52 85 3.400000000000000e+01 +52 86 3.500000000000000e+01 +52 87 3.600000000000000e+01 +52 88 3.700000000000000e+01 +52 89 3.800000000000000e+01 +52 90 3.900000000000000e+01 +52 91 4.000000000000000e+01 +52 92 4.100000000000000e+01 +52 93 4.200000000000000e+01 +52 94 4.300000000000000e+01 +52 95 4.400000000000000e+01 +52 96 4.500000000000000e+01 +52 97 4.600000000000000e+01 +52 98 4.700000000000000e+01 +52 99 4.800000000000000e+01 +52 100 4.900000000000000e+01 +53 1 5.300000000000000e+01 +53 2 5.200000000000000e+01 +53 3 5.100000000000000e+01 +53 4 5.000000000000000e+01 +53 5 4.900000000000000e+01 +53 6 4.800000000000000e+01 +53 7 4.700000000000000e+01 +53 8 4.600000000000000e+01 +53 9 4.500000000000000e+01 +53 10 4.400000000000000e+01 +53 11 4.300000000000000e+01 +53 12 4.200000000000000e+01 +53 13 4.100000000000000e+01 +53 14 4.000000000000000e+01 +53 15 3.900000000000000e+01 +53 16 3.800000000000000e+01 +53 17 3.700000000000000e+01 +53 18 3.600000000000000e+01 +53 19 3.500000000000000e+01 +53 20 3.400000000000000e+01 +53 21 3.300000000000000e+01 +53 22 3.200000000000000e+01 +53 23 3.100000000000000e+01 +53 24 3.000000000000000e+01 +53 25 2.900000000000000e+01 +53 26 2.800000000000000e+01 +53 27 2.700000000000000e+01 +53 28 2.600000000000000e+01 +53 29 2.500000000000000e+01 +53 30 2.400000000000000e+01 +53 31 2.300000000000000e+01 +53 32 2.200000000000000e+01 +53 33 2.100000000000000e+01 +53 34 2.000000000000000e+01 +53 35 1.900000000000000e+01 +53 36 1.800000000000000e+01 +53 37 1.700000000000000e+01 +53 38 1.600000000000000e+01 +53 39 1.500000000000000e+01 +53 40 1.400000000000000e+01 +53 41 1.300000000000000e+01 +53 42 1.200000000000000e+01 +53 43 1.100000000000000e+01 +53 44 1.000000000000000e+01 +53 45 9.000000000000000e+00 +53 46 8.000000000000000e+00 +53 47 7.000000000000000e+00 +53 48 6.000000000000000e+00 +53 49 5.000000000000000e+00 +53 50 4.000000000000000e+00 +53 51 3.000000000000000e+00 +53 52 2.000000000000000e+00 +53 53 2.000000000000000e+00 +53 54 2.000000000000000e+00 +53 55 3.000000000000000e+00 +53 56 4.000000000000000e+00 +53 57 5.000000000000000e+00 +53 58 6.000000000000000e+00 +53 59 7.000000000000000e+00 +53 60 8.000000000000000e+00 +53 61 9.000000000000000e+00 +53 62 1.000000000000000e+01 +53 63 1.100000000000000e+01 +53 64 1.200000000000000e+01 +53 65 1.300000000000000e+01 +53 66 1.400000000000000e+01 +53 67 1.500000000000000e+01 +53 68 1.600000000000000e+01 +53 69 1.700000000000000e+01 +53 70 1.800000000000000e+01 +53 71 1.900000000000000e+01 +53 72 2.000000000000000e+01 +53 73 2.100000000000000e+01 +53 74 2.200000000000000e+01 +53 75 2.300000000000000e+01 +53 76 2.400000000000000e+01 +53 77 2.500000000000000e+01 +53 78 2.600000000000000e+01 +53 79 2.700000000000000e+01 +53 80 2.800000000000000e+01 +53 81 2.900000000000000e+01 +53 82 3.000000000000000e+01 +53 83 3.100000000000000e+01 +53 84 3.200000000000000e+01 +53 85 3.300000000000000e+01 +53 86 3.400000000000000e+01 +53 87 3.500000000000000e+01 +53 88 3.600000000000000e+01 +53 89 3.700000000000000e+01 +53 90 3.800000000000000e+01 +53 91 3.900000000000000e+01 +53 92 4.000000000000000e+01 +53 93 4.100000000000000e+01 +53 94 4.200000000000000e+01 +53 95 4.300000000000000e+01 +53 96 4.400000000000000e+01 +53 97 4.500000000000000e+01 +53 98 4.600000000000000e+01 +53 99 4.700000000000000e+01 +53 100 4.800000000000000e+01 +54 1 5.400000000000000e+01 +54 2 5.300000000000000e+01 +54 3 5.200000000000000e+01 +54 4 5.100000000000000e+01 +54 5 5.000000000000000e+01 +54 6 4.900000000000000e+01 +54 7 4.800000000000000e+01 +54 8 4.700000000000000e+01 +54 9 4.600000000000000e+01 +54 10 4.500000000000000e+01 +54 11 4.400000000000000e+01 +54 12 4.300000000000000e+01 +54 13 4.200000000000000e+01 +54 14 4.100000000000000e+01 +54 15 4.000000000000000e+01 +54 16 3.900000000000000e+01 +54 17 3.800000000000000e+01 +54 18 3.700000000000000e+01 +54 19 3.600000000000000e+01 +54 20 3.500000000000000e+01 +54 21 3.400000000000000e+01 +54 22 3.300000000000000e+01 +54 23 3.200000000000000e+01 +54 24 3.100000000000000e+01 +54 25 3.000000000000000e+01 +54 26 2.900000000000000e+01 +54 27 2.800000000000000e+01 +54 28 2.700000000000000e+01 +54 29 2.600000000000000e+01 +54 30 2.500000000000000e+01 +54 31 2.400000000000000e+01 +54 32 2.300000000000000e+01 +54 33 2.200000000000000e+01 +54 34 2.100000000000000e+01 +54 35 2.000000000000000e+01 +54 36 1.900000000000000e+01 +54 37 1.800000000000000e+01 +54 38 1.700000000000000e+01 +54 39 1.600000000000000e+01 +54 40 1.500000000000000e+01 +54 41 1.400000000000000e+01 +54 42 1.300000000000000e+01 +54 43 1.200000000000000e+01 +54 44 1.100000000000000e+01 +54 45 1.000000000000000e+01 +54 46 9.000000000000000e+00 +54 47 8.000000000000000e+00 +54 48 7.000000000000000e+00 +54 49 6.000000000000000e+00 +54 50 5.000000000000000e+00 +54 51 4.000000000000000e+00 +54 52 3.000000000000000e+00 +54 53 2.000000000000000e+00 +54 54 2.000000000000000e+00 +54 55 2.000000000000000e+00 +54 56 3.000000000000000e+00 +54 57 4.000000000000000e+00 +54 58 5.000000000000000e+00 +54 59 6.000000000000000e+00 +54 60 7.000000000000000e+00 +54 61 8.000000000000000e+00 +54 62 9.000000000000000e+00 +54 63 1.000000000000000e+01 +54 64 1.100000000000000e+01 +54 65 1.200000000000000e+01 +54 66 1.300000000000000e+01 +54 67 1.400000000000000e+01 +54 68 1.500000000000000e+01 +54 69 1.600000000000000e+01 +54 70 1.700000000000000e+01 +54 71 1.800000000000000e+01 +54 72 1.900000000000000e+01 +54 73 2.000000000000000e+01 +54 74 2.100000000000000e+01 +54 75 2.200000000000000e+01 +54 76 2.300000000000000e+01 +54 77 2.400000000000000e+01 +54 78 2.500000000000000e+01 +54 79 2.600000000000000e+01 +54 80 2.700000000000000e+01 +54 81 2.800000000000000e+01 +54 82 2.900000000000000e+01 +54 83 3.000000000000000e+01 +54 84 3.100000000000000e+01 +54 85 3.200000000000000e+01 +54 86 3.300000000000000e+01 +54 87 3.400000000000000e+01 +54 88 3.500000000000000e+01 +54 89 3.600000000000000e+01 +54 90 3.700000000000000e+01 +54 91 3.800000000000000e+01 +54 92 3.900000000000000e+01 +54 93 4.000000000000000e+01 +54 94 4.100000000000000e+01 +54 95 4.200000000000000e+01 +54 96 4.300000000000000e+01 +54 97 4.400000000000000e+01 +54 98 4.500000000000000e+01 +54 99 4.600000000000000e+01 +54 100 4.700000000000000e+01 +55 1 5.500000000000000e+01 +55 2 5.400000000000000e+01 +55 3 5.300000000000000e+01 +55 4 5.200000000000000e+01 +55 5 5.100000000000000e+01 +55 6 5.000000000000000e+01 +55 7 4.900000000000000e+01 +55 8 4.800000000000000e+01 +55 9 4.700000000000000e+01 +55 10 4.600000000000000e+01 +55 11 4.500000000000000e+01 +55 12 4.400000000000000e+01 +55 13 4.300000000000000e+01 +55 14 4.200000000000000e+01 +55 15 4.100000000000000e+01 +55 16 4.000000000000000e+01 +55 17 3.900000000000000e+01 +55 18 3.800000000000000e+01 +55 19 3.700000000000000e+01 +55 20 3.600000000000000e+01 +55 21 3.500000000000000e+01 +55 22 3.400000000000000e+01 +55 23 3.300000000000000e+01 +55 24 3.200000000000000e+01 +55 25 3.100000000000000e+01 +55 26 3.000000000000000e+01 +55 27 2.900000000000000e+01 +55 28 2.800000000000000e+01 +55 29 2.700000000000000e+01 +55 30 2.600000000000000e+01 +55 31 2.500000000000000e+01 +55 32 2.400000000000000e+01 +55 33 2.300000000000000e+01 +55 34 2.200000000000000e+01 +55 35 2.100000000000000e+01 +55 36 2.000000000000000e+01 +55 37 1.900000000000000e+01 +55 38 1.800000000000000e+01 +55 39 1.700000000000000e+01 +55 40 1.600000000000000e+01 +55 41 1.500000000000000e+01 +55 42 1.400000000000000e+01 +55 43 1.300000000000000e+01 +55 44 1.200000000000000e+01 +55 45 1.100000000000000e+01 +55 46 1.000000000000000e+01 +55 47 9.000000000000000e+00 +55 48 8.000000000000000e+00 +55 49 7.000000000000000e+00 +55 50 6.000000000000000e+00 +55 51 5.000000000000000e+00 +55 52 4.000000000000000e+00 +55 53 3.000000000000000e+00 +55 54 2.000000000000000e+00 +55 55 2.000000000000000e+00 +55 56 2.000000000000000e+00 +55 57 3.000000000000000e+00 +55 58 4.000000000000000e+00 +55 59 5.000000000000000e+00 +55 60 6.000000000000000e+00 +55 61 7.000000000000000e+00 +55 62 8.000000000000000e+00 +55 63 9.000000000000000e+00 +55 64 1.000000000000000e+01 +55 65 1.100000000000000e+01 +55 66 1.200000000000000e+01 +55 67 1.300000000000000e+01 +55 68 1.400000000000000e+01 +55 69 1.500000000000000e+01 +55 70 1.600000000000000e+01 +55 71 1.700000000000000e+01 +55 72 1.800000000000000e+01 +55 73 1.900000000000000e+01 +55 74 2.000000000000000e+01 +55 75 2.100000000000000e+01 +55 76 2.200000000000000e+01 +55 77 2.300000000000000e+01 +55 78 2.400000000000000e+01 +55 79 2.500000000000000e+01 +55 80 2.600000000000000e+01 +55 81 2.700000000000000e+01 +55 82 2.800000000000000e+01 +55 83 2.900000000000000e+01 +55 84 3.000000000000000e+01 +55 85 3.100000000000000e+01 +55 86 3.200000000000000e+01 +55 87 3.300000000000000e+01 +55 88 3.400000000000000e+01 +55 89 3.500000000000000e+01 +55 90 3.600000000000000e+01 +55 91 3.700000000000000e+01 +55 92 3.800000000000000e+01 +55 93 3.900000000000000e+01 +55 94 4.000000000000000e+01 +55 95 4.100000000000000e+01 +55 96 4.200000000000000e+01 +55 97 4.300000000000000e+01 +55 98 4.400000000000000e+01 +55 99 4.500000000000000e+01 +55 100 4.600000000000000e+01 +56 1 5.600000000000000e+01 +56 2 5.500000000000000e+01 +56 3 5.400000000000000e+01 +56 4 5.300000000000000e+01 +56 5 5.200000000000000e+01 +56 6 5.100000000000000e+01 +56 7 5.000000000000000e+01 +56 8 4.900000000000000e+01 +56 9 4.800000000000000e+01 +56 10 4.700000000000000e+01 +56 11 4.600000000000000e+01 +56 12 4.500000000000000e+01 +56 13 4.400000000000000e+01 +56 14 4.300000000000000e+01 +56 15 4.200000000000000e+01 +56 16 4.100000000000000e+01 +56 17 4.000000000000000e+01 +56 18 3.900000000000000e+01 +56 19 3.800000000000000e+01 +56 20 3.700000000000000e+01 +56 21 3.600000000000000e+01 +56 22 3.500000000000000e+01 +56 23 3.400000000000000e+01 +56 24 3.300000000000000e+01 +56 25 3.200000000000000e+01 +56 26 3.100000000000000e+01 +56 27 3.000000000000000e+01 +56 28 2.900000000000000e+01 +56 29 2.800000000000000e+01 +56 30 2.700000000000000e+01 +56 31 2.600000000000000e+01 +56 32 2.500000000000000e+01 +56 33 2.400000000000000e+01 +56 34 2.300000000000000e+01 +56 35 2.200000000000000e+01 +56 36 2.100000000000000e+01 +56 37 2.000000000000000e+01 +56 38 1.900000000000000e+01 +56 39 1.800000000000000e+01 +56 40 1.700000000000000e+01 +56 41 1.600000000000000e+01 +56 42 1.500000000000000e+01 +56 43 1.400000000000000e+01 +56 44 1.300000000000000e+01 +56 45 1.200000000000000e+01 +56 46 1.100000000000000e+01 +56 47 1.000000000000000e+01 +56 48 9.000000000000000e+00 +56 49 8.000000000000000e+00 +56 50 7.000000000000000e+00 +56 51 6.000000000000000e+00 +56 52 5.000000000000000e+00 +56 53 4.000000000000000e+00 +56 54 3.000000000000000e+00 +56 55 2.000000000000000e+00 +56 56 2.000000000000000e+00 +56 57 2.000000000000000e+00 +56 58 3.000000000000000e+00 +56 59 4.000000000000000e+00 +56 60 5.000000000000000e+00 +56 61 6.000000000000000e+00 +56 62 7.000000000000000e+00 +56 63 8.000000000000000e+00 +56 64 9.000000000000000e+00 +56 65 1.000000000000000e+01 +56 66 1.100000000000000e+01 +56 67 1.200000000000000e+01 +56 68 1.300000000000000e+01 +56 69 1.400000000000000e+01 +56 70 1.500000000000000e+01 +56 71 1.600000000000000e+01 +56 72 1.700000000000000e+01 +56 73 1.800000000000000e+01 +56 74 1.900000000000000e+01 +56 75 2.000000000000000e+01 +56 76 2.100000000000000e+01 +56 77 2.200000000000000e+01 +56 78 2.300000000000000e+01 +56 79 2.400000000000000e+01 +56 80 2.500000000000000e+01 +56 81 2.600000000000000e+01 +56 82 2.700000000000000e+01 +56 83 2.800000000000000e+01 +56 84 2.900000000000000e+01 +56 85 3.000000000000000e+01 +56 86 3.100000000000000e+01 +56 87 3.200000000000000e+01 +56 88 3.300000000000000e+01 +56 89 3.400000000000000e+01 +56 90 3.500000000000000e+01 +56 91 3.600000000000000e+01 +56 92 3.700000000000000e+01 +56 93 3.800000000000000e+01 +56 94 3.900000000000000e+01 +56 95 4.000000000000000e+01 +56 96 4.100000000000000e+01 +56 97 4.200000000000000e+01 +56 98 4.300000000000000e+01 +56 99 4.400000000000000e+01 +56 100 4.500000000000000e+01 +57 1 5.700000000000000e+01 +57 2 5.600000000000000e+01 +57 3 5.500000000000000e+01 +57 4 5.400000000000000e+01 +57 5 5.300000000000000e+01 +57 6 5.200000000000000e+01 +57 7 5.100000000000000e+01 +57 8 5.000000000000000e+01 +57 9 4.900000000000000e+01 +57 10 4.800000000000000e+01 +57 11 4.700000000000000e+01 +57 12 4.600000000000000e+01 +57 13 4.500000000000000e+01 +57 14 4.400000000000000e+01 +57 15 4.300000000000000e+01 +57 16 4.200000000000000e+01 +57 17 4.100000000000000e+01 +57 18 4.000000000000000e+01 +57 19 3.900000000000000e+01 +57 20 3.800000000000000e+01 +57 21 3.700000000000000e+01 +57 22 3.600000000000000e+01 +57 23 3.500000000000000e+01 +57 24 3.400000000000000e+01 +57 25 3.300000000000000e+01 +57 26 3.200000000000000e+01 +57 27 3.100000000000000e+01 +57 28 3.000000000000000e+01 +57 29 2.900000000000000e+01 +57 30 2.800000000000000e+01 +57 31 2.700000000000000e+01 +57 32 2.600000000000000e+01 +57 33 2.500000000000000e+01 +57 34 2.400000000000000e+01 +57 35 2.300000000000000e+01 +57 36 2.200000000000000e+01 +57 37 2.100000000000000e+01 +57 38 2.000000000000000e+01 +57 39 1.900000000000000e+01 +57 40 1.800000000000000e+01 +57 41 1.700000000000000e+01 +57 42 1.600000000000000e+01 +57 43 1.500000000000000e+01 +57 44 1.400000000000000e+01 +57 45 1.300000000000000e+01 +57 46 1.200000000000000e+01 +57 47 1.100000000000000e+01 +57 48 1.000000000000000e+01 +57 49 9.000000000000000e+00 +57 50 8.000000000000000e+00 +57 51 7.000000000000000e+00 +57 52 6.000000000000000e+00 +57 53 5.000000000000000e+00 +57 54 4.000000000000000e+00 +57 55 3.000000000000000e+00 +57 56 2.000000000000000e+00 +57 57 2.000000000000000e+00 +57 58 2.000000000000000e+00 +57 59 3.000000000000000e+00 +57 60 4.000000000000000e+00 +57 61 5.000000000000000e+00 +57 62 6.000000000000000e+00 +57 63 7.000000000000000e+00 +57 64 8.000000000000000e+00 +57 65 9.000000000000000e+00 +57 66 1.000000000000000e+01 +57 67 1.100000000000000e+01 +57 68 1.200000000000000e+01 +57 69 1.300000000000000e+01 +57 70 1.400000000000000e+01 +57 71 1.500000000000000e+01 +57 72 1.600000000000000e+01 +57 73 1.700000000000000e+01 +57 74 1.800000000000000e+01 +57 75 1.900000000000000e+01 +57 76 2.000000000000000e+01 +57 77 2.100000000000000e+01 +57 78 2.200000000000000e+01 +57 79 2.300000000000000e+01 +57 80 2.400000000000000e+01 +57 81 2.500000000000000e+01 +57 82 2.600000000000000e+01 +57 83 2.700000000000000e+01 +57 84 2.800000000000000e+01 +57 85 2.900000000000000e+01 +57 86 3.000000000000000e+01 +57 87 3.100000000000000e+01 +57 88 3.200000000000000e+01 +57 89 3.300000000000000e+01 +57 90 3.400000000000000e+01 +57 91 3.500000000000000e+01 +57 92 3.600000000000000e+01 +57 93 3.700000000000000e+01 +57 94 3.800000000000000e+01 +57 95 3.900000000000000e+01 +57 96 4.000000000000000e+01 +57 97 4.100000000000000e+01 +57 98 4.200000000000000e+01 +57 99 4.300000000000000e+01 +57 100 4.400000000000000e+01 +58 1 5.800000000000000e+01 +58 2 5.700000000000000e+01 +58 3 5.600000000000000e+01 +58 4 5.500000000000000e+01 +58 5 5.400000000000000e+01 +58 6 5.300000000000000e+01 +58 7 5.200000000000000e+01 +58 8 5.100000000000000e+01 +58 9 5.000000000000000e+01 +58 10 4.900000000000000e+01 +58 11 4.800000000000000e+01 +58 12 4.700000000000000e+01 +58 13 4.600000000000000e+01 +58 14 4.500000000000000e+01 +58 15 4.400000000000000e+01 +58 16 4.300000000000000e+01 +58 17 4.200000000000000e+01 +58 18 4.100000000000000e+01 +58 19 4.000000000000000e+01 +58 20 3.900000000000000e+01 +58 21 3.800000000000000e+01 +58 22 3.700000000000000e+01 +58 23 3.600000000000000e+01 +58 24 3.500000000000000e+01 +58 25 3.400000000000000e+01 +58 26 3.300000000000000e+01 +58 27 3.200000000000000e+01 +58 28 3.100000000000000e+01 +58 29 3.000000000000000e+01 +58 30 2.900000000000000e+01 +58 31 2.800000000000000e+01 +58 32 2.700000000000000e+01 +58 33 2.600000000000000e+01 +58 34 2.500000000000000e+01 +58 35 2.400000000000000e+01 +58 36 2.300000000000000e+01 +58 37 2.200000000000000e+01 +58 38 2.100000000000000e+01 +58 39 2.000000000000000e+01 +58 40 1.900000000000000e+01 +58 41 1.800000000000000e+01 +58 42 1.700000000000000e+01 +58 43 1.600000000000000e+01 +58 44 1.500000000000000e+01 +58 45 1.400000000000000e+01 +58 46 1.300000000000000e+01 +58 47 1.200000000000000e+01 +58 48 1.100000000000000e+01 +58 49 1.000000000000000e+01 +58 50 9.000000000000000e+00 +58 51 8.000000000000000e+00 +58 52 7.000000000000000e+00 +58 53 6.000000000000000e+00 +58 54 5.000000000000000e+00 +58 55 4.000000000000000e+00 +58 56 3.000000000000000e+00 +58 57 2.000000000000000e+00 +58 58 2.000000000000000e+00 +58 59 2.000000000000000e+00 +58 60 3.000000000000000e+00 +58 61 4.000000000000000e+00 +58 62 5.000000000000000e+00 +58 63 6.000000000000000e+00 +58 64 7.000000000000000e+00 +58 65 8.000000000000000e+00 +58 66 9.000000000000000e+00 +58 67 1.000000000000000e+01 +58 68 1.100000000000000e+01 +58 69 1.200000000000000e+01 +58 70 1.300000000000000e+01 +58 71 1.400000000000000e+01 +58 72 1.500000000000000e+01 +58 73 1.600000000000000e+01 +58 74 1.700000000000000e+01 +58 75 1.800000000000000e+01 +58 76 1.900000000000000e+01 +58 77 2.000000000000000e+01 +58 78 2.100000000000000e+01 +58 79 2.200000000000000e+01 +58 80 2.300000000000000e+01 +58 81 2.400000000000000e+01 +58 82 2.500000000000000e+01 +58 83 2.600000000000000e+01 +58 84 2.700000000000000e+01 +58 85 2.800000000000000e+01 +58 86 2.900000000000000e+01 +58 87 3.000000000000000e+01 +58 88 3.100000000000000e+01 +58 89 3.200000000000000e+01 +58 90 3.300000000000000e+01 +58 91 3.400000000000000e+01 +58 92 3.500000000000000e+01 +58 93 3.600000000000000e+01 +58 94 3.700000000000000e+01 +58 95 3.800000000000000e+01 +58 96 3.900000000000000e+01 +58 97 4.000000000000000e+01 +58 98 4.100000000000000e+01 +58 99 4.200000000000000e+01 +58 100 4.300000000000000e+01 +59 1 5.900000000000000e+01 +59 2 5.800000000000000e+01 +59 3 5.700000000000000e+01 +59 4 5.600000000000000e+01 +59 5 5.500000000000000e+01 +59 6 5.400000000000000e+01 +59 7 5.300000000000000e+01 +59 8 5.200000000000000e+01 +59 9 5.100000000000000e+01 +59 10 5.000000000000000e+01 +59 11 4.900000000000000e+01 +59 12 4.800000000000000e+01 +59 13 4.700000000000000e+01 +59 14 4.600000000000000e+01 +59 15 4.500000000000000e+01 +59 16 4.400000000000000e+01 +59 17 4.300000000000000e+01 +59 18 4.200000000000000e+01 +59 19 4.100000000000000e+01 +59 20 4.000000000000000e+01 +59 21 3.900000000000000e+01 +59 22 3.800000000000000e+01 +59 23 3.700000000000000e+01 +59 24 3.600000000000000e+01 +59 25 3.500000000000000e+01 +59 26 3.400000000000000e+01 +59 27 3.300000000000000e+01 +59 28 3.200000000000000e+01 +59 29 3.100000000000000e+01 +59 30 3.000000000000000e+01 +59 31 2.900000000000000e+01 +59 32 2.800000000000000e+01 +59 33 2.700000000000000e+01 +59 34 2.600000000000000e+01 +59 35 2.500000000000000e+01 +59 36 2.400000000000000e+01 +59 37 2.300000000000000e+01 +59 38 2.200000000000000e+01 +59 39 2.100000000000000e+01 +59 40 2.000000000000000e+01 +59 41 1.900000000000000e+01 +59 42 1.800000000000000e+01 +59 43 1.700000000000000e+01 +59 44 1.600000000000000e+01 +59 45 1.500000000000000e+01 +59 46 1.400000000000000e+01 +59 47 1.300000000000000e+01 +59 48 1.200000000000000e+01 +59 49 1.100000000000000e+01 +59 50 1.000000000000000e+01 +59 51 9.000000000000000e+00 +59 52 8.000000000000000e+00 +59 53 7.000000000000000e+00 +59 54 6.000000000000000e+00 +59 55 5.000000000000000e+00 +59 56 4.000000000000000e+00 +59 57 3.000000000000000e+00 +59 58 2.000000000000000e+00 +59 59 2.000000000000000e+00 +59 60 2.000000000000000e+00 +59 61 3.000000000000000e+00 +59 62 4.000000000000000e+00 +59 63 5.000000000000000e+00 +59 64 6.000000000000000e+00 +59 65 7.000000000000000e+00 +59 66 8.000000000000000e+00 +59 67 9.000000000000000e+00 +59 68 1.000000000000000e+01 +59 69 1.100000000000000e+01 +59 70 1.200000000000000e+01 +59 71 1.300000000000000e+01 +59 72 1.400000000000000e+01 +59 73 1.500000000000000e+01 +59 74 1.600000000000000e+01 +59 75 1.700000000000000e+01 +59 76 1.800000000000000e+01 +59 77 1.900000000000000e+01 +59 78 2.000000000000000e+01 +59 79 2.100000000000000e+01 +59 80 2.200000000000000e+01 +59 81 2.300000000000000e+01 +59 82 2.400000000000000e+01 +59 83 2.500000000000000e+01 +59 84 2.600000000000000e+01 +59 85 2.700000000000000e+01 +59 86 2.800000000000000e+01 +59 87 2.900000000000000e+01 +59 88 3.000000000000000e+01 +59 89 3.100000000000000e+01 +59 90 3.200000000000000e+01 +59 91 3.300000000000000e+01 +59 92 3.400000000000000e+01 +59 93 3.500000000000000e+01 +59 94 3.600000000000000e+01 +59 95 3.700000000000000e+01 +59 96 3.800000000000000e+01 +59 97 3.900000000000000e+01 +59 98 4.000000000000000e+01 +59 99 4.100000000000000e+01 +59 100 4.200000000000000e+01 +60 1 6.000000000000000e+01 +60 2 5.900000000000000e+01 +60 3 5.800000000000000e+01 +60 4 5.700000000000000e+01 +60 5 5.600000000000000e+01 +60 6 5.500000000000000e+01 +60 7 5.400000000000000e+01 +60 8 5.300000000000000e+01 +60 9 5.200000000000000e+01 +60 10 5.100000000000000e+01 +60 11 5.000000000000000e+01 +60 12 4.900000000000000e+01 +60 13 4.800000000000000e+01 +60 14 4.700000000000000e+01 +60 15 4.600000000000000e+01 +60 16 4.500000000000000e+01 +60 17 4.400000000000000e+01 +60 18 4.300000000000000e+01 +60 19 4.200000000000000e+01 +60 20 4.100000000000000e+01 +60 21 4.000000000000000e+01 +60 22 3.900000000000000e+01 +60 23 3.800000000000000e+01 +60 24 3.700000000000000e+01 +60 25 3.600000000000000e+01 +60 26 3.500000000000000e+01 +60 27 3.400000000000000e+01 +60 28 3.300000000000000e+01 +60 29 3.200000000000000e+01 +60 30 3.100000000000000e+01 +60 31 3.000000000000000e+01 +60 32 2.900000000000000e+01 +60 33 2.800000000000000e+01 +60 34 2.700000000000000e+01 +60 35 2.600000000000000e+01 +60 36 2.500000000000000e+01 +60 37 2.400000000000000e+01 +60 38 2.300000000000000e+01 +60 39 2.200000000000000e+01 +60 40 2.100000000000000e+01 +60 41 2.000000000000000e+01 +60 42 1.900000000000000e+01 +60 43 1.800000000000000e+01 +60 44 1.700000000000000e+01 +60 45 1.600000000000000e+01 +60 46 1.500000000000000e+01 +60 47 1.400000000000000e+01 +60 48 1.300000000000000e+01 +60 49 1.200000000000000e+01 +60 50 1.100000000000000e+01 +60 51 1.000000000000000e+01 +60 52 9.000000000000000e+00 +60 53 8.000000000000000e+00 +60 54 7.000000000000000e+00 +60 55 6.000000000000000e+00 +60 56 5.000000000000000e+00 +60 57 4.000000000000000e+00 +60 58 3.000000000000000e+00 +60 59 2.000000000000000e+00 +60 60 2.000000000000000e+00 +60 61 2.000000000000000e+00 +60 62 3.000000000000000e+00 +60 63 4.000000000000000e+00 +60 64 5.000000000000000e+00 +60 65 6.000000000000000e+00 +60 66 7.000000000000000e+00 +60 67 8.000000000000000e+00 +60 68 9.000000000000000e+00 +60 69 1.000000000000000e+01 +60 70 1.100000000000000e+01 +60 71 1.200000000000000e+01 +60 72 1.300000000000000e+01 +60 73 1.400000000000000e+01 +60 74 1.500000000000000e+01 +60 75 1.600000000000000e+01 +60 76 1.700000000000000e+01 +60 77 1.800000000000000e+01 +60 78 1.900000000000000e+01 +60 79 2.000000000000000e+01 +60 80 2.100000000000000e+01 +60 81 2.200000000000000e+01 +60 82 2.300000000000000e+01 +60 83 2.400000000000000e+01 +60 84 2.500000000000000e+01 +60 85 2.600000000000000e+01 +60 86 2.700000000000000e+01 +60 87 2.800000000000000e+01 +60 88 2.900000000000000e+01 +60 89 3.000000000000000e+01 +60 90 3.100000000000000e+01 +60 91 3.200000000000000e+01 +60 92 3.300000000000000e+01 +60 93 3.400000000000000e+01 +60 94 3.500000000000000e+01 +60 95 3.600000000000000e+01 +60 96 3.700000000000000e+01 +60 97 3.800000000000000e+01 +60 98 3.900000000000000e+01 +60 99 4.000000000000000e+01 +60 100 4.100000000000000e+01 +61 1 6.100000000000000e+01 +61 2 6.000000000000000e+01 +61 3 5.900000000000000e+01 +61 4 5.800000000000000e+01 +61 5 5.700000000000000e+01 +61 6 5.600000000000000e+01 +61 7 5.500000000000000e+01 +61 8 5.400000000000000e+01 +61 9 5.300000000000000e+01 +61 10 5.200000000000000e+01 +61 11 5.100000000000000e+01 +61 12 5.000000000000000e+01 +61 13 4.900000000000000e+01 +61 14 4.800000000000000e+01 +61 15 4.700000000000000e+01 +61 16 4.600000000000000e+01 +61 17 4.500000000000000e+01 +61 18 4.400000000000000e+01 +61 19 4.300000000000000e+01 +61 20 4.200000000000000e+01 +61 21 4.100000000000000e+01 +61 22 4.000000000000000e+01 +61 23 3.900000000000000e+01 +61 24 3.800000000000000e+01 +61 25 3.700000000000000e+01 +61 26 3.600000000000000e+01 +61 27 3.500000000000000e+01 +61 28 3.400000000000000e+01 +61 29 3.300000000000000e+01 +61 30 3.200000000000000e+01 +61 31 3.100000000000000e+01 +61 32 3.000000000000000e+01 +61 33 2.900000000000000e+01 +61 34 2.800000000000000e+01 +61 35 2.700000000000000e+01 +61 36 2.600000000000000e+01 +61 37 2.500000000000000e+01 +61 38 2.400000000000000e+01 +61 39 2.300000000000000e+01 +61 40 2.200000000000000e+01 +61 41 2.100000000000000e+01 +61 42 2.000000000000000e+01 +61 43 1.900000000000000e+01 +61 44 1.800000000000000e+01 +61 45 1.700000000000000e+01 +61 46 1.600000000000000e+01 +61 47 1.500000000000000e+01 +61 48 1.400000000000000e+01 +61 49 1.300000000000000e+01 +61 50 1.200000000000000e+01 +61 51 1.100000000000000e+01 +61 52 1.000000000000000e+01 +61 53 9.000000000000000e+00 +61 54 8.000000000000000e+00 +61 55 7.000000000000000e+00 +61 56 6.000000000000000e+00 +61 57 5.000000000000000e+00 +61 58 4.000000000000000e+00 +61 59 3.000000000000000e+00 +61 60 2.000000000000000e+00 +61 61 2.000000000000000e+00 +61 62 2.000000000000000e+00 +61 63 3.000000000000000e+00 +61 64 4.000000000000000e+00 +61 65 5.000000000000000e+00 +61 66 6.000000000000000e+00 +61 67 7.000000000000000e+00 +61 68 8.000000000000000e+00 +61 69 9.000000000000000e+00 +61 70 1.000000000000000e+01 +61 71 1.100000000000000e+01 +61 72 1.200000000000000e+01 +61 73 1.300000000000000e+01 +61 74 1.400000000000000e+01 +61 75 1.500000000000000e+01 +61 76 1.600000000000000e+01 +61 77 1.700000000000000e+01 +61 78 1.800000000000000e+01 +61 79 1.900000000000000e+01 +61 80 2.000000000000000e+01 +61 81 2.100000000000000e+01 +61 82 2.200000000000000e+01 +61 83 2.300000000000000e+01 +61 84 2.400000000000000e+01 +61 85 2.500000000000000e+01 +61 86 2.600000000000000e+01 +61 87 2.700000000000000e+01 +61 88 2.800000000000000e+01 +61 89 2.900000000000000e+01 +61 90 3.000000000000000e+01 +61 91 3.100000000000000e+01 +61 92 3.200000000000000e+01 +61 93 3.300000000000000e+01 +61 94 3.400000000000000e+01 +61 95 3.500000000000000e+01 +61 96 3.600000000000000e+01 +61 97 3.700000000000000e+01 +61 98 3.800000000000000e+01 +61 99 3.900000000000000e+01 +61 100 4.000000000000000e+01 +62 1 6.200000000000000e+01 +62 2 6.100000000000000e+01 +62 3 6.000000000000000e+01 +62 4 5.900000000000000e+01 +62 5 5.800000000000000e+01 +62 6 5.700000000000000e+01 +62 7 5.600000000000000e+01 +62 8 5.500000000000000e+01 +62 9 5.400000000000000e+01 +62 10 5.300000000000000e+01 +62 11 5.200000000000000e+01 +62 12 5.100000000000000e+01 +62 13 5.000000000000000e+01 +62 14 4.900000000000000e+01 +62 15 4.800000000000000e+01 +62 16 4.700000000000000e+01 +62 17 4.600000000000000e+01 +62 18 4.500000000000000e+01 +62 19 4.400000000000000e+01 +62 20 4.300000000000000e+01 +62 21 4.200000000000000e+01 +62 22 4.100000000000000e+01 +62 23 4.000000000000000e+01 +62 24 3.900000000000000e+01 +62 25 3.800000000000000e+01 +62 26 3.700000000000000e+01 +62 27 3.600000000000000e+01 +62 28 3.500000000000000e+01 +62 29 3.400000000000000e+01 +62 30 3.300000000000000e+01 +62 31 3.200000000000000e+01 +62 32 3.100000000000000e+01 +62 33 3.000000000000000e+01 +62 34 2.900000000000000e+01 +62 35 2.800000000000000e+01 +62 36 2.700000000000000e+01 +62 37 2.600000000000000e+01 +62 38 2.500000000000000e+01 +62 39 2.400000000000000e+01 +62 40 2.300000000000000e+01 +62 41 2.200000000000000e+01 +62 42 2.100000000000000e+01 +62 43 2.000000000000000e+01 +62 44 1.900000000000000e+01 +62 45 1.800000000000000e+01 +62 46 1.700000000000000e+01 +62 47 1.600000000000000e+01 +62 48 1.500000000000000e+01 +62 49 1.400000000000000e+01 +62 50 1.300000000000000e+01 +62 51 1.200000000000000e+01 +62 52 1.100000000000000e+01 +62 53 1.000000000000000e+01 +62 54 9.000000000000000e+00 +62 55 8.000000000000000e+00 +62 56 7.000000000000000e+00 +62 57 6.000000000000000e+00 +62 58 5.000000000000000e+00 +62 59 4.000000000000000e+00 +62 60 3.000000000000000e+00 +62 61 2.000000000000000e+00 +62 62 2.000000000000000e+00 +62 63 2.000000000000000e+00 +62 64 3.000000000000000e+00 +62 65 4.000000000000000e+00 +62 66 5.000000000000000e+00 +62 67 6.000000000000000e+00 +62 68 7.000000000000000e+00 +62 69 8.000000000000000e+00 +62 70 9.000000000000000e+00 +62 71 1.000000000000000e+01 +62 72 1.100000000000000e+01 +62 73 1.200000000000000e+01 +62 74 1.300000000000000e+01 +62 75 1.400000000000000e+01 +62 76 1.500000000000000e+01 +62 77 1.600000000000000e+01 +62 78 1.700000000000000e+01 +62 79 1.800000000000000e+01 +62 80 1.900000000000000e+01 +62 81 2.000000000000000e+01 +62 82 2.100000000000000e+01 +62 83 2.200000000000000e+01 +62 84 2.300000000000000e+01 +62 85 2.400000000000000e+01 +62 86 2.500000000000000e+01 +62 87 2.600000000000000e+01 +62 88 2.700000000000000e+01 +62 89 2.800000000000000e+01 +62 90 2.900000000000000e+01 +62 91 3.000000000000000e+01 +62 92 3.100000000000000e+01 +62 93 3.200000000000000e+01 +62 94 3.300000000000000e+01 +62 95 3.400000000000000e+01 +62 96 3.500000000000000e+01 +62 97 3.600000000000000e+01 +62 98 3.700000000000000e+01 +62 99 3.800000000000000e+01 +62 100 3.900000000000000e+01 +63 1 6.300000000000000e+01 +63 2 6.200000000000000e+01 +63 3 6.100000000000000e+01 +63 4 6.000000000000000e+01 +63 5 5.900000000000000e+01 +63 6 5.800000000000000e+01 +63 7 5.700000000000000e+01 +63 8 5.600000000000000e+01 +63 9 5.500000000000000e+01 +63 10 5.400000000000000e+01 +63 11 5.300000000000000e+01 +63 12 5.200000000000000e+01 +63 13 5.100000000000000e+01 +63 14 5.000000000000000e+01 +63 15 4.900000000000000e+01 +63 16 4.800000000000000e+01 +63 17 4.700000000000000e+01 +63 18 4.600000000000000e+01 +63 19 4.500000000000000e+01 +63 20 4.400000000000000e+01 +63 21 4.300000000000000e+01 +63 22 4.200000000000000e+01 +63 23 4.100000000000000e+01 +63 24 4.000000000000000e+01 +63 25 3.900000000000000e+01 +63 26 3.800000000000000e+01 +63 27 3.700000000000000e+01 +63 28 3.600000000000000e+01 +63 29 3.500000000000000e+01 +63 30 3.400000000000000e+01 +63 31 3.300000000000000e+01 +63 32 3.200000000000000e+01 +63 33 3.100000000000000e+01 +63 34 3.000000000000000e+01 +63 35 2.900000000000000e+01 +63 36 2.800000000000000e+01 +63 37 2.700000000000000e+01 +63 38 2.600000000000000e+01 +63 39 2.500000000000000e+01 +63 40 2.400000000000000e+01 +63 41 2.300000000000000e+01 +63 42 2.200000000000000e+01 +63 43 2.100000000000000e+01 +63 44 2.000000000000000e+01 +63 45 1.900000000000000e+01 +63 46 1.800000000000000e+01 +63 47 1.700000000000000e+01 +63 48 1.600000000000000e+01 +63 49 1.500000000000000e+01 +63 50 1.400000000000000e+01 +63 51 1.300000000000000e+01 +63 52 1.200000000000000e+01 +63 53 1.100000000000000e+01 +63 54 1.000000000000000e+01 +63 55 9.000000000000000e+00 +63 56 8.000000000000000e+00 +63 57 7.000000000000000e+00 +63 58 6.000000000000000e+00 +63 59 5.000000000000000e+00 +63 60 4.000000000000000e+00 +63 61 3.000000000000000e+00 +63 62 2.000000000000000e+00 +63 63 2.000000000000000e+00 +63 64 2.000000000000000e+00 +63 65 3.000000000000000e+00 +63 66 4.000000000000000e+00 +63 67 5.000000000000000e+00 +63 68 6.000000000000000e+00 +63 69 7.000000000000000e+00 +63 70 8.000000000000000e+00 +63 71 9.000000000000000e+00 +63 72 1.000000000000000e+01 +63 73 1.100000000000000e+01 +63 74 1.200000000000000e+01 +63 75 1.300000000000000e+01 +63 76 1.400000000000000e+01 +63 77 1.500000000000000e+01 +63 78 1.600000000000000e+01 +63 79 1.700000000000000e+01 +63 80 1.800000000000000e+01 +63 81 1.900000000000000e+01 +63 82 2.000000000000000e+01 +63 83 2.100000000000000e+01 +63 84 2.200000000000000e+01 +63 85 2.300000000000000e+01 +63 86 2.400000000000000e+01 +63 87 2.500000000000000e+01 +63 88 2.600000000000000e+01 +63 89 2.700000000000000e+01 +63 90 2.800000000000000e+01 +63 91 2.900000000000000e+01 +63 92 3.000000000000000e+01 +63 93 3.100000000000000e+01 +63 94 3.200000000000000e+01 +63 95 3.300000000000000e+01 +63 96 3.400000000000000e+01 +63 97 3.500000000000000e+01 +63 98 3.600000000000000e+01 +63 99 3.700000000000000e+01 +63 100 3.800000000000000e+01 +64 1 6.400000000000000e+01 +64 2 6.300000000000000e+01 +64 3 6.200000000000000e+01 +64 4 6.100000000000000e+01 +64 5 6.000000000000000e+01 +64 6 5.900000000000000e+01 +64 7 5.800000000000000e+01 +64 8 5.700000000000000e+01 +64 9 5.600000000000000e+01 +64 10 5.500000000000000e+01 +64 11 5.400000000000000e+01 +64 12 5.300000000000000e+01 +64 13 5.200000000000000e+01 +64 14 5.100000000000000e+01 +64 15 5.000000000000000e+01 +64 16 4.900000000000000e+01 +64 17 4.800000000000000e+01 +64 18 4.700000000000000e+01 +64 19 4.600000000000000e+01 +64 20 4.500000000000000e+01 +64 21 4.400000000000000e+01 +64 22 4.300000000000000e+01 +64 23 4.200000000000000e+01 +64 24 4.100000000000000e+01 +64 25 4.000000000000000e+01 +64 26 3.900000000000000e+01 +64 27 3.800000000000000e+01 +64 28 3.700000000000000e+01 +64 29 3.600000000000000e+01 +64 30 3.500000000000000e+01 +64 31 3.400000000000000e+01 +64 32 3.300000000000000e+01 +64 33 3.200000000000000e+01 +64 34 3.100000000000000e+01 +64 35 3.000000000000000e+01 +64 36 2.900000000000000e+01 +64 37 2.800000000000000e+01 +64 38 2.700000000000000e+01 +64 39 2.600000000000000e+01 +64 40 2.500000000000000e+01 +64 41 2.400000000000000e+01 +64 42 2.300000000000000e+01 +64 43 2.200000000000000e+01 +64 44 2.100000000000000e+01 +64 45 2.000000000000000e+01 +64 46 1.900000000000000e+01 +64 47 1.800000000000000e+01 +64 48 1.700000000000000e+01 +64 49 1.600000000000000e+01 +64 50 1.500000000000000e+01 +64 51 1.400000000000000e+01 +64 52 1.300000000000000e+01 +64 53 1.200000000000000e+01 +64 54 1.100000000000000e+01 +64 55 1.000000000000000e+01 +64 56 9.000000000000000e+00 +64 57 8.000000000000000e+00 +64 58 7.000000000000000e+00 +64 59 6.000000000000000e+00 +64 60 5.000000000000000e+00 +64 61 4.000000000000000e+00 +64 62 3.000000000000000e+00 +64 63 2.000000000000000e+00 +64 64 2.000000000000000e+00 +64 65 2.000000000000000e+00 +64 66 3.000000000000000e+00 +64 67 4.000000000000000e+00 +64 68 5.000000000000000e+00 +64 69 6.000000000000000e+00 +64 70 7.000000000000000e+00 +64 71 8.000000000000000e+00 +64 72 9.000000000000000e+00 +64 73 1.000000000000000e+01 +64 74 1.100000000000000e+01 +64 75 1.200000000000000e+01 +64 76 1.300000000000000e+01 +64 77 1.400000000000000e+01 +64 78 1.500000000000000e+01 +64 79 1.600000000000000e+01 +64 80 1.700000000000000e+01 +64 81 1.800000000000000e+01 +64 82 1.900000000000000e+01 +64 83 2.000000000000000e+01 +64 84 2.100000000000000e+01 +64 85 2.200000000000000e+01 +64 86 2.300000000000000e+01 +64 87 2.400000000000000e+01 +64 88 2.500000000000000e+01 +64 89 2.600000000000000e+01 +64 90 2.700000000000000e+01 +64 91 2.800000000000000e+01 +64 92 2.900000000000000e+01 +64 93 3.000000000000000e+01 +64 94 3.100000000000000e+01 +64 95 3.200000000000000e+01 +64 96 3.300000000000000e+01 +64 97 3.400000000000000e+01 +64 98 3.500000000000000e+01 +64 99 3.600000000000000e+01 +64 100 3.700000000000000e+01 +65 1 6.500000000000000e+01 +65 2 6.400000000000000e+01 +65 3 6.300000000000000e+01 +65 4 6.200000000000000e+01 +65 5 6.100000000000000e+01 +65 6 6.000000000000000e+01 +65 7 5.900000000000000e+01 +65 8 5.800000000000000e+01 +65 9 5.700000000000000e+01 +65 10 5.600000000000000e+01 +65 11 5.500000000000000e+01 +65 12 5.400000000000000e+01 +65 13 5.300000000000000e+01 +65 14 5.200000000000000e+01 +65 15 5.100000000000000e+01 +65 16 5.000000000000000e+01 +65 17 4.900000000000000e+01 +65 18 4.800000000000000e+01 +65 19 4.700000000000000e+01 +65 20 4.600000000000000e+01 +65 21 4.500000000000000e+01 +65 22 4.400000000000000e+01 +65 23 4.300000000000000e+01 +65 24 4.200000000000000e+01 +65 25 4.100000000000000e+01 +65 26 4.000000000000000e+01 +65 27 3.900000000000000e+01 +65 28 3.800000000000000e+01 +65 29 3.700000000000000e+01 +65 30 3.600000000000000e+01 +65 31 3.500000000000000e+01 +65 32 3.400000000000000e+01 +65 33 3.300000000000000e+01 +65 34 3.200000000000000e+01 +65 35 3.100000000000000e+01 +65 36 3.000000000000000e+01 +65 37 2.900000000000000e+01 +65 38 2.800000000000000e+01 +65 39 2.700000000000000e+01 +65 40 2.600000000000000e+01 +65 41 2.500000000000000e+01 +65 42 2.400000000000000e+01 +65 43 2.300000000000000e+01 +65 44 2.200000000000000e+01 +65 45 2.100000000000000e+01 +65 46 2.000000000000000e+01 +65 47 1.900000000000000e+01 +65 48 1.800000000000000e+01 +65 49 1.700000000000000e+01 +65 50 1.600000000000000e+01 +65 51 1.500000000000000e+01 +65 52 1.400000000000000e+01 +65 53 1.300000000000000e+01 +65 54 1.200000000000000e+01 +65 55 1.100000000000000e+01 +65 56 1.000000000000000e+01 +65 57 9.000000000000000e+00 +65 58 8.000000000000000e+00 +65 59 7.000000000000000e+00 +65 60 6.000000000000000e+00 +65 61 5.000000000000000e+00 +65 62 4.000000000000000e+00 +65 63 3.000000000000000e+00 +65 64 2.000000000000000e+00 +65 65 2.000000000000000e+00 +65 66 2.000000000000000e+00 +65 67 3.000000000000000e+00 +65 68 4.000000000000000e+00 +65 69 5.000000000000000e+00 +65 70 6.000000000000000e+00 +65 71 7.000000000000000e+00 +65 72 8.000000000000000e+00 +65 73 9.000000000000000e+00 +65 74 1.000000000000000e+01 +65 75 1.100000000000000e+01 +65 76 1.200000000000000e+01 +65 77 1.300000000000000e+01 +65 78 1.400000000000000e+01 +65 79 1.500000000000000e+01 +65 80 1.600000000000000e+01 +65 81 1.700000000000000e+01 +65 82 1.800000000000000e+01 +65 83 1.900000000000000e+01 +65 84 2.000000000000000e+01 +65 85 2.100000000000000e+01 +65 86 2.200000000000000e+01 +65 87 2.300000000000000e+01 +65 88 2.400000000000000e+01 +65 89 2.500000000000000e+01 +65 90 2.600000000000000e+01 +65 91 2.700000000000000e+01 +65 92 2.800000000000000e+01 +65 93 2.900000000000000e+01 +65 94 3.000000000000000e+01 +65 95 3.100000000000000e+01 +65 96 3.200000000000000e+01 +65 97 3.300000000000000e+01 +65 98 3.400000000000000e+01 +65 99 3.500000000000000e+01 +65 100 3.600000000000000e+01 +66 1 6.600000000000000e+01 +66 2 6.500000000000000e+01 +66 3 6.400000000000000e+01 +66 4 6.300000000000000e+01 +66 5 6.200000000000000e+01 +66 6 6.100000000000000e+01 +66 7 6.000000000000000e+01 +66 8 5.900000000000000e+01 +66 9 5.800000000000000e+01 +66 10 5.700000000000000e+01 +66 11 5.600000000000000e+01 +66 12 5.500000000000000e+01 +66 13 5.400000000000000e+01 +66 14 5.300000000000000e+01 +66 15 5.200000000000000e+01 +66 16 5.100000000000000e+01 +66 17 5.000000000000000e+01 +66 18 4.900000000000000e+01 +66 19 4.800000000000000e+01 +66 20 4.700000000000000e+01 +66 21 4.600000000000000e+01 +66 22 4.500000000000000e+01 +66 23 4.400000000000000e+01 +66 24 4.300000000000000e+01 +66 25 4.200000000000000e+01 +66 26 4.100000000000000e+01 +66 27 4.000000000000000e+01 +66 28 3.900000000000000e+01 +66 29 3.800000000000000e+01 +66 30 3.700000000000000e+01 +66 31 3.600000000000000e+01 +66 32 3.500000000000000e+01 +66 33 3.400000000000000e+01 +66 34 3.300000000000000e+01 +66 35 3.200000000000000e+01 +66 36 3.100000000000000e+01 +66 37 3.000000000000000e+01 +66 38 2.900000000000000e+01 +66 39 2.800000000000000e+01 +66 40 2.700000000000000e+01 +66 41 2.600000000000000e+01 +66 42 2.500000000000000e+01 +66 43 2.400000000000000e+01 +66 44 2.300000000000000e+01 +66 45 2.200000000000000e+01 +66 46 2.100000000000000e+01 +66 47 2.000000000000000e+01 +66 48 1.900000000000000e+01 +66 49 1.800000000000000e+01 +66 50 1.700000000000000e+01 +66 51 1.600000000000000e+01 +66 52 1.500000000000000e+01 +66 53 1.400000000000000e+01 +66 54 1.300000000000000e+01 +66 55 1.200000000000000e+01 +66 56 1.100000000000000e+01 +66 57 1.000000000000000e+01 +66 58 9.000000000000000e+00 +66 59 8.000000000000000e+00 +66 60 7.000000000000000e+00 +66 61 6.000000000000000e+00 +66 62 5.000000000000000e+00 +66 63 4.000000000000000e+00 +66 64 3.000000000000000e+00 +66 65 2.000000000000000e+00 +66 66 2.000000000000000e+00 +66 67 2.000000000000000e+00 +66 68 3.000000000000000e+00 +66 69 4.000000000000000e+00 +66 70 5.000000000000000e+00 +66 71 6.000000000000000e+00 +66 72 7.000000000000000e+00 +66 73 8.000000000000000e+00 +66 74 9.000000000000000e+00 +66 75 1.000000000000000e+01 +66 76 1.100000000000000e+01 +66 77 1.200000000000000e+01 +66 78 1.300000000000000e+01 +66 79 1.400000000000000e+01 +66 80 1.500000000000000e+01 +66 81 1.600000000000000e+01 +66 82 1.700000000000000e+01 +66 83 1.800000000000000e+01 +66 84 1.900000000000000e+01 +66 85 2.000000000000000e+01 +66 86 2.100000000000000e+01 +66 87 2.200000000000000e+01 +66 88 2.300000000000000e+01 +66 89 2.400000000000000e+01 +66 90 2.500000000000000e+01 +66 91 2.600000000000000e+01 +66 92 2.700000000000000e+01 +66 93 2.800000000000000e+01 +66 94 2.900000000000000e+01 +66 95 3.000000000000000e+01 +66 96 3.100000000000000e+01 +66 97 3.200000000000000e+01 +66 98 3.300000000000000e+01 +66 99 3.400000000000000e+01 +66 100 3.500000000000000e+01 +67 1 6.700000000000000e+01 +67 2 6.600000000000000e+01 +67 3 6.500000000000000e+01 +67 4 6.400000000000000e+01 +67 5 6.300000000000000e+01 +67 6 6.200000000000000e+01 +67 7 6.100000000000000e+01 +67 8 6.000000000000000e+01 +67 9 5.900000000000000e+01 +67 10 5.800000000000000e+01 +67 11 5.700000000000000e+01 +67 12 5.600000000000000e+01 +67 13 5.500000000000000e+01 +67 14 5.400000000000000e+01 +67 15 5.300000000000000e+01 +67 16 5.200000000000000e+01 +67 17 5.100000000000000e+01 +67 18 5.000000000000000e+01 +67 19 4.900000000000000e+01 +67 20 4.800000000000000e+01 +67 21 4.700000000000000e+01 +67 22 4.600000000000000e+01 +67 23 4.500000000000000e+01 +67 24 4.400000000000000e+01 +67 25 4.300000000000000e+01 +67 26 4.200000000000000e+01 +67 27 4.100000000000000e+01 +67 28 4.000000000000000e+01 +67 29 3.900000000000000e+01 +67 30 3.800000000000000e+01 +67 31 3.700000000000000e+01 +67 32 3.600000000000000e+01 +67 33 3.500000000000000e+01 +67 34 3.400000000000000e+01 +67 35 3.300000000000000e+01 +67 36 3.200000000000000e+01 +67 37 3.100000000000000e+01 +67 38 3.000000000000000e+01 +67 39 2.900000000000000e+01 +67 40 2.800000000000000e+01 +67 41 2.700000000000000e+01 +67 42 2.600000000000000e+01 +67 43 2.500000000000000e+01 +67 44 2.400000000000000e+01 +67 45 2.300000000000000e+01 +67 46 2.200000000000000e+01 +67 47 2.100000000000000e+01 +67 48 2.000000000000000e+01 +67 49 1.900000000000000e+01 +67 50 1.800000000000000e+01 +67 51 1.700000000000000e+01 +67 52 1.600000000000000e+01 +67 53 1.500000000000000e+01 +67 54 1.400000000000000e+01 +67 55 1.300000000000000e+01 +67 56 1.200000000000000e+01 +67 57 1.100000000000000e+01 +67 58 1.000000000000000e+01 +67 59 9.000000000000000e+00 +67 60 8.000000000000000e+00 +67 61 7.000000000000000e+00 +67 62 6.000000000000000e+00 +67 63 5.000000000000000e+00 +67 64 4.000000000000000e+00 +67 65 3.000000000000000e+00 +67 66 2.000000000000000e+00 +67 67 2.000000000000000e+00 +67 68 2.000000000000000e+00 +67 69 3.000000000000000e+00 +67 70 4.000000000000000e+00 +67 71 5.000000000000000e+00 +67 72 6.000000000000000e+00 +67 73 7.000000000000000e+00 +67 74 8.000000000000000e+00 +67 75 9.000000000000000e+00 +67 76 1.000000000000000e+01 +67 77 1.100000000000000e+01 +67 78 1.200000000000000e+01 +67 79 1.300000000000000e+01 +67 80 1.400000000000000e+01 +67 81 1.500000000000000e+01 +67 82 1.600000000000000e+01 +67 83 1.700000000000000e+01 +67 84 1.800000000000000e+01 +67 85 1.900000000000000e+01 +67 86 2.000000000000000e+01 +67 87 2.100000000000000e+01 +67 88 2.200000000000000e+01 +67 89 2.300000000000000e+01 +67 90 2.400000000000000e+01 +67 91 2.500000000000000e+01 +67 92 2.600000000000000e+01 +67 93 2.700000000000000e+01 +67 94 2.800000000000000e+01 +67 95 2.900000000000000e+01 +67 96 3.000000000000000e+01 +67 97 3.100000000000000e+01 +67 98 3.200000000000000e+01 +67 99 3.300000000000000e+01 +67 100 3.400000000000000e+01 +68 1 6.800000000000000e+01 +68 2 6.700000000000000e+01 +68 3 6.600000000000000e+01 +68 4 6.500000000000000e+01 +68 5 6.400000000000000e+01 +68 6 6.300000000000000e+01 +68 7 6.200000000000000e+01 +68 8 6.100000000000000e+01 +68 9 6.000000000000000e+01 +68 10 5.900000000000000e+01 +68 11 5.800000000000000e+01 +68 12 5.700000000000000e+01 +68 13 5.600000000000000e+01 +68 14 5.500000000000000e+01 +68 15 5.400000000000000e+01 +68 16 5.300000000000000e+01 +68 17 5.200000000000000e+01 +68 18 5.100000000000000e+01 +68 19 5.000000000000000e+01 +68 20 4.900000000000000e+01 +68 21 4.800000000000000e+01 +68 22 4.700000000000000e+01 +68 23 4.600000000000000e+01 +68 24 4.500000000000000e+01 +68 25 4.400000000000000e+01 +68 26 4.300000000000000e+01 +68 27 4.200000000000000e+01 +68 28 4.100000000000000e+01 +68 29 4.000000000000000e+01 +68 30 3.900000000000000e+01 +68 31 3.800000000000000e+01 +68 32 3.700000000000000e+01 +68 33 3.600000000000000e+01 +68 34 3.500000000000000e+01 +68 35 3.400000000000000e+01 +68 36 3.300000000000000e+01 +68 37 3.200000000000000e+01 +68 38 3.100000000000000e+01 +68 39 3.000000000000000e+01 +68 40 2.900000000000000e+01 +68 41 2.800000000000000e+01 +68 42 2.700000000000000e+01 +68 43 2.600000000000000e+01 +68 44 2.500000000000000e+01 +68 45 2.400000000000000e+01 +68 46 2.300000000000000e+01 +68 47 2.200000000000000e+01 +68 48 2.100000000000000e+01 +68 49 2.000000000000000e+01 +68 50 1.900000000000000e+01 +68 51 1.800000000000000e+01 +68 52 1.700000000000000e+01 +68 53 1.600000000000000e+01 +68 54 1.500000000000000e+01 +68 55 1.400000000000000e+01 +68 56 1.300000000000000e+01 +68 57 1.200000000000000e+01 +68 58 1.100000000000000e+01 +68 59 1.000000000000000e+01 +68 60 9.000000000000000e+00 +68 61 8.000000000000000e+00 +68 62 7.000000000000000e+00 +68 63 6.000000000000000e+00 +68 64 5.000000000000000e+00 +68 65 4.000000000000000e+00 +68 66 3.000000000000000e+00 +68 67 2.000000000000000e+00 +68 68 2.000000000000000e+00 +68 69 2.000000000000000e+00 +68 70 3.000000000000000e+00 +68 71 4.000000000000000e+00 +68 72 5.000000000000000e+00 +68 73 6.000000000000000e+00 +68 74 7.000000000000000e+00 +68 75 8.000000000000000e+00 +68 76 9.000000000000000e+00 +68 77 1.000000000000000e+01 +68 78 1.100000000000000e+01 +68 79 1.200000000000000e+01 +68 80 1.300000000000000e+01 +68 81 1.400000000000000e+01 +68 82 1.500000000000000e+01 +68 83 1.600000000000000e+01 +68 84 1.700000000000000e+01 +68 85 1.800000000000000e+01 +68 86 1.900000000000000e+01 +68 87 2.000000000000000e+01 +68 88 2.100000000000000e+01 +68 89 2.200000000000000e+01 +68 90 2.300000000000000e+01 +68 91 2.400000000000000e+01 +68 92 2.500000000000000e+01 +68 93 2.600000000000000e+01 +68 94 2.700000000000000e+01 +68 95 2.800000000000000e+01 +68 96 2.900000000000000e+01 +68 97 3.000000000000000e+01 +68 98 3.100000000000000e+01 +68 99 3.200000000000000e+01 +68 100 3.300000000000000e+01 +69 1 6.900000000000000e+01 +69 2 6.800000000000000e+01 +69 3 6.700000000000000e+01 +69 4 6.600000000000000e+01 +69 5 6.500000000000000e+01 +69 6 6.400000000000000e+01 +69 7 6.300000000000000e+01 +69 8 6.200000000000000e+01 +69 9 6.100000000000000e+01 +69 10 6.000000000000000e+01 +69 11 5.900000000000000e+01 +69 12 5.800000000000000e+01 +69 13 5.700000000000000e+01 +69 14 5.600000000000000e+01 +69 15 5.500000000000000e+01 +69 16 5.400000000000000e+01 +69 17 5.300000000000000e+01 +69 18 5.200000000000000e+01 +69 19 5.100000000000000e+01 +69 20 5.000000000000000e+01 +69 21 4.900000000000000e+01 +69 22 4.800000000000000e+01 +69 23 4.700000000000000e+01 +69 24 4.600000000000000e+01 +69 25 4.500000000000000e+01 +69 26 4.400000000000000e+01 +69 27 4.300000000000000e+01 +69 28 4.200000000000000e+01 +69 29 4.100000000000000e+01 +69 30 4.000000000000000e+01 +69 31 3.900000000000000e+01 +69 32 3.800000000000000e+01 +69 33 3.700000000000000e+01 +69 34 3.600000000000000e+01 +69 35 3.500000000000000e+01 +69 36 3.400000000000000e+01 +69 37 3.300000000000000e+01 +69 38 3.200000000000000e+01 +69 39 3.100000000000000e+01 +69 40 3.000000000000000e+01 +69 41 2.900000000000000e+01 +69 42 2.800000000000000e+01 +69 43 2.700000000000000e+01 +69 44 2.600000000000000e+01 +69 45 2.500000000000000e+01 +69 46 2.400000000000000e+01 +69 47 2.300000000000000e+01 +69 48 2.200000000000000e+01 +69 49 2.100000000000000e+01 +69 50 2.000000000000000e+01 +69 51 1.900000000000000e+01 +69 52 1.800000000000000e+01 +69 53 1.700000000000000e+01 +69 54 1.600000000000000e+01 +69 55 1.500000000000000e+01 +69 56 1.400000000000000e+01 +69 57 1.300000000000000e+01 +69 58 1.200000000000000e+01 +69 59 1.100000000000000e+01 +69 60 1.000000000000000e+01 +69 61 9.000000000000000e+00 +69 62 8.000000000000000e+00 +69 63 7.000000000000000e+00 +69 64 6.000000000000000e+00 +69 65 5.000000000000000e+00 +69 66 4.000000000000000e+00 +69 67 3.000000000000000e+00 +69 68 2.000000000000000e+00 +69 69 2.000000000000000e+00 +69 70 2.000000000000000e+00 +69 71 3.000000000000000e+00 +69 72 4.000000000000000e+00 +69 73 5.000000000000000e+00 +69 74 6.000000000000000e+00 +69 75 7.000000000000000e+00 +69 76 8.000000000000000e+00 +69 77 9.000000000000000e+00 +69 78 1.000000000000000e+01 +69 79 1.100000000000000e+01 +69 80 1.200000000000000e+01 +69 81 1.300000000000000e+01 +69 82 1.400000000000000e+01 +69 83 1.500000000000000e+01 +69 84 1.600000000000000e+01 +69 85 1.700000000000000e+01 +69 86 1.800000000000000e+01 +69 87 1.900000000000000e+01 +69 88 2.000000000000000e+01 +69 89 2.100000000000000e+01 +69 90 2.200000000000000e+01 +69 91 2.300000000000000e+01 +69 92 2.400000000000000e+01 +69 93 2.500000000000000e+01 +69 94 2.600000000000000e+01 +69 95 2.700000000000000e+01 +69 96 2.800000000000000e+01 +69 97 2.900000000000000e+01 +69 98 3.000000000000000e+01 +69 99 3.100000000000000e+01 +69 100 3.200000000000000e+01 +70 1 7.000000000000000e+01 +70 2 6.900000000000000e+01 +70 3 6.800000000000000e+01 +70 4 6.700000000000000e+01 +70 5 6.600000000000000e+01 +70 6 6.500000000000000e+01 +70 7 6.400000000000000e+01 +70 8 6.300000000000000e+01 +70 9 6.200000000000000e+01 +70 10 6.100000000000000e+01 +70 11 6.000000000000000e+01 +70 12 5.900000000000000e+01 +70 13 5.800000000000000e+01 +70 14 5.700000000000000e+01 +70 15 5.600000000000000e+01 +70 16 5.500000000000000e+01 +70 17 5.400000000000000e+01 +70 18 5.300000000000000e+01 +70 19 5.200000000000000e+01 +70 20 5.100000000000000e+01 +70 21 5.000000000000000e+01 +70 22 4.900000000000000e+01 +70 23 4.800000000000000e+01 +70 24 4.700000000000000e+01 +70 25 4.600000000000000e+01 +70 26 4.500000000000000e+01 +70 27 4.400000000000000e+01 +70 28 4.300000000000000e+01 +70 29 4.200000000000000e+01 +70 30 4.100000000000000e+01 +70 31 4.000000000000000e+01 +70 32 3.900000000000000e+01 +70 33 3.800000000000000e+01 +70 34 3.700000000000000e+01 +70 35 3.600000000000000e+01 +70 36 3.500000000000000e+01 +70 37 3.400000000000000e+01 +70 38 3.300000000000000e+01 +70 39 3.200000000000000e+01 +70 40 3.100000000000000e+01 +70 41 3.000000000000000e+01 +70 42 2.900000000000000e+01 +70 43 2.800000000000000e+01 +70 44 2.700000000000000e+01 +70 45 2.600000000000000e+01 +70 46 2.500000000000000e+01 +70 47 2.400000000000000e+01 +70 48 2.300000000000000e+01 +70 49 2.200000000000000e+01 +70 50 2.100000000000000e+01 +70 51 2.000000000000000e+01 +70 52 1.900000000000000e+01 +70 53 1.800000000000000e+01 +70 54 1.700000000000000e+01 +70 55 1.600000000000000e+01 +70 56 1.500000000000000e+01 +70 57 1.400000000000000e+01 +70 58 1.300000000000000e+01 +70 59 1.200000000000000e+01 +70 60 1.100000000000000e+01 +70 61 1.000000000000000e+01 +70 62 9.000000000000000e+00 +70 63 8.000000000000000e+00 +70 64 7.000000000000000e+00 +70 65 6.000000000000000e+00 +70 66 5.000000000000000e+00 +70 67 4.000000000000000e+00 +70 68 3.000000000000000e+00 +70 69 2.000000000000000e+00 +70 70 2.000000000000000e+00 +70 71 2.000000000000000e+00 +70 72 3.000000000000000e+00 +70 73 4.000000000000000e+00 +70 74 5.000000000000000e+00 +70 75 6.000000000000000e+00 +70 76 7.000000000000000e+00 +70 77 8.000000000000000e+00 +70 78 9.000000000000000e+00 +70 79 1.000000000000000e+01 +70 80 1.100000000000000e+01 +70 81 1.200000000000000e+01 +70 82 1.300000000000000e+01 +70 83 1.400000000000000e+01 +70 84 1.500000000000000e+01 +70 85 1.600000000000000e+01 +70 86 1.700000000000000e+01 +70 87 1.800000000000000e+01 +70 88 1.900000000000000e+01 +70 89 2.000000000000000e+01 +70 90 2.100000000000000e+01 +70 91 2.200000000000000e+01 +70 92 2.300000000000000e+01 +70 93 2.400000000000000e+01 +70 94 2.500000000000000e+01 +70 95 2.600000000000000e+01 +70 96 2.700000000000000e+01 +70 97 2.800000000000000e+01 +70 98 2.900000000000000e+01 +70 99 3.000000000000000e+01 +70 100 3.100000000000000e+01 +71 1 7.100000000000000e+01 +71 2 7.000000000000000e+01 +71 3 6.900000000000000e+01 +71 4 6.800000000000000e+01 +71 5 6.700000000000000e+01 +71 6 6.600000000000000e+01 +71 7 6.500000000000000e+01 +71 8 6.400000000000000e+01 +71 9 6.300000000000000e+01 +71 10 6.200000000000000e+01 +71 11 6.100000000000000e+01 +71 12 6.000000000000000e+01 +71 13 5.900000000000000e+01 +71 14 5.800000000000000e+01 +71 15 5.700000000000000e+01 +71 16 5.600000000000000e+01 +71 17 5.500000000000000e+01 +71 18 5.400000000000000e+01 +71 19 5.300000000000000e+01 +71 20 5.200000000000000e+01 +71 21 5.100000000000000e+01 +71 22 5.000000000000000e+01 +71 23 4.900000000000000e+01 +71 24 4.800000000000000e+01 +71 25 4.700000000000000e+01 +71 26 4.600000000000000e+01 +71 27 4.500000000000000e+01 +71 28 4.400000000000000e+01 +71 29 4.300000000000000e+01 +71 30 4.200000000000000e+01 +71 31 4.100000000000000e+01 +71 32 4.000000000000000e+01 +71 33 3.900000000000000e+01 +71 34 3.800000000000000e+01 +71 35 3.700000000000000e+01 +71 36 3.600000000000000e+01 +71 37 3.500000000000000e+01 +71 38 3.400000000000000e+01 +71 39 3.300000000000000e+01 +71 40 3.200000000000000e+01 +71 41 3.100000000000000e+01 +71 42 3.000000000000000e+01 +71 43 2.900000000000000e+01 +71 44 2.800000000000000e+01 +71 45 2.700000000000000e+01 +71 46 2.600000000000000e+01 +71 47 2.500000000000000e+01 +71 48 2.400000000000000e+01 +71 49 2.300000000000000e+01 +71 50 2.200000000000000e+01 +71 51 2.100000000000000e+01 +71 52 2.000000000000000e+01 +71 53 1.900000000000000e+01 +71 54 1.800000000000000e+01 +71 55 1.700000000000000e+01 +71 56 1.600000000000000e+01 +71 57 1.500000000000000e+01 +71 58 1.400000000000000e+01 +71 59 1.300000000000000e+01 +71 60 1.200000000000000e+01 +71 61 1.100000000000000e+01 +71 62 1.000000000000000e+01 +71 63 9.000000000000000e+00 +71 64 8.000000000000000e+00 +71 65 7.000000000000000e+00 +71 66 6.000000000000000e+00 +71 67 5.000000000000000e+00 +71 68 4.000000000000000e+00 +71 69 3.000000000000000e+00 +71 70 2.000000000000000e+00 +71 71 2.000000000000000e+00 +71 72 2.000000000000000e+00 +71 73 3.000000000000000e+00 +71 74 4.000000000000000e+00 +71 75 5.000000000000000e+00 +71 76 6.000000000000000e+00 +71 77 7.000000000000000e+00 +71 78 8.000000000000000e+00 +71 79 9.000000000000000e+00 +71 80 1.000000000000000e+01 +71 81 1.100000000000000e+01 +71 82 1.200000000000000e+01 +71 83 1.300000000000000e+01 +71 84 1.400000000000000e+01 +71 85 1.500000000000000e+01 +71 86 1.600000000000000e+01 +71 87 1.700000000000000e+01 +71 88 1.800000000000000e+01 +71 89 1.900000000000000e+01 +71 90 2.000000000000000e+01 +71 91 2.100000000000000e+01 +71 92 2.200000000000000e+01 +71 93 2.300000000000000e+01 +71 94 2.400000000000000e+01 +71 95 2.500000000000000e+01 +71 96 2.600000000000000e+01 +71 97 2.700000000000000e+01 +71 98 2.800000000000000e+01 +71 99 2.900000000000000e+01 +71 100 3.000000000000000e+01 +72 1 7.200000000000000e+01 +72 2 7.100000000000000e+01 +72 3 7.000000000000000e+01 +72 4 6.900000000000000e+01 +72 5 6.800000000000000e+01 +72 6 6.700000000000000e+01 +72 7 6.600000000000000e+01 +72 8 6.500000000000000e+01 +72 9 6.400000000000000e+01 +72 10 6.300000000000000e+01 +72 11 6.200000000000000e+01 +72 12 6.100000000000000e+01 +72 13 6.000000000000000e+01 +72 14 5.900000000000000e+01 +72 15 5.800000000000000e+01 +72 16 5.700000000000000e+01 +72 17 5.600000000000000e+01 +72 18 5.500000000000000e+01 +72 19 5.400000000000000e+01 +72 20 5.300000000000000e+01 +72 21 5.200000000000000e+01 +72 22 5.100000000000000e+01 +72 23 5.000000000000000e+01 +72 24 4.900000000000000e+01 +72 25 4.800000000000000e+01 +72 26 4.700000000000000e+01 +72 27 4.600000000000000e+01 +72 28 4.500000000000000e+01 +72 29 4.400000000000000e+01 +72 30 4.300000000000000e+01 +72 31 4.200000000000000e+01 +72 32 4.100000000000000e+01 +72 33 4.000000000000000e+01 +72 34 3.900000000000000e+01 +72 35 3.800000000000000e+01 +72 36 3.700000000000000e+01 +72 37 3.600000000000000e+01 +72 38 3.500000000000000e+01 +72 39 3.400000000000000e+01 +72 40 3.300000000000000e+01 +72 41 3.200000000000000e+01 +72 42 3.100000000000000e+01 +72 43 3.000000000000000e+01 +72 44 2.900000000000000e+01 +72 45 2.800000000000000e+01 +72 46 2.700000000000000e+01 +72 47 2.600000000000000e+01 +72 48 2.500000000000000e+01 +72 49 2.400000000000000e+01 +72 50 2.300000000000000e+01 +72 51 2.200000000000000e+01 +72 52 2.100000000000000e+01 +72 53 2.000000000000000e+01 +72 54 1.900000000000000e+01 +72 55 1.800000000000000e+01 +72 56 1.700000000000000e+01 +72 57 1.600000000000000e+01 +72 58 1.500000000000000e+01 +72 59 1.400000000000000e+01 +72 60 1.300000000000000e+01 +72 61 1.200000000000000e+01 +72 62 1.100000000000000e+01 +72 63 1.000000000000000e+01 +72 64 9.000000000000000e+00 +72 65 8.000000000000000e+00 +72 66 7.000000000000000e+00 +72 67 6.000000000000000e+00 +72 68 5.000000000000000e+00 +72 69 4.000000000000000e+00 +72 70 3.000000000000000e+00 +72 71 2.000000000000000e+00 +72 72 2.000000000000000e+00 +72 73 2.000000000000000e+00 +72 74 3.000000000000000e+00 +72 75 4.000000000000000e+00 +72 76 5.000000000000000e+00 +72 77 6.000000000000000e+00 +72 78 7.000000000000000e+00 +72 79 8.000000000000000e+00 +72 80 9.000000000000000e+00 +72 81 1.000000000000000e+01 +72 82 1.100000000000000e+01 +72 83 1.200000000000000e+01 +72 84 1.300000000000000e+01 +72 85 1.400000000000000e+01 +72 86 1.500000000000000e+01 +72 87 1.600000000000000e+01 +72 88 1.700000000000000e+01 +72 89 1.800000000000000e+01 +72 90 1.900000000000000e+01 +72 91 2.000000000000000e+01 +72 92 2.100000000000000e+01 +72 93 2.200000000000000e+01 +72 94 2.300000000000000e+01 +72 95 2.400000000000000e+01 +72 96 2.500000000000000e+01 +72 97 2.600000000000000e+01 +72 98 2.700000000000000e+01 +72 99 2.800000000000000e+01 +72 100 2.900000000000000e+01 +73 1 7.300000000000000e+01 +73 2 7.200000000000000e+01 +73 3 7.100000000000000e+01 +73 4 7.000000000000000e+01 +73 5 6.900000000000000e+01 +73 6 6.800000000000000e+01 +73 7 6.700000000000000e+01 +73 8 6.600000000000000e+01 +73 9 6.500000000000000e+01 +73 10 6.400000000000000e+01 +73 11 6.300000000000000e+01 +73 12 6.200000000000000e+01 +73 13 6.100000000000000e+01 +73 14 6.000000000000000e+01 +73 15 5.900000000000000e+01 +73 16 5.800000000000000e+01 +73 17 5.700000000000000e+01 +73 18 5.600000000000000e+01 +73 19 5.500000000000000e+01 +73 20 5.400000000000000e+01 +73 21 5.300000000000000e+01 +73 22 5.200000000000000e+01 +73 23 5.100000000000000e+01 +73 24 5.000000000000000e+01 +73 25 4.900000000000000e+01 +73 26 4.800000000000000e+01 +73 27 4.700000000000000e+01 +73 28 4.600000000000000e+01 +73 29 4.500000000000000e+01 +73 30 4.400000000000000e+01 +73 31 4.300000000000000e+01 +73 32 4.200000000000000e+01 +73 33 4.100000000000000e+01 +73 34 4.000000000000000e+01 +73 35 3.900000000000000e+01 +73 36 3.800000000000000e+01 +73 37 3.700000000000000e+01 +73 38 3.600000000000000e+01 +73 39 3.500000000000000e+01 +73 40 3.400000000000000e+01 +73 41 3.300000000000000e+01 +73 42 3.200000000000000e+01 +73 43 3.100000000000000e+01 +73 44 3.000000000000000e+01 +73 45 2.900000000000000e+01 +73 46 2.800000000000000e+01 +73 47 2.700000000000000e+01 +73 48 2.600000000000000e+01 +73 49 2.500000000000000e+01 +73 50 2.400000000000000e+01 +73 51 2.300000000000000e+01 +73 52 2.200000000000000e+01 +73 53 2.100000000000000e+01 +73 54 2.000000000000000e+01 +73 55 1.900000000000000e+01 +73 56 1.800000000000000e+01 +73 57 1.700000000000000e+01 +73 58 1.600000000000000e+01 +73 59 1.500000000000000e+01 +73 60 1.400000000000000e+01 +73 61 1.300000000000000e+01 +73 62 1.200000000000000e+01 +73 63 1.100000000000000e+01 +73 64 1.000000000000000e+01 +73 65 9.000000000000000e+00 +73 66 8.000000000000000e+00 +73 67 7.000000000000000e+00 +73 68 6.000000000000000e+00 +73 69 5.000000000000000e+00 +73 70 4.000000000000000e+00 +73 71 3.000000000000000e+00 +73 72 2.000000000000000e+00 +73 73 2.000000000000000e+00 +73 74 2.000000000000000e+00 +73 75 3.000000000000000e+00 +73 76 4.000000000000000e+00 +73 77 5.000000000000000e+00 +73 78 6.000000000000000e+00 +73 79 7.000000000000000e+00 +73 80 8.000000000000000e+00 +73 81 9.000000000000000e+00 +73 82 1.000000000000000e+01 +73 83 1.100000000000000e+01 +73 84 1.200000000000000e+01 +73 85 1.300000000000000e+01 +73 86 1.400000000000000e+01 +73 87 1.500000000000000e+01 +73 88 1.600000000000000e+01 +73 89 1.700000000000000e+01 +73 90 1.800000000000000e+01 +73 91 1.900000000000000e+01 +73 92 2.000000000000000e+01 +73 93 2.100000000000000e+01 +73 94 2.200000000000000e+01 +73 95 2.300000000000000e+01 +73 96 2.400000000000000e+01 +73 97 2.500000000000000e+01 +73 98 2.600000000000000e+01 +73 99 2.700000000000000e+01 +73 100 2.800000000000000e+01 +74 1 7.400000000000000e+01 +74 2 7.300000000000000e+01 +74 3 7.200000000000000e+01 +74 4 7.100000000000000e+01 +74 5 7.000000000000000e+01 +74 6 6.900000000000000e+01 +74 7 6.800000000000000e+01 +74 8 6.700000000000000e+01 +74 9 6.600000000000000e+01 +74 10 6.500000000000000e+01 +74 11 6.400000000000000e+01 +74 12 6.300000000000000e+01 +74 13 6.200000000000000e+01 +74 14 6.100000000000000e+01 +74 15 6.000000000000000e+01 +74 16 5.900000000000000e+01 +74 17 5.800000000000000e+01 +74 18 5.700000000000000e+01 +74 19 5.600000000000000e+01 +74 20 5.500000000000000e+01 +74 21 5.400000000000000e+01 +74 22 5.300000000000000e+01 +74 23 5.200000000000000e+01 +74 24 5.100000000000000e+01 +74 25 5.000000000000000e+01 +74 26 4.900000000000000e+01 +74 27 4.800000000000000e+01 +74 28 4.700000000000000e+01 +74 29 4.600000000000000e+01 +74 30 4.500000000000000e+01 +74 31 4.400000000000000e+01 +74 32 4.300000000000000e+01 +74 33 4.200000000000000e+01 +74 34 4.100000000000000e+01 +74 35 4.000000000000000e+01 +74 36 3.900000000000000e+01 +74 37 3.800000000000000e+01 +74 38 3.700000000000000e+01 +74 39 3.600000000000000e+01 +74 40 3.500000000000000e+01 +74 41 3.400000000000000e+01 +74 42 3.300000000000000e+01 +74 43 3.200000000000000e+01 +74 44 3.100000000000000e+01 +74 45 3.000000000000000e+01 +74 46 2.900000000000000e+01 +74 47 2.800000000000000e+01 +74 48 2.700000000000000e+01 +74 49 2.600000000000000e+01 +74 50 2.500000000000000e+01 +74 51 2.400000000000000e+01 +74 52 2.300000000000000e+01 +74 53 2.200000000000000e+01 +74 54 2.100000000000000e+01 +74 55 2.000000000000000e+01 +74 56 1.900000000000000e+01 +74 57 1.800000000000000e+01 +74 58 1.700000000000000e+01 +74 59 1.600000000000000e+01 +74 60 1.500000000000000e+01 +74 61 1.400000000000000e+01 +74 62 1.300000000000000e+01 +74 63 1.200000000000000e+01 +74 64 1.100000000000000e+01 +74 65 1.000000000000000e+01 +74 66 9.000000000000000e+00 +74 67 8.000000000000000e+00 +74 68 7.000000000000000e+00 +74 69 6.000000000000000e+00 +74 70 5.000000000000000e+00 +74 71 4.000000000000000e+00 +74 72 3.000000000000000e+00 +74 73 2.000000000000000e+00 +74 74 2.000000000000000e+00 +74 75 2.000000000000000e+00 +74 76 3.000000000000000e+00 +74 77 4.000000000000000e+00 +74 78 5.000000000000000e+00 +74 79 6.000000000000000e+00 +74 80 7.000000000000000e+00 +74 81 8.000000000000000e+00 +74 82 9.000000000000000e+00 +74 83 1.000000000000000e+01 +74 84 1.100000000000000e+01 +74 85 1.200000000000000e+01 +74 86 1.300000000000000e+01 +74 87 1.400000000000000e+01 +74 88 1.500000000000000e+01 +74 89 1.600000000000000e+01 +74 90 1.700000000000000e+01 +74 91 1.800000000000000e+01 +74 92 1.900000000000000e+01 +74 93 2.000000000000000e+01 +74 94 2.100000000000000e+01 +74 95 2.200000000000000e+01 +74 96 2.300000000000000e+01 +74 97 2.400000000000000e+01 +74 98 2.500000000000000e+01 +74 99 2.600000000000000e+01 +74 100 2.700000000000000e+01 +75 1 7.500000000000000e+01 +75 2 7.400000000000000e+01 +75 3 7.300000000000000e+01 +75 4 7.200000000000000e+01 +75 5 7.100000000000000e+01 +75 6 7.000000000000000e+01 +75 7 6.900000000000000e+01 +75 8 6.800000000000000e+01 +75 9 6.700000000000000e+01 +75 10 6.600000000000000e+01 +75 11 6.500000000000000e+01 +75 12 6.400000000000000e+01 +75 13 6.300000000000000e+01 +75 14 6.200000000000000e+01 +75 15 6.100000000000000e+01 +75 16 6.000000000000000e+01 +75 17 5.900000000000000e+01 +75 18 5.800000000000000e+01 +75 19 5.700000000000000e+01 +75 20 5.600000000000000e+01 +75 21 5.500000000000000e+01 +75 22 5.400000000000000e+01 +75 23 5.300000000000000e+01 +75 24 5.200000000000000e+01 +75 25 5.100000000000000e+01 +75 26 5.000000000000000e+01 +75 27 4.900000000000000e+01 +75 28 4.800000000000000e+01 +75 29 4.700000000000000e+01 +75 30 4.600000000000000e+01 +75 31 4.500000000000000e+01 +75 32 4.400000000000000e+01 +75 33 4.300000000000000e+01 +75 34 4.200000000000000e+01 +75 35 4.100000000000000e+01 +75 36 4.000000000000000e+01 +75 37 3.900000000000000e+01 +75 38 3.800000000000000e+01 +75 39 3.700000000000000e+01 +75 40 3.600000000000000e+01 +75 41 3.500000000000000e+01 +75 42 3.400000000000000e+01 +75 43 3.300000000000000e+01 +75 44 3.200000000000000e+01 +75 45 3.100000000000000e+01 +75 46 3.000000000000000e+01 +75 47 2.900000000000000e+01 +75 48 2.800000000000000e+01 +75 49 2.700000000000000e+01 +75 50 2.600000000000000e+01 +75 51 2.500000000000000e+01 +75 52 2.400000000000000e+01 +75 53 2.300000000000000e+01 +75 54 2.200000000000000e+01 +75 55 2.100000000000000e+01 +75 56 2.000000000000000e+01 +75 57 1.900000000000000e+01 +75 58 1.800000000000000e+01 +75 59 1.700000000000000e+01 +75 60 1.600000000000000e+01 +75 61 1.500000000000000e+01 +75 62 1.400000000000000e+01 +75 63 1.300000000000000e+01 +75 64 1.200000000000000e+01 +75 65 1.100000000000000e+01 +75 66 1.000000000000000e+01 +75 67 9.000000000000000e+00 +75 68 8.000000000000000e+00 +75 69 7.000000000000000e+00 +75 70 6.000000000000000e+00 +75 71 5.000000000000000e+00 +75 72 4.000000000000000e+00 +75 73 3.000000000000000e+00 +75 74 2.000000000000000e+00 +75 75 2.000000000000000e+00 +75 76 2.000000000000000e+00 +75 77 3.000000000000000e+00 +75 78 4.000000000000000e+00 +75 79 5.000000000000000e+00 +75 80 6.000000000000000e+00 +75 81 7.000000000000000e+00 +75 82 8.000000000000000e+00 +75 83 9.000000000000000e+00 +75 84 1.000000000000000e+01 +75 85 1.100000000000000e+01 +75 86 1.200000000000000e+01 +75 87 1.300000000000000e+01 +75 88 1.400000000000000e+01 +75 89 1.500000000000000e+01 +75 90 1.600000000000000e+01 +75 91 1.700000000000000e+01 +75 92 1.800000000000000e+01 +75 93 1.900000000000000e+01 +75 94 2.000000000000000e+01 +75 95 2.100000000000000e+01 +75 96 2.200000000000000e+01 +75 97 2.300000000000000e+01 +75 98 2.400000000000000e+01 +75 99 2.500000000000000e+01 +75 100 2.600000000000000e+01 +76 1 7.600000000000000e+01 +76 2 7.500000000000000e+01 +76 3 7.400000000000000e+01 +76 4 7.300000000000000e+01 +76 5 7.200000000000000e+01 +76 6 7.100000000000000e+01 +76 7 7.000000000000000e+01 +76 8 6.900000000000000e+01 +76 9 6.800000000000000e+01 +76 10 6.700000000000000e+01 +76 11 6.600000000000000e+01 +76 12 6.500000000000000e+01 +76 13 6.400000000000000e+01 +76 14 6.300000000000000e+01 +76 15 6.200000000000000e+01 +76 16 6.100000000000000e+01 +76 17 6.000000000000000e+01 +76 18 5.900000000000000e+01 +76 19 5.800000000000000e+01 +76 20 5.700000000000000e+01 +76 21 5.600000000000000e+01 +76 22 5.500000000000000e+01 +76 23 5.400000000000000e+01 +76 24 5.300000000000000e+01 +76 25 5.200000000000000e+01 +76 26 5.100000000000000e+01 +76 27 5.000000000000000e+01 +76 28 4.900000000000000e+01 +76 29 4.800000000000000e+01 +76 30 4.700000000000000e+01 +76 31 4.600000000000000e+01 +76 32 4.500000000000000e+01 +76 33 4.400000000000000e+01 +76 34 4.300000000000000e+01 +76 35 4.200000000000000e+01 +76 36 4.100000000000000e+01 +76 37 4.000000000000000e+01 +76 38 3.900000000000000e+01 +76 39 3.800000000000000e+01 +76 40 3.700000000000000e+01 +76 41 3.600000000000000e+01 +76 42 3.500000000000000e+01 +76 43 3.400000000000000e+01 +76 44 3.300000000000000e+01 +76 45 3.200000000000000e+01 +76 46 3.100000000000000e+01 +76 47 3.000000000000000e+01 +76 48 2.900000000000000e+01 +76 49 2.800000000000000e+01 +76 50 2.700000000000000e+01 +76 51 2.600000000000000e+01 +76 52 2.500000000000000e+01 +76 53 2.400000000000000e+01 +76 54 2.300000000000000e+01 +76 55 2.200000000000000e+01 +76 56 2.100000000000000e+01 +76 57 2.000000000000000e+01 +76 58 1.900000000000000e+01 +76 59 1.800000000000000e+01 +76 60 1.700000000000000e+01 +76 61 1.600000000000000e+01 +76 62 1.500000000000000e+01 +76 63 1.400000000000000e+01 +76 64 1.300000000000000e+01 +76 65 1.200000000000000e+01 +76 66 1.100000000000000e+01 +76 67 1.000000000000000e+01 +76 68 9.000000000000000e+00 +76 69 8.000000000000000e+00 +76 70 7.000000000000000e+00 +76 71 6.000000000000000e+00 +76 72 5.000000000000000e+00 +76 73 4.000000000000000e+00 +76 74 3.000000000000000e+00 +76 75 2.000000000000000e+00 +76 76 2.000000000000000e+00 +76 77 2.000000000000000e+00 +76 78 3.000000000000000e+00 +76 79 4.000000000000000e+00 +76 80 5.000000000000000e+00 +76 81 6.000000000000000e+00 +76 82 7.000000000000000e+00 +76 83 8.000000000000000e+00 +76 84 9.000000000000000e+00 +76 85 1.000000000000000e+01 +76 86 1.100000000000000e+01 +76 87 1.200000000000000e+01 +76 88 1.300000000000000e+01 +76 89 1.400000000000000e+01 +76 90 1.500000000000000e+01 +76 91 1.600000000000000e+01 +76 92 1.700000000000000e+01 +76 93 1.800000000000000e+01 +76 94 1.900000000000000e+01 +76 95 2.000000000000000e+01 +76 96 2.100000000000000e+01 +76 97 2.200000000000000e+01 +76 98 2.300000000000000e+01 +76 99 2.400000000000000e+01 +76 100 2.500000000000000e+01 +77 1 7.700000000000000e+01 +77 2 7.600000000000000e+01 +77 3 7.500000000000000e+01 +77 4 7.400000000000000e+01 +77 5 7.300000000000000e+01 +77 6 7.200000000000000e+01 +77 7 7.100000000000000e+01 +77 8 7.000000000000000e+01 +77 9 6.900000000000000e+01 +77 10 6.800000000000000e+01 +77 11 6.700000000000000e+01 +77 12 6.600000000000000e+01 +77 13 6.500000000000000e+01 +77 14 6.400000000000000e+01 +77 15 6.300000000000000e+01 +77 16 6.200000000000000e+01 +77 17 6.100000000000000e+01 +77 18 6.000000000000000e+01 +77 19 5.900000000000000e+01 +77 20 5.800000000000000e+01 +77 21 5.700000000000000e+01 +77 22 5.600000000000000e+01 +77 23 5.500000000000000e+01 +77 24 5.400000000000000e+01 +77 25 5.300000000000000e+01 +77 26 5.200000000000000e+01 +77 27 5.100000000000000e+01 +77 28 5.000000000000000e+01 +77 29 4.900000000000000e+01 +77 30 4.800000000000000e+01 +77 31 4.700000000000000e+01 +77 32 4.600000000000000e+01 +77 33 4.500000000000000e+01 +77 34 4.400000000000000e+01 +77 35 4.300000000000000e+01 +77 36 4.200000000000000e+01 +77 37 4.100000000000000e+01 +77 38 4.000000000000000e+01 +77 39 3.900000000000000e+01 +77 40 3.800000000000000e+01 +77 41 3.700000000000000e+01 +77 42 3.600000000000000e+01 +77 43 3.500000000000000e+01 +77 44 3.400000000000000e+01 +77 45 3.300000000000000e+01 +77 46 3.200000000000000e+01 +77 47 3.100000000000000e+01 +77 48 3.000000000000000e+01 +77 49 2.900000000000000e+01 +77 50 2.800000000000000e+01 +77 51 2.700000000000000e+01 +77 52 2.600000000000000e+01 +77 53 2.500000000000000e+01 +77 54 2.400000000000000e+01 +77 55 2.300000000000000e+01 +77 56 2.200000000000000e+01 +77 57 2.100000000000000e+01 +77 58 2.000000000000000e+01 +77 59 1.900000000000000e+01 +77 60 1.800000000000000e+01 +77 61 1.700000000000000e+01 +77 62 1.600000000000000e+01 +77 63 1.500000000000000e+01 +77 64 1.400000000000000e+01 +77 65 1.300000000000000e+01 +77 66 1.200000000000000e+01 +77 67 1.100000000000000e+01 +77 68 1.000000000000000e+01 +77 69 9.000000000000000e+00 +77 70 8.000000000000000e+00 +77 71 7.000000000000000e+00 +77 72 6.000000000000000e+00 +77 73 5.000000000000000e+00 +77 74 4.000000000000000e+00 +77 75 3.000000000000000e+00 +77 76 2.000000000000000e+00 +77 77 2.000000000000000e+00 +77 78 2.000000000000000e+00 +77 79 3.000000000000000e+00 +77 80 4.000000000000000e+00 +77 81 5.000000000000000e+00 +77 82 6.000000000000000e+00 +77 83 7.000000000000000e+00 +77 84 8.000000000000000e+00 +77 85 9.000000000000000e+00 +77 86 1.000000000000000e+01 +77 87 1.100000000000000e+01 +77 88 1.200000000000000e+01 +77 89 1.300000000000000e+01 +77 90 1.400000000000000e+01 +77 91 1.500000000000000e+01 +77 92 1.600000000000000e+01 +77 93 1.700000000000000e+01 +77 94 1.800000000000000e+01 +77 95 1.900000000000000e+01 +77 96 2.000000000000000e+01 +77 97 2.100000000000000e+01 +77 98 2.200000000000000e+01 +77 99 2.300000000000000e+01 +77 100 2.400000000000000e+01 +78 1 7.800000000000000e+01 +78 2 7.700000000000000e+01 +78 3 7.600000000000000e+01 +78 4 7.500000000000000e+01 +78 5 7.400000000000000e+01 +78 6 7.300000000000000e+01 +78 7 7.200000000000000e+01 +78 8 7.100000000000000e+01 +78 9 7.000000000000000e+01 +78 10 6.900000000000000e+01 +78 11 6.800000000000000e+01 +78 12 6.700000000000000e+01 +78 13 6.600000000000000e+01 +78 14 6.500000000000000e+01 +78 15 6.400000000000000e+01 +78 16 6.300000000000000e+01 +78 17 6.200000000000000e+01 +78 18 6.100000000000000e+01 +78 19 6.000000000000000e+01 +78 20 5.900000000000000e+01 +78 21 5.800000000000000e+01 +78 22 5.700000000000000e+01 +78 23 5.600000000000000e+01 +78 24 5.500000000000000e+01 +78 25 5.400000000000000e+01 +78 26 5.300000000000000e+01 +78 27 5.200000000000000e+01 +78 28 5.100000000000000e+01 +78 29 5.000000000000000e+01 +78 30 4.900000000000000e+01 +78 31 4.800000000000000e+01 +78 32 4.700000000000000e+01 +78 33 4.600000000000000e+01 +78 34 4.500000000000000e+01 +78 35 4.400000000000000e+01 +78 36 4.300000000000000e+01 +78 37 4.200000000000000e+01 +78 38 4.100000000000000e+01 +78 39 4.000000000000000e+01 +78 40 3.900000000000000e+01 +78 41 3.800000000000000e+01 +78 42 3.700000000000000e+01 +78 43 3.600000000000000e+01 +78 44 3.500000000000000e+01 +78 45 3.400000000000000e+01 +78 46 3.300000000000000e+01 +78 47 3.200000000000000e+01 +78 48 3.100000000000000e+01 +78 49 3.000000000000000e+01 +78 50 2.900000000000000e+01 +78 51 2.800000000000000e+01 +78 52 2.700000000000000e+01 +78 53 2.600000000000000e+01 +78 54 2.500000000000000e+01 +78 55 2.400000000000000e+01 +78 56 2.300000000000000e+01 +78 57 2.200000000000000e+01 +78 58 2.100000000000000e+01 +78 59 2.000000000000000e+01 +78 60 1.900000000000000e+01 +78 61 1.800000000000000e+01 +78 62 1.700000000000000e+01 +78 63 1.600000000000000e+01 +78 64 1.500000000000000e+01 +78 65 1.400000000000000e+01 +78 66 1.300000000000000e+01 +78 67 1.200000000000000e+01 +78 68 1.100000000000000e+01 +78 69 1.000000000000000e+01 +78 70 9.000000000000000e+00 +78 71 8.000000000000000e+00 +78 72 7.000000000000000e+00 +78 73 6.000000000000000e+00 +78 74 5.000000000000000e+00 +78 75 4.000000000000000e+00 +78 76 3.000000000000000e+00 +78 77 2.000000000000000e+00 +78 78 2.000000000000000e+00 +78 79 2.000000000000000e+00 +78 80 3.000000000000000e+00 +78 81 4.000000000000000e+00 +78 82 5.000000000000000e+00 +78 83 6.000000000000000e+00 +78 84 7.000000000000000e+00 +78 85 8.000000000000000e+00 +78 86 9.000000000000000e+00 +78 87 1.000000000000000e+01 +78 88 1.100000000000000e+01 +78 89 1.200000000000000e+01 +78 90 1.300000000000000e+01 +78 91 1.400000000000000e+01 +78 92 1.500000000000000e+01 +78 93 1.600000000000000e+01 +78 94 1.700000000000000e+01 +78 95 1.800000000000000e+01 +78 96 1.900000000000000e+01 +78 97 2.000000000000000e+01 +78 98 2.100000000000000e+01 +78 99 2.200000000000000e+01 +78 100 2.300000000000000e+01 +79 1 7.900000000000000e+01 +79 2 7.800000000000000e+01 +79 3 7.700000000000000e+01 +79 4 7.600000000000000e+01 +79 5 7.500000000000000e+01 +79 6 7.400000000000000e+01 +79 7 7.300000000000000e+01 +79 8 7.200000000000000e+01 +79 9 7.100000000000000e+01 +79 10 7.000000000000000e+01 +79 11 6.900000000000000e+01 +79 12 6.800000000000000e+01 +79 13 6.700000000000000e+01 +79 14 6.600000000000000e+01 +79 15 6.500000000000000e+01 +79 16 6.400000000000000e+01 +79 17 6.300000000000000e+01 +79 18 6.200000000000000e+01 +79 19 6.100000000000000e+01 +79 20 6.000000000000000e+01 +79 21 5.900000000000000e+01 +79 22 5.800000000000000e+01 +79 23 5.700000000000000e+01 +79 24 5.600000000000000e+01 +79 25 5.500000000000000e+01 +79 26 5.400000000000000e+01 +79 27 5.300000000000000e+01 +79 28 5.200000000000000e+01 +79 29 5.100000000000000e+01 +79 30 5.000000000000000e+01 +79 31 4.900000000000000e+01 +79 32 4.800000000000000e+01 +79 33 4.700000000000000e+01 +79 34 4.600000000000000e+01 +79 35 4.500000000000000e+01 +79 36 4.400000000000000e+01 +79 37 4.300000000000000e+01 +79 38 4.200000000000000e+01 +79 39 4.100000000000000e+01 +79 40 4.000000000000000e+01 +79 41 3.900000000000000e+01 +79 42 3.800000000000000e+01 +79 43 3.700000000000000e+01 +79 44 3.600000000000000e+01 +79 45 3.500000000000000e+01 +79 46 3.400000000000000e+01 +79 47 3.300000000000000e+01 +79 48 3.200000000000000e+01 +79 49 3.100000000000000e+01 +79 50 3.000000000000000e+01 +79 51 2.900000000000000e+01 +79 52 2.800000000000000e+01 +79 53 2.700000000000000e+01 +79 54 2.600000000000000e+01 +79 55 2.500000000000000e+01 +79 56 2.400000000000000e+01 +79 57 2.300000000000000e+01 +79 58 2.200000000000000e+01 +79 59 2.100000000000000e+01 +79 60 2.000000000000000e+01 +79 61 1.900000000000000e+01 +79 62 1.800000000000000e+01 +79 63 1.700000000000000e+01 +79 64 1.600000000000000e+01 +79 65 1.500000000000000e+01 +79 66 1.400000000000000e+01 +79 67 1.300000000000000e+01 +79 68 1.200000000000000e+01 +79 69 1.100000000000000e+01 +79 70 1.000000000000000e+01 +79 71 9.000000000000000e+00 +79 72 8.000000000000000e+00 +79 73 7.000000000000000e+00 +79 74 6.000000000000000e+00 +79 75 5.000000000000000e+00 +79 76 4.000000000000000e+00 +79 77 3.000000000000000e+00 +79 78 2.000000000000000e+00 +79 79 2.000000000000000e+00 +79 80 2.000000000000000e+00 +79 81 3.000000000000000e+00 +79 82 4.000000000000000e+00 +79 83 5.000000000000000e+00 +79 84 6.000000000000000e+00 +79 85 7.000000000000000e+00 +79 86 8.000000000000000e+00 +79 87 9.000000000000000e+00 +79 88 1.000000000000000e+01 +79 89 1.100000000000000e+01 +79 90 1.200000000000000e+01 +79 91 1.300000000000000e+01 +79 92 1.400000000000000e+01 +79 93 1.500000000000000e+01 +79 94 1.600000000000000e+01 +79 95 1.700000000000000e+01 +79 96 1.800000000000000e+01 +79 97 1.900000000000000e+01 +79 98 2.000000000000000e+01 +79 99 2.100000000000000e+01 +79 100 2.200000000000000e+01 +80 1 8.000000000000000e+01 +80 2 7.900000000000000e+01 +80 3 7.800000000000000e+01 +80 4 7.700000000000000e+01 +80 5 7.600000000000000e+01 +80 6 7.500000000000000e+01 +80 7 7.400000000000000e+01 +80 8 7.300000000000000e+01 +80 9 7.200000000000000e+01 +80 10 7.100000000000000e+01 +80 11 7.000000000000000e+01 +80 12 6.900000000000000e+01 +80 13 6.800000000000000e+01 +80 14 6.700000000000000e+01 +80 15 6.600000000000000e+01 +80 16 6.500000000000000e+01 +80 17 6.400000000000000e+01 +80 18 6.300000000000000e+01 +80 19 6.200000000000000e+01 +80 20 6.100000000000000e+01 +80 21 6.000000000000000e+01 +80 22 5.900000000000000e+01 +80 23 5.800000000000000e+01 +80 24 5.700000000000000e+01 +80 25 5.600000000000000e+01 +80 26 5.500000000000000e+01 +80 27 5.400000000000000e+01 +80 28 5.300000000000000e+01 +80 29 5.200000000000000e+01 +80 30 5.100000000000000e+01 +80 31 5.000000000000000e+01 +80 32 4.900000000000000e+01 +80 33 4.800000000000000e+01 +80 34 4.700000000000000e+01 +80 35 4.600000000000000e+01 +80 36 4.500000000000000e+01 +80 37 4.400000000000000e+01 +80 38 4.300000000000000e+01 +80 39 4.200000000000000e+01 +80 40 4.100000000000000e+01 +80 41 4.000000000000000e+01 +80 42 3.900000000000000e+01 +80 43 3.800000000000000e+01 +80 44 3.700000000000000e+01 +80 45 3.600000000000000e+01 +80 46 3.500000000000000e+01 +80 47 3.400000000000000e+01 +80 48 3.300000000000000e+01 +80 49 3.200000000000000e+01 +80 50 3.100000000000000e+01 +80 51 3.000000000000000e+01 +80 52 2.900000000000000e+01 +80 53 2.800000000000000e+01 +80 54 2.700000000000000e+01 +80 55 2.600000000000000e+01 +80 56 2.500000000000000e+01 +80 57 2.400000000000000e+01 +80 58 2.300000000000000e+01 +80 59 2.200000000000000e+01 +80 60 2.100000000000000e+01 +80 61 2.000000000000000e+01 +80 62 1.900000000000000e+01 +80 63 1.800000000000000e+01 +80 64 1.700000000000000e+01 +80 65 1.600000000000000e+01 +80 66 1.500000000000000e+01 +80 67 1.400000000000000e+01 +80 68 1.300000000000000e+01 +80 69 1.200000000000000e+01 +80 70 1.100000000000000e+01 +80 71 1.000000000000000e+01 +80 72 9.000000000000000e+00 +80 73 8.000000000000000e+00 +80 74 7.000000000000000e+00 +80 75 6.000000000000000e+00 +80 76 5.000000000000000e+00 +80 77 4.000000000000000e+00 +80 78 3.000000000000000e+00 +80 79 2.000000000000000e+00 +80 80 2.000000000000000e+00 +80 81 2.000000000000000e+00 +80 82 3.000000000000000e+00 +80 83 4.000000000000000e+00 +80 84 5.000000000000000e+00 +80 85 6.000000000000000e+00 +80 86 7.000000000000000e+00 +80 87 8.000000000000000e+00 +80 88 9.000000000000000e+00 +80 89 1.000000000000000e+01 +80 90 1.100000000000000e+01 +80 91 1.200000000000000e+01 +80 92 1.300000000000000e+01 +80 93 1.400000000000000e+01 +80 94 1.500000000000000e+01 +80 95 1.600000000000000e+01 +80 96 1.700000000000000e+01 +80 97 1.800000000000000e+01 +80 98 1.900000000000000e+01 +80 99 2.000000000000000e+01 +80 100 2.100000000000000e+01 +81 1 8.100000000000000e+01 +81 2 8.000000000000000e+01 +81 3 7.900000000000000e+01 +81 4 7.800000000000000e+01 +81 5 7.700000000000000e+01 +81 6 7.600000000000000e+01 +81 7 7.500000000000000e+01 +81 8 7.400000000000000e+01 +81 9 7.300000000000000e+01 +81 10 7.200000000000000e+01 +81 11 7.100000000000000e+01 +81 12 7.000000000000000e+01 +81 13 6.900000000000000e+01 +81 14 6.800000000000000e+01 +81 15 6.700000000000000e+01 +81 16 6.600000000000000e+01 +81 17 6.500000000000000e+01 +81 18 6.400000000000000e+01 +81 19 6.300000000000000e+01 +81 20 6.200000000000000e+01 +81 21 6.100000000000000e+01 +81 22 6.000000000000000e+01 +81 23 5.900000000000000e+01 +81 24 5.800000000000000e+01 +81 25 5.700000000000000e+01 +81 26 5.600000000000000e+01 +81 27 5.500000000000000e+01 +81 28 5.400000000000000e+01 +81 29 5.300000000000000e+01 +81 30 5.200000000000000e+01 +81 31 5.100000000000000e+01 +81 32 5.000000000000000e+01 +81 33 4.900000000000000e+01 +81 34 4.800000000000000e+01 +81 35 4.700000000000000e+01 +81 36 4.600000000000000e+01 +81 37 4.500000000000000e+01 +81 38 4.400000000000000e+01 +81 39 4.300000000000000e+01 +81 40 4.200000000000000e+01 +81 41 4.100000000000000e+01 +81 42 4.000000000000000e+01 +81 43 3.900000000000000e+01 +81 44 3.800000000000000e+01 +81 45 3.700000000000000e+01 +81 46 3.600000000000000e+01 +81 47 3.500000000000000e+01 +81 48 3.400000000000000e+01 +81 49 3.300000000000000e+01 +81 50 3.200000000000000e+01 +81 51 3.100000000000000e+01 +81 52 3.000000000000000e+01 +81 53 2.900000000000000e+01 +81 54 2.800000000000000e+01 +81 55 2.700000000000000e+01 +81 56 2.600000000000000e+01 +81 57 2.500000000000000e+01 +81 58 2.400000000000000e+01 +81 59 2.300000000000000e+01 +81 60 2.200000000000000e+01 +81 61 2.100000000000000e+01 +81 62 2.000000000000000e+01 +81 63 1.900000000000000e+01 +81 64 1.800000000000000e+01 +81 65 1.700000000000000e+01 +81 66 1.600000000000000e+01 +81 67 1.500000000000000e+01 +81 68 1.400000000000000e+01 +81 69 1.300000000000000e+01 +81 70 1.200000000000000e+01 +81 71 1.100000000000000e+01 +81 72 1.000000000000000e+01 +81 73 9.000000000000000e+00 +81 74 8.000000000000000e+00 +81 75 7.000000000000000e+00 +81 76 6.000000000000000e+00 +81 77 5.000000000000000e+00 +81 78 4.000000000000000e+00 +81 79 3.000000000000000e+00 +81 80 2.000000000000000e+00 +81 81 2.000000000000000e+00 +81 82 2.000000000000000e+00 +81 83 3.000000000000000e+00 +81 84 4.000000000000000e+00 +81 85 5.000000000000000e+00 +81 86 6.000000000000000e+00 +81 87 7.000000000000000e+00 +81 88 8.000000000000000e+00 +81 89 9.000000000000000e+00 +81 90 1.000000000000000e+01 +81 91 1.100000000000000e+01 +81 92 1.200000000000000e+01 +81 93 1.300000000000000e+01 +81 94 1.400000000000000e+01 +81 95 1.500000000000000e+01 +81 96 1.600000000000000e+01 +81 97 1.700000000000000e+01 +81 98 1.800000000000000e+01 +81 99 1.900000000000000e+01 +81 100 2.000000000000000e+01 +82 1 8.200000000000000e+01 +82 2 8.100000000000000e+01 +82 3 8.000000000000000e+01 +82 4 7.900000000000000e+01 +82 5 7.800000000000000e+01 +82 6 7.700000000000000e+01 +82 7 7.600000000000000e+01 +82 8 7.500000000000000e+01 +82 9 7.400000000000000e+01 +82 10 7.300000000000000e+01 +82 11 7.200000000000000e+01 +82 12 7.100000000000000e+01 +82 13 7.000000000000000e+01 +82 14 6.900000000000000e+01 +82 15 6.800000000000000e+01 +82 16 6.700000000000000e+01 +82 17 6.600000000000000e+01 +82 18 6.500000000000000e+01 +82 19 6.400000000000000e+01 +82 20 6.300000000000000e+01 +82 21 6.200000000000000e+01 +82 22 6.100000000000000e+01 +82 23 6.000000000000000e+01 +82 24 5.900000000000000e+01 +82 25 5.800000000000000e+01 +82 26 5.700000000000000e+01 +82 27 5.600000000000000e+01 +82 28 5.500000000000000e+01 +82 29 5.400000000000000e+01 +82 30 5.300000000000000e+01 +82 31 5.200000000000000e+01 +82 32 5.100000000000000e+01 +82 33 5.000000000000000e+01 +82 34 4.900000000000000e+01 +82 35 4.800000000000000e+01 +82 36 4.700000000000000e+01 +82 37 4.600000000000000e+01 +82 38 4.500000000000000e+01 +82 39 4.400000000000000e+01 +82 40 4.300000000000000e+01 +82 41 4.200000000000000e+01 +82 42 4.100000000000000e+01 +82 43 4.000000000000000e+01 +82 44 3.900000000000000e+01 +82 45 3.800000000000000e+01 +82 46 3.700000000000000e+01 +82 47 3.600000000000000e+01 +82 48 3.500000000000000e+01 +82 49 3.400000000000000e+01 +82 50 3.300000000000000e+01 +82 51 3.200000000000000e+01 +82 52 3.100000000000000e+01 +82 53 3.000000000000000e+01 +82 54 2.900000000000000e+01 +82 55 2.800000000000000e+01 +82 56 2.700000000000000e+01 +82 57 2.600000000000000e+01 +82 58 2.500000000000000e+01 +82 59 2.400000000000000e+01 +82 60 2.300000000000000e+01 +82 61 2.200000000000000e+01 +82 62 2.100000000000000e+01 +82 63 2.000000000000000e+01 +82 64 1.900000000000000e+01 +82 65 1.800000000000000e+01 +82 66 1.700000000000000e+01 +82 67 1.600000000000000e+01 +82 68 1.500000000000000e+01 +82 69 1.400000000000000e+01 +82 70 1.300000000000000e+01 +82 71 1.200000000000000e+01 +82 72 1.100000000000000e+01 +82 73 1.000000000000000e+01 +82 74 9.000000000000000e+00 +82 75 8.000000000000000e+00 +82 76 7.000000000000000e+00 +82 77 6.000000000000000e+00 +82 78 5.000000000000000e+00 +82 79 4.000000000000000e+00 +82 80 3.000000000000000e+00 +82 81 2.000000000000000e+00 +82 82 2.000000000000000e+00 +82 83 2.000000000000000e+00 +82 84 3.000000000000000e+00 +82 85 4.000000000000000e+00 +82 86 5.000000000000000e+00 +82 87 6.000000000000000e+00 +82 88 7.000000000000000e+00 +82 89 8.000000000000000e+00 +82 90 9.000000000000000e+00 +82 91 1.000000000000000e+01 +82 92 1.100000000000000e+01 +82 93 1.200000000000000e+01 +82 94 1.300000000000000e+01 +82 95 1.400000000000000e+01 +82 96 1.500000000000000e+01 +82 97 1.600000000000000e+01 +82 98 1.700000000000000e+01 +82 99 1.800000000000000e+01 +82 100 1.900000000000000e+01 +83 1 8.300000000000000e+01 +83 2 8.200000000000000e+01 +83 3 8.100000000000000e+01 +83 4 8.000000000000000e+01 +83 5 7.900000000000000e+01 +83 6 7.800000000000000e+01 +83 7 7.700000000000000e+01 +83 8 7.600000000000000e+01 +83 9 7.500000000000000e+01 +83 10 7.400000000000000e+01 +83 11 7.300000000000000e+01 +83 12 7.200000000000000e+01 +83 13 7.100000000000000e+01 +83 14 7.000000000000000e+01 +83 15 6.900000000000000e+01 +83 16 6.800000000000000e+01 +83 17 6.700000000000000e+01 +83 18 6.600000000000000e+01 +83 19 6.500000000000000e+01 +83 20 6.400000000000000e+01 +83 21 6.300000000000000e+01 +83 22 6.200000000000000e+01 +83 23 6.100000000000000e+01 +83 24 6.000000000000000e+01 +83 25 5.900000000000000e+01 +83 26 5.800000000000000e+01 +83 27 5.700000000000000e+01 +83 28 5.600000000000000e+01 +83 29 5.500000000000000e+01 +83 30 5.400000000000000e+01 +83 31 5.300000000000000e+01 +83 32 5.200000000000000e+01 +83 33 5.100000000000000e+01 +83 34 5.000000000000000e+01 +83 35 4.900000000000000e+01 +83 36 4.800000000000000e+01 +83 37 4.700000000000000e+01 +83 38 4.600000000000000e+01 +83 39 4.500000000000000e+01 +83 40 4.400000000000000e+01 +83 41 4.300000000000000e+01 +83 42 4.200000000000000e+01 +83 43 4.100000000000000e+01 +83 44 4.000000000000000e+01 +83 45 3.900000000000000e+01 +83 46 3.800000000000000e+01 +83 47 3.700000000000000e+01 +83 48 3.600000000000000e+01 +83 49 3.500000000000000e+01 +83 50 3.400000000000000e+01 +83 51 3.300000000000000e+01 +83 52 3.200000000000000e+01 +83 53 3.100000000000000e+01 +83 54 3.000000000000000e+01 +83 55 2.900000000000000e+01 +83 56 2.800000000000000e+01 +83 57 2.700000000000000e+01 +83 58 2.600000000000000e+01 +83 59 2.500000000000000e+01 +83 60 2.400000000000000e+01 +83 61 2.300000000000000e+01 +83 62 2.200000000000000e+01 +83 63 2.100000000000000e+01 +83 64 2.000000000000000e+01 +83 65 1.900000000000000e+01 +83 66 1.800000000000000e+01 +83 67 1.700000000000000e+01 +83 68 1.600000000000000e+01 +83 69 1.500000000000000e+01 +83 70 1.400000000000000e+01 +83 71 1.300000000000000e+01 +83 72 1.200000000000000e+01 +83 73 1.100000000000000e+01 +83 74 1.000000000000000e+01 +83 75 9.000000000000000e+00 +83 76 8.000000000000000e+00 +83 77 7.000000000000000e+00 +83 78 6.000000000000000e+00 +83 79 5.000000000000000e+00 +83 80 4.000000000000000e+00 +83 81 3.000000000000000e+00 +83 82 2.000000000000000e+00 +83 83 2.000000000000000e+00 +83 84 2.000000000000000e+00 +83 85 3.000000000000000e+00 +83 86 4.000000000000000e+00 +83 87 5.000000000000000e+00 +83 88 6.000000000000000e+00 +83 89 7.000000000000000e+00 +83 90 8.000000000000000e+00 +83 91 9.000000000000000e+00 +83 92 1.000000000000000e+01 +83 93 1.100000000000000e+01 +83 94 1.200000000000000e+01 +83 95 1.300000000000000e+01 +83 96 1.400000000000000e+01 +83 97 1.500000000000000e+01 +83 98 1.600000000000000e+01 +83 99 1.700000000000000e+01 +83 100 1.800000000000000e+01 +84 1 8.400000000000000e+01 +84 2 8.300000000000000e+01 +84 3 8.200000000000000e+01 +84 4 8.100000000000000e+01 +84 5 8.000000000000000e+01 +84 6 7.900000000000000e+01 +84 7 7.800000000000000e+01 +84 8 7.700000000000000e+01 +84 9 7.600000000000000e+01 +84 10 7.500000000000000e+01 +84 11 7.400000000000000e+01 +84 12 7.300000000000000e+01 +84 13 7.200000000000000e+01 +84 14 7.100000000000000e+01 +84 15 7.000000000000000e+01 +84 16 6.900000000000000e+01 +84 17 6.800000000000000e+01 +84 18 6.700000000000000e+01 +84 19 6.600000000000000e+01 +84 20 6.500000000000000e+01 +84 21 6.400000000000000e+01 +84 22 6.300000000000000e+01 +84 23 6.200000000000000e+01 +84 24 6.100000000000000e+01 +84 25 6.000000000000000e+01 +84 26 5.900000000000000e+01 +84 27 5.800000000000000e+01 +84 28 5.700000000000000e+01 +84 29 5.600000000000000e+01 +84 30 5.500000000000000e+01 +84 31 5.400000000000000e+01 +84 32 5.300000000000000e+01 +84 33 5.200000000000000e+01 +84 34 5.100000000000000e+01 +84 35 5.000000000000000e+01 +84 36 4.900000000000000e+01 +84 37 4.800000000000000e+01 +84 38 4.700000000000000e+01 +84 39 4.600000000000000e+01 +84 40 4.500000000000000e+01 +84 41 4.400000000000000e+01 +84 42 4.300000000000000e+01 +84 43 4.200000000000000e+01 +84 44 4.100000000000000e+01 +84 45 4.000000000000000e+01 +84 46 3.900000000000000e+01 +84 47 3.800000000000000e+01 +84 48 3.700000000000000e+01 +84 49 3.600000000000000e+01 +84 50 3.500000000000000e+01 +84 51 3.400000000000000e+01 +84 52 3.300000000000000e+01 +84 53 3.200000000000000e+01 +84 54 3.100000000000000e+01 +84 55 3.000000000000000e+01 +84 56 2.900000000000000e+01 +84 57 2.800000000000000e+01 +84 58 2.700000000000000e+01 +84 59 2.600000000000000e+01 +84 60 2.500000000000000e+01 +84 61 2.400000000000000e+01 +84 62 2.300000000000000e+01 +84 63 2.200000000000000e+01 +84 64 2.100000000000000e+01 +84 65 2.000000000000000e+01 +84 66 1.900000000000000e+01 +84 67 1.800000000000000e+01 +84 68 1.700000000000000e+01 +84 69 1.600000000000000e+01 +84 70 1.500000000000000e+01 +84 71 1.400000000000000e+01 +84 72 1.300000000000000e+01 +84 73 1.200000000000000e+01 +84 74 1.100000000000000e+01 +84 75 1.000000000000000e+01 +84 76 9.000000000000000e+00 +84 77 8.000000000000000e+00 +84 78 7.000000000000000e+00 +84 79 6.000000000000000e+00 +84 80 5.000000000000000e+00 +84 81 4.000000000000000e+00 +84 82 3.000000000000000e+00 +84 83 2.000000000000000e+00 +84 84 2.000000000000000e+00 +84 85 2.000000000000000e+00 +84 86 3.000000000000000e+00 +84 87 4.000000000000000e+00 +84 88 5.000000000000000e+00 +84 89 6.000000000000000e+00 +84 90 7.000000000000000e+00 +84 91 8.000000000000000e+00 +84 92 9.000000000000000e+00 +84 93 1.000000000000000e+01 +84 94 1.100000000000000e+01 +84 95 1.200000000000000e+01 +84 96 1.300000000000000e+01 +84 97 1.400000000000000e+01 +84 98 1.500000000000000e+01 +84 99 1.600000000000000e+01 +84 100 1.700000000000000e+01 +85 1 8.500000000000000e+01 +85 2 8.400000000000000e+01 +85 3 8.300000000000000e+01 +85 4 8.200000000000000e+01 +85 5 8.100000000000000e+01 +85 6 8.000000000000000e+01 +85 7 7.900000000000000e+01 +85 8 7.800000000000000e+01 +85 9 7.700000000000000e+01 +85 10 7.600000000000000e+01 +85 11 7.500000000000000e+01 +85 12 7.400000000000000e+01 +85 13 7.300000000000000e+01 +85 14 7.200000000000000e+01 +85 15 7.100000000000000e+01 +85 16 7.000000000000000e+01 +85 17 6.900000000000000e+01 +85 18 6.800000000000000e+01 +85 19 6.700000000000000e+01 +85 20 6.600000000000000e+01 +85 21 6.500000000000000e+01 +85 22 6.400000000000000e+01 +85 23 6.300000000000000e+01 +85 24 6.200000000000000e+01 +85 25 6.100000000000000e+01 +85 26 6.000000000000000e+01 +85 27 5.900000000000000e+01 +85 28 5.800000000000000e+01 +85 29 5.700000000000000e+01 +85 30 5.600000000000000e+01 +85 31 5.500000000000000e+01 +85 32 5.400000000000000e+01 +85 33 5.300000000000000e+01 +85 34 5.200000000000000e+01 +85 35 5.100000000000000e+01 +85 36 5.000000000000000e+01 +85 37 4.900000000000000e+01 +85 38 4.800000000000000e+01 +85 39 4.700000000000000e+01 +85 40 4.600000000000000e+01 +85 41 4.500000000000000e+01 +85 42 4.400000000000000e+01 +85 43 4.300000000000000e+01 +85 44 4.200000000000000e+01 +85 45 4.100000000000000e+01 +85 46 4.000000000000000e+01 +85 47 3.900000000000000e+01 +85 48 3.800000000000000e+01 +85 49 3.700000000000000e+01 +85 50 3.600000000000000e+01 +85 51 3.500000000000000e+01 +85 52 3.400000000000000e+01 +85 53 3.300000000000000e+01 +85 54 3.200000000000000e+01 +85 55 3.100000000000000e+01 +85 56 3.000000000000000e+01 +85 57 2.900000000000000e+01 +85 58 2.800000000000000e+01 +85 59 2.700000000000000e+01 +85 60 2.600000000000000e+01 +85 61 2.500000000000000e+01 +85 62 2.400000000000000e+01 +85 63 2.300000000000000e+01 +85 64 2.200000000000000e+01 +85 65 2.100000000000000e+01 +85 66 2.000000000000000e+01 +85 67 1.900000000000000e+01 +85 68 1.800000000000000e+01 +85 69 1.700000000000000e+01 +85 70 1.600000000000000e+01 +85 71 1.500000000000000e+01 +85 72 1.400000000000000e+01 +85 73 1.300000000000000e+01 +85 74 1.200000000000000e+01 +85 75 1.100000000000000e+01 +85 76 1.000000000000000e+01 +85 77 9.000000000000000e+00 +85 78 8.000000000000000e+00 +85 79 7.000000000000000e+00 +85 80 6.000000000000000e+00 +85 81 5.000000000000000e+00 +85 82 4.000000000000000e+00 +85 83 3.000000000000000e+00 +85 84 2.000000000000000e+00 +85 85 2.000000000000000e+00 +85 86 2.000000000000000e+00 +85 87 3.000000000000000e+00 +85 88 4.000000000000000e+00 +85 89 5.000000000000000e+00 +85 90 6.000000000000000e+00 +85 91 7.000000000000000e+00 +85 92 8.000000000000000e+00 +85 93 9.000000000000000e+00 +85 94 1.000000000000000e+01 +85 95 1.100000000000000e+01 +85 96 1.200000000000000e+01 +85 97 1.300000000000000e+01 +85 98 1.400000000000000e+01 +85 99 1.500000000000000e+01 +85 100 1.600000000000000e+01 +86 1 8.600000000000000e+01 +86 2 8.500000000000000e+01 +86 3 8.400000000000000e+01 +86 4 8.300000000000000e+01 +86 5 8.200000000000000e+01 +86 6 8.100000000000000e+01 +86 7 8.000000000000000e+01 +86 8 7.900000000000000e+01 +86 9 7.800000000000000e+01 +86 10 7.700000000000000e+01 +86 11 7.600000000000000e+01 +86 12 7.500000000000000e+01 +86 13 7.400000000000000e+01 +86 14 7.300000000000000e+01 +86 15 7.200000000000000e+01 +86 16 7.100000000000000e+01 +86 17 7.000000000000000e+01 +86 18 6.900000000000000e+01 +86 19 6.800000000000000e+01 +86 20 6.700000000000000e+01 +86 21 6.600000000000000e+01 +86 22 6.500000000000000e+01 +86 23 6.400000000000000e+01 +86 24 6.300000000000000e+01 +86 25 6.200000000000000e+01 +86 26 6.100000000000000e+01 +86 27 6.000000000000000e+01 +86 28 5.900000000000000e+01 +86 29 5.800000000000000e+01 +86 30 5.700000000000000e+01 +86 31 5.600000000000000e+01 +86 32 5.500000000000000e+01 +86 33 5.400000000000000e+01 +86 34 5.300000000000000e+01 +86 35 5.200000000000000e+01 +86 36 5.100000000000000e+01 +86 37 5.000000000000000e+01 +86 38 4.900000000000000e+01 +86 39 4.800000000000000e+01 +86 40 4.700000000000000e+01 +86 41 4.600000000000000e+01 +86 42 4.500000000000000e+01 +86 43 4.400000000000000e+01 +86 44 4.300000000000000e+01 +86 45 4.200000000000000e+01 +86 46 4.100000000000000e+01 +86 47 4.000000000000000e+01 +86 48 3.900000000000000e+01 +86 49 3.800000000000000e+01 +86 50 3.700000000000000e+01 +86 51 3.600000000000000e+01 +86 52 3.500000000000000e+01 +86 53 3.400000000000000e+01 +86 54 3.300000000000000e+01 +86 55 3.200000000000000e+01 +86 56 3.100000000000000e+01 +86 57 3.000000000000000e+01 +86 58 2.900000000000000e+01 +86 59 2.800000000000000e+01 +86 60 2.700000000000000e+01 +86 61 2.600000000000000e+01 +86 62 2.500000000000000e+01 +86 63 2.400000000000000e+01 +86 64 2.300000000000000e+01 +86 65 2.200000000000000e+01 +86 66 2.100000000000000e+01 +86 67 2.000000000000000e+01 +86 68 1.900000000000000e+01 +86 69 1.800000000000000e+01 +86 70 1.700000000000000e+01 +86 71 1.600000000000000e+01 +86 72 1.500000000000000e+01 +86 73 1.400000000000000e+01 +86 74 1.300000000000000e+01 +86 75 1.200000000000000e+01 +86 76 1.100000000000000e+01 +86 77 1.000000000000000e+01 +86 78 9.000000000000000e+00 +86 79 8.000000000000000e+00 +86 80 7.000000000000000e+00 +86 81 6.000000000000000e+00 +86 82 5.000000000000000e+00 +86 83 4.000000000000000e+00 +86 84 3.000000000000000e+00 +86 85 2.000000000000000e+00 +86 86 2.000000000000000e+00 +86 87 2.000000000000000e+00 +86 88 3.000000000000000e+00 +86 89 4.000000000000000e+00 +86 90 5.000000000000000e+00 +86 91 6.000000000000000e+00 +86 92 7.000000000000000e+00 +86 93 8.000000000000000e+00 +86 94 9.000000000000000e+00 +86 95 1.000000000000000e+01 +86 96 1.100000000000000e+01 +86 97 1.200000000000000e+01 +86 98 1.300000000000000e+01 +86 99 1.400000000000000e+01 +86 100 1.500000000000000e+01 +87 1 8.700000000000000e+01 +87 2 8.600000000000000e+01 +87 3 8.500000000000000e+01 +87 4 8.400000000000000e+01 +87 5 8.300000000000000e+01 +87 6 8.200000000000000e+01 +87 7 8.100000000000000e+01 +87 8 8.000000000000000e+01 +87 9 7.900000000000000e+01 +87 10 7.800000000000000e+01 +87 11 7.700000000000000e+01 +87 12 7.600000000000000e+01 +87 13 7.500000000000000e+01 +87 14 7.400000000000000e+01 +87 15 7.300000000000000e+01 +87 16 7.200000000000000e+01 +87 17 7.100000000000000e+01 +87 18 7.000000000000000e+01 +87 19 6.900000000000000e+01 +87 20 6.800000000000000e+01 +87 21 6.700000000000000e+01 +87 22 6.600000000000000e+01 +87 23 6.500000000000000e+01 +87 24 6.400000000000000e+01 +87 25 6.300000000000000e+01 +87 26 6.200000000000000e+01 +87 27 6.100000000000000e+01 +87 28 6.000000000000000e+01 +87 29 5.900000000000000e+01 +87 30 5.800000000000000e+01 +87 31 5.700000000000000e+01 +87 32 5.600000000000000e+01 +87 33 5.500000000000000e+01 +87 34 5.400000000000000e+01 +87 35 5.300000000000000e+01 +87 36 5.200000000000000e+01 +87 37 5.100000000000000e+01 +87 38 5.000000000000000e+01 +87 39 4.900000000000000e+01 +87 40 4.800000000000000e+01 +87 41 4.700000000000000e+01 +87 42 4.600000000000000e+01 +87 43 4.500000000000000e+01 +87 44 4.400000000000000e+01 +87 45 4.300000000000000e+01 +87 46 4.200000000000000e+01 +87 47 4.100000000000000e+01 +87 48 4.000000000000000e+01 +87 49 3.900000000000000e+01 +87 50 3.800000000000000e+01 +87 51 3.700000000000000e+01 +87 52 3.600000000000000e+01 +87 53 3.500000000000000e+01 +87 54 3.400000000000000e+01 +87 55 3.300000000000000e+01 +87 56 3.200000000000000e+01 +87 57 3.100000000000000e+01 +87 58 3.000000000000000e+01 +87 59 2.900000000000000e+01 +87 60 2.800000000000000e+01 +87 61 2.700000000000000e+01 +87 62 2.600000000000000e+01 +87 63 2.500000000000000e+01 +87 64 2.400000000000000e+01 +87 65 2.300000000000000e+01 +87 66 2.200000000000000e+01 +87 67 2.100000000000000e+01 +87 68 2.000000000000000e+01 +87 69 1.900000000000000e+01 +87 70 1.800000000000000e+01 +87 71 1.700000000000000e+01 +87 72 1.600000000000000e+01 +87 73 1.500000000000000e+01 +87 74 1.400000000000000e+01 +87 75 1.300000000000000e+01 +87 76 1.200000000000000e+01 +87 77 1.100000000000000e+01 +87 78 1.000000000000000e+01 +87 79 9.000000000000000e+00 +87 80 8.000000000000000e+00 +87 81 7.000000000000000e+00 +87 82 6.000000000000000e+00 +87 83 5.000000000000000e+00 +87 84 4.000000000000000e+00 +87 85 3.000000000000000e+00 +87 86 2.000000000000000e+00 +87 87 2.000000000000000e+00 +87 88 2.000000000000000e+00 +87 89 3.000000000000000e+00 +87 90 4.000000000000000e+00 +87 91 5.000000000000000e+00 +87 92 6.000000000000000e+00 +87 93 7.000000000000000e+00 +87 94 8.000000000000000e+00 +87 95 9.000000000000000e+00 +87 96 1.000000000000000e+01 +87 97 1.100000000000000e+01 +87 98 1.200000000000000e+01 +87 99 1.300000000000000e+01 +87 100 1.400000000000000e+01 +88 1 8.800000000000000e+01 +88 2 8.700000000000000e+01 +88 3 8.600000000000000e+01 +88 4 8.500000000000000e+01 +88 5 8.400000000000000e+01 +88 6 8.300000000000000e+01 +88 7 8.200000000000000e+01 +88 8 8.100000000000000e+01 +88 9 8.000000000000000e+01 +88 10 7.900000000000000e+01 +88 11 7.800000000000000e+01 +88 12 7.700000000000000e+01 +88 13 7.600000000000000e+01 +88 14 7.500000000000000e+01 +88 15 7.400000000000000e+01 +88 16 7.300000000000000e+01 +88 17 7.200000000000000e+01 +88 18 7.100000000000000e+01 +88 19 7.000000000000000e+01 +88 20 6.900000000000000e+01 +88 21 6.800000000000000e+01 +88 22 6.700000000000000e+01 +88 23 6.600000000000000e+01 +88 24 6.500000000000000e+01 +88 25 6.400000000000000e+01 +88 26 6.300000000000000e+01 +88 27 6.200000000000000e+01 +88 28 6.100000000000000e+01 +88 29 6.000000000000000e+01 +88 30 5.900000000000000e+01 +88 31 5.800000000000000e+01 +88 32 5.700000000000000e+01 +88 33 5.600000000000000e+01 +88 34 5.500000000000000e+01 +88 35 5.400000000000000e+01 +88 36 5.300000000000000e+01 +88 37 5.200000000000000e+01 +88 38 5.100000000000000e+01 +88 39 5.000000000000000e+01 +88 40 4.900000000000000e+01 +88 41 4.800000000000000e+01 +88 42 4.700000000000000e+01 +88 43 4.600000000000000e+01 +88 44 4.500000000000000e+01 +88 45 4.400000000000000e+01 +88 46 4.300000000000000e+01 +88 47 4.200000000000000e+01 +88 48 4.100000000000000e+01 +88 49 4.000000000000000e+01 +88 50 3.900000000000000e+01 +88 51 3.800000000000000e+01 +88 52 3.700000000000000e+01 +88 53 3.600000000000000e+01 +88 54 3.500000000000000e+01 +88 55 3.400000000000000e+01 +88 56 3.300000000000000e+01 +88 57 3.200000000000000e+01 +88 58 3.100000000000000e+01 +88 59 3.000000000000000e+01 +88 60 2.900000000000000e+01 +88 61 2.800000000000000e+01 +88 62 2.700000000000000e+01 +88 63 2.600000000000000e+01 +88 64 2.500000000000000e+01 +88 65 2.400000000000000e+01 +88 66 2.300000000000000e+01 +88 67 2.200000000000000e+01 +88 68 2.100000000000000e+01 +88 69 2.000000000000000e+01 +88 70 1.900000000000000e+01 +88 71 1.800000000000000e+01 +88 72 1.700000000000000e+01 +88 73 1.600000000000000e+01 +88 74 1.500000000000000e+01 +88 75 1.400000000000000e+01 +88 76 1.300000000000000e+01 +88 77 1.200000000000000e+01 +88 78 1.100000000000000e+01 +88 79 1.000000000000000e+01 +88 80 9.000000000000000e+00 +88 81 8.000000000000000e+00 +88 82 7.000000000000000e+00 +88 83 6.000000000000000e+00 +88 84 5.000000000000000e+00 +88 85 4.000000000000000e+00 +88 86 3.000000000000000e+00 +88 87 2.000000000000000e+00 +88 88 2.000000000000000e+00 +88 89 2.000000000000000e+00 +88 90 3.000000000000000e+00 +88 91 4.000000000000000e+00 +88 92 5.000000000000000e+00 +88 93 6.000000000000000e+00 +88 94 7.000000000000000e+00 +88 95 8.000000000000000e+00 +88 96 9.000000000000000e+00 +88 97 1.000000000000000e+01 +88 98 1.100000000000000e+01 +88 99 1.200000000000000e+01 +88 100 1.300000000000000e+01 +89 1 8.900000000000000e+01 +89 2 8.800000000000000e+01 +89 3 8.700000000000000e+01 +89 4 8.600000000000000e+01 +89 5 8.500000000000000e+01 +89 6 8.400000000000000e+01 +89 7 8.300000000000000e+01 +89 8 8.200000000000000e+01 +89 9 8.100000000000000e+01 +89 10 8.000000000000000e+01 +89 11 7.900000000000000e+01 +89 12 7.800000000000000e+01 +89 13 7.700000000000000e+01 +89 14 7.600000000000000e+01 +89 15 7.500000000000000e+01 +89 16 7.400000000000000e+01 +89 17 7.300000000000000e+01 +89 18 7.200000000000000e+01 +89 19 7.100000000000000e+01 +89 20 7.000000000000000e+01 +89 21 6.900000000000000e+01 +89 22 6.800000000000000e+01 +89 23 6.700000000000000e+01 +89 24 6.600000000000000e+01 +89 25 6.500000000000000e+01 +89 26 6.400000000000000e+01 +89 27 6.300000000000000e+01 +89 28 6.200000000000000e+01 +89 29 6.100000000000000e+01 +89 30 6.000000000000000e+01 +89 31 5.900000000000000e+01 +89 32 5.800000000000000e+01 +89 33 5.700000000000000e+01 +89 34 5.600000000000000e+01 +89 35 5.500000000000000e+01 +89 36 5.400000000000000e+01 +89 37 5.300000000000000e+01 +89 38 5.200000000000000e+01 +89 39 5.100000000000000e+01 +89 40 5.000000000000000e+01 +89 41 4.900000000000000e+01 +89 42 4.800000000000000e+01 +89 43 4.700000000000000e+01 +89 44 4.600000000000000e+01 +89 45 4.500000000000000e+01 +89 46 4.400000000000000e+01 +89 47 4.300000000000000e+01 +89 48 4.200000000000000e+01 +89 49 4.100000000000000e+01 +89 50 4.000000000000000e+01 +89 51 3.900000000000000e+01 +89 52 3.800000000000000e+01 +89 53 3.700000000000000e+01 +89 54 3.600000000000000e+01 +89 55 3.500000000000000e+01 +89 56 3.400000000000000e+01 +89 57 3.300000000000000e+01 +89 58 3.200000000000000e+01 +89 59 3.100000000000000e+01 +89 60 3.000000000000000e+01 +89 61 2.900000000000000e+01 +89 62 2.800000000000000e+01 +89 63 2.700000000000000e+01 +89 64 2.600000000000000e+01 +89 65 2.500000000000000e+01 +89 66 2.400000000000000e+01 +89 67 2.300000000000000e+01 +89 68 2.200000000000000e+01 +89 69 2.100000000000000e+01 +89 70 2.000000000000000e+01 +89 71 1.900000000000000e+01 +89 72 1.800000000000000e+01 +89 73 1.700000000000000e+01 +89 74 1.600000000000000e+01 +89 75 1.500000000000000e+01 +89 76 1.400000000000000e+01 +89 77 1.300000000000000e+01 +89 78 1.200000000000000e+01 +89 79 1.100000000000000e+01 +89 80 1.000000000000000e+01 +89 81 9.000000000000000e+00 +89 82 8.000000000000000e+00 +89 83 7.000000000000000e+00 +89 84 6.000000000000000e+00 +89 85 5.000000000000000e+00 +89 86 4.000000000000000e+00 +89 87 3.000000000000000e+00 +89 88 2.000000000000000e+00 +89 89 2.000000000000000e+00 +89 90 2.000000000000000e+00 +89 91 3.000000000000000e+00 +89 92 4.000000000000000e+00 +89 93 5.000000000000000e+00 +89 94 6.000000000000000e+00 +89 95 7.000000000000000e+00 +89 96 8.000000000000000e+00 +89 97 9.000000000000000e+00 +89 98 1.000000000000000e+01 +89 99 1.100000000000000e+01 +89 100 1.200000000000000e+01 +90 1 9.000000000000000e+01 +90 2 8.900000000000000e+01 +90 3 8.800000000000000e+01 +90 4 8.700000000000000e+01 +90 5 8.600000000000000e+01 +90 6 8.500000000000000e+01 +90 7 8.400000000000000e+01 +90 8 8.300000000000000e+01 +90 9 8.200000000000000e+01 +90 10 8.100000000000000e+01 +90 11 8.000000000000000e+01 +90 12 7.900000000000000e+01 +90 13 7.800000000000000e+01 +90 14 7.700000000000000e+01 +90 15 7.600000000000000e+01 +90 16 7.500000000000000e+01 +90 17 7.400000000000000e+01 +90 18 7.300000000000000e+01 +90 19 7.200000000000000e+01 +90 20 7.100000000000000e+01 +90 21 7.000000000000000e+01 +90 22 6.900000000000000e+01 +90 23 6.800000000000000e+01 +90 24 6.700000000000000e+01 +90 25 6.600000000000000e+01 +90 26 6.500000000000000e+01 +90 27 6.400000000000000e+01 +90 28 6.300000000000000e+01 +90 29 6.200000000000000e+01 +90 30 6.100000000000000e+01 +90 31 6.000000000000000e+01 +90 32 5.900000000000000e+01 +90 33 5.800000000000000e+01 +90 34 5.700000000000000e+01 +90 35 5.600000000000000e+01 +90 36 5.500000000000000e+01 +90 37 5.400000000000000e+01 +90 38 5.300000000000000e+01 +90 39 5.200000000000000e+01 +90 40 5.100000000000000e+01 +90 41 5.000000000000000e+01 +90 42 4.900000000000000e+01 +90 43 4.800000000000000e+01 +90 44 4.700000000000000e+01 +90 45 4.600000000000000e+01 +90 46 4.500000000000000e+01 +90 47 4.400000000000000e+01 +90 48 4.300000000000000e+01 +90 49 4.200000000000000e+01 +90 50 4.100000000000000e+01 +90 51 4.000000000000000e+01 +90 52 3.900000000000000e+01 +90 53 3.800000000000000e+01 +90 54 3.700000000000000e+01 +90 55 3.600000000000000e+01 +90 56 3.500000000000000e+01 +90 57 3.400000000000000e+01 +90 58 3.300000000000000e+01 +90 59 3.200000000000000e+01 +90 60 3.100000000000000e+01 +90 61 3.000000000000000e+01 +90 62 2.900000000000000e+01 +90 63 2.800000000000000e+01 +90 64 2.700000000000000e+01 +90 65 2.600000000000000e+01 +90 66 2.500000000000000e+01 +90 67 2.400000000000000e+01 +90 68 2.300000000000000e+01 +90 69 2.200000000000000e+01 +90 70 2.100000000000000e+01 +90 71 2.000000000000000e+01 +90 72 1.900000000000000e+01 +90 73 1.800000000000000e+01 +90 74 1.700000000000000e+01 +90 75 1.600000000000000e+01 +90 76 1.500000000000000e+01 +90 77 1.400000000000000e+01 +90 78 1.300000000000000e+01 +90 79 1.200000000000000e+01 +90 80 1.100000000000000e+01 +90 81 1.000000000000000e+01 +90 82 9.000000000000000e+00 +90 83 8.000000000000000e+00 +90 84 7.000000000000000e+00 +90 85 6.000000000000000e+00 +90 86 5.000000000000000e+00 +90 87 4.000000000000000e+00 +90 88 3.000000000000000e+00 +90 89 2.000000000000000e+00 +90 90 2.000000000000000e+00 +90 91 2.000000000000000e+00 +90 92 3.000000000000000e+00 +90 93 4.000000000000000e+00 +90 94 5.000000000000000e+00 +90 95 6.000000000000000e+00 +90 96 7.000000000000000e+00 +90 97 8.000000000000000e+00 +90 98 9.000000000000000e+00 +90 99 1.000000000000000e+01 +90 100 1.100000000000000e+01 +91 1 9.100000000000000e+01 +91 2 9.000000000000000e+01 +91 3 8.900000000000000e+01 +91 4 8.800000000000000e+01 +91 5 8.700000000000000e+01 +91 6 8.600000000000000e+01 +91 7 8.500000000000000e+01 +91 8 8.400000000000000e+01 +91 9 8.300000000000000e+01 +91 10 8.200000000000000e+01 +91 11 8.100000000000000e+01 +91 12 8.000000000000000e+01 +91 13 7.900000000000000e+01 +91 14 7.800000000000000e+01 +91 15 7.700000000000000e+01 +91 16 7.600000000000000e+01 +91 17 7.500000000000000e+01 +91 18 7.400000000000000e+01 +91 19 7.300000000000000e+01 +91 20 7.200000000000000e+01 +91 21 7.100000000000000e+01 +91 22 7.000000000000000e+01 +91 23 6.900000000000000e+01 +91 24 6.800000000000000e+01 +91 25 6.700000000000000e+01 +91 26 6.600000000000000e+01 +91 27 6.500000000000000e+01 +91 28 6.400000000000000e+01 +91 29 6.300000000000000e+01 +91 30 6.200000000000000e+01 +91 31 6.100000000000000e+01 +91 32 6.000000000000000e+01 +91 33 5.900000000000000e+01 +91 34 5.800000000000000e+01 +91 35 5.700000000000000e+01 +91 36 5.600000000000000e+01 +91 37 5.500000000000000e+01 +91 38 5.400000000000000e+01 +91 39 5.300000000000000e+01 +91 40 5.200000000000000e+01 +91 41 5.100000000000000e+01 +91 42 5.000000000000000e+01 +91 43 4.900000000000000e+01 +91 44 4.800000000000000e+01 +91 45 4.700000000000000e+01 +91 46 4.600000000000000e+01 +91 47 4.500000000000000e+01 +91 48 4.400000000000000e+01 +91 49 4.300000000000000e+01 +91 50 4.200000000000000e+01 +91 51 4.100000000000000e+01 +91 52 4.000000000000000e+01 +91 53 3.900000000000000e+01 +91 54 3.800000000000000e+01 +91 55 3.700000000000000e+01 +91 56 3.600000000000000e+01 +91 57 3.500000000000000e+01 +91 58 3.400000000000000e+01 +91 59 3.300000000000000e+01 +91 60 3.200000000000000e+01 +91 61 3.100000000000000e+01 +91 62 3.000000000000000e+01 +91 63 2.900000000000000e+01 +91 64 2.800000000000000e+01 +91 65 2.700000000000000e+01 +91 66 2.600000000000000e+01 +91 67 2.500000000000000e+01 +91 68 2.400000000000000e+01 +91 69 2.300000000000000e+01 +91 70 2.200000000000000e+01 +91 71 2.100000000000000e+01 +91 72 2.000000000000000e+01 +91 73 1.900000000000000e+01 +91 74 1.800000000000000e+01 +91 75 1.700000000000000e+01 +91 76 1.600000000000000e+01 +91 77 1.500000000000000e+01 +91 78 1.400000000000000e+01 +91 79 1.300000000000000e+01 +91 80 1.200000000000000e+01 +91 81 1.100000000000000e+01 +91 82 1.000000000000000e+01 +91 83 9.000000000000000e+00 +91 84 8.000000000000000e+00 +91 85 7.000000000000000e+00 +91 86 6.000000000000000e+00 +91 87 5.000000000000000e+00 +91 88 4.000000000000000e+00 +91 89 3.000000000000000e+00 +91 90 2.000000000000000e+00 +91 91 2.000000000000000e+00 +91 92 2.000000000000000e+00 +91 93 3.000000000000000e+00 +91 94 4.000000000000000e+00 +91 95 5.000000000000000e+00 +91 96 6.000000000000000e+00 +91 97 7.000000000000000e+00 +91 98 8.000000000000000e+00 +91 99 9.000000000000000e+00 +91 100 1.000000000000000e+01 +92 1 9.200000000000000e+01 +92 2 9.100000000000000e+01 +92 3 9.000000000000000e+01 +92 4 8.900000000000000e+01 +92 5 8.800000000000000e+01 +92 6 8.700000000000000e+01 +92 7 8.600000000000000e+01 +92 8 8.500000000000000e+01 +92 9 8.400000000000000e+01 +92 10 8.300000000000000e+01 +92 11 8.200000000000000e+01 +92 12 8.100000000000000e+01 +92 13 8.000000000000000e+01 +92 14 7.900000000000000e+01 +92 15 7.800000000000000e+01 +92 16 7.700000000000000e+01 +92 17 7.600000000000000e+01 +92 18 7.500000000000000e+01 +92 19 7.400000000000000e+01 +92 20 7.300000000000000e+01 +92 21 7.200000000000000e+01 +92 22 7.100000000000000e+01 +92 23 7.000000000000000e+01 +92 24 6.900000000000000e+01 +92 25 6.800000000000000e+01 +92 26 6.700000000000000e+01 +92 27 6.600000000000000e+01 +92 28 6.500000000000000e+01 +92 29 6.400000000000000e+01 +92 30 6.300000000000000e+01 +92 31 6.200000000000000e+01 +92 32 6.100000000000000e+01 +92 33 6.000000000000000e+01 +92 34 5.900000000000000e+01 +92 35 5.800000000000000e+01 +92 36 5.700000000000000e+01 +92 37 5.600000000000000e+01 +92 38 5.500000000000000e+01 +92 39 5.400000000000000e+01 +92 40 5.300000000000000e+01 +92 41 5.200000000000000e+01 +92 42 5.100000000000000e+01 +92 43 5.000000000000000e+01 +92 44 4.900000000000000e+01 +92 45 4.800000000000000e+01 +92 46 4.700000000000000e+01 +92 47 4.600000000000000e+01 +92 48 4.500000000000000e+01 +92 49 4.400000000000000e+01 +92 50 4.300000000000000e+01 +92 51 4.200000000000000e+01 +92 52 4.100000000000000e+01 +92 53 4.000000000000000e+01 +92 54 3.900000000000000e+01 +92 55 3.800000000000000e+01 +92 56 3.700000000000000e+01 +92 57 3.600000000000000e+01 +92 58 3.500000000000000e+01 +92 59 3.400000000000000e+01 +92 60 3.300000000000000e+01 +92 61 3.200000000000000e+01 +92 62 3.100000000000000e+01 +92 63 3.000000000000000e+01 +92 64 2.900000000000000e+01 +92 65 2.800000000000000e+01 +92 66 2.700000000000000e+01 +92 67 2.600000000000000e+01 +92 68 2.500000000000000e+01 +92 69 2.400000000000000e+01 +92 70 2.300000000000000e+01 +92 71 2.200000000000000e+01 +92 72 2.100000000000000e+01 +92 73 2.000000000000000e+01 +92 74 1.900000000000000e+01 +92 75 1.800000000000000e+01 +92 76 1.700000000000000e+01 +92 77 1.600000000000000e+01 +92 78 1.500000000000000e+01 +92 79 1.400000000000000e+01 +92 80 1.300000000000000e+01 +92 81 1.200000000000000e+01 +92 82 1.100000000000000e+01 +92 83 1.000000000000000e+01 +92 84 9.000000000000000e+00 +92 85 8.000000000000000e+00 +92 86 7.000000000000000e+00 +92 87 6.000000000000000e+00 +92 88 5.000000000000000e+00 +92 89 4.000000000000000e+00 +92 90 3.000000000000000e+00 +92 91 2.000000000000000e+00 +92 92 2.000000000000000e+00 +92 93 2.000000000000000e+00 +92 94 3.000000000000000e+00 +92 95 4.000000000000000e+00 +92 96 5.000000000000000e+00 +92 97 6.000000000000000e+00 +92 98 7.000000000000000e+00 +92 99 8.000000000000000e+00 +92 100 9.000000000000000e+00 +93 1 9.300000000000000e+01 +93 2 9.200000000000000e+01 +93 3 9.100000000000000e+01 +93 4 9.000000000000000e+01 +93 5 8.900000000000000e+01 +93 6 8.800000000000000e+01 +93 7 8.700000000000000e+01 +93 8 8.600000000000000e+01 +93 9 8.500000000000000e+01 +93 10 8.400000000000000e+01 +93 11 8.300000000000000e+01 +93 12 8.200000000000000e+01 +93 13 8.100000000000000e+01 +93 14 8.000000000000000e+01 +93 15 7.900000000000000e+01 +93 16 7.800000000000000e+01 +93 17 7.700000000000000e+01 +93 18 7.600000000000000e+01 +93 19 7.500000000000000e+01 +93 20 7.400000000000000e+01 +93 21 7.300000000000000e+01 +93 22 7.200000000000000e+01 +93 23 7.100000000000000e+01 +93 24 7.000000000000000e+01 +93 25 6.900000000000000e+01 +93 26 6.800000000000000e+01 +93 27 6.700000000000000e+01 +93 28 6.600000000000000e+01 +93 29 6.500000000000000e+01 +93 30 6.400000000000000e+01 +93 31 6.300000000000000e+01 +93 32 6.200000000000000e+01 +93 33 6.100000000000000e+01 +93 34 6.000000000000000e+01 +93 35 5.900000000000000e+01 +93 36 5.800000000000000e+01 +93 37 5.700000000000000e+01 +93 38 5.600000000000000e+01 +93 39 5.500000000000000e+01 +93 40 5.400000000000000e+01 +93 41 5.300000000000000e+01 +93 42 5.200000000000000e+01 +93 43 5.100000000000000e+01 +93 44 5.000000000000000e+01 +93 45 4.900000000000000e+01 +93 46 4.800000000000000e+01 +93 47 4.700000000000000e+01 +93 48 4.600000000000000e+01 +93 49 4.500000000000000e+01 +93 50 4.400000000000000e+01 +93 51 4.300000000000000e+01 +93 52 4.200000000000000e+01 +93 53 4.100000000000000e+01 +93 54 4.000000000000000e+01 +93 55 3.900000000000000e+01 +93 56 3.800000000000000e+01 +93 57 3.700000000000000e+01 +93 58 3.600000000000000e+01 +93 59 3.500000000000000e+01 +93 60 3.400000000000000e+01 +93 61 3.300000000000000e+01 +93 62 3.200000000000000e+01 +93 63 3.100000000000000e+01 +93 64 3.000000000000000e+01 +93 65 2.900000000000000e+01 +93 66 2.800000000000000e+01 +93 67 2.700000000000000e+01 +93 68 2.600000000000000e+01 +93 69 2.500000000000000e+01 +93 70 2.400000000000000e+01 +93 71 2.300000000000000e+01 +93 72 2.200000000000000e+01 +93 73 2.100000000000000e+01 +93 74 2.000000000000000e+01 +93 75 1.900000000000000e+01 +93 76 1.800000000000000e+01 +93 77 1.700000000000000e+01 +93 78 1.600000000000000e+01 +93 79 1.500000000000000e+01 +93 80 1.400000000000000e+01 +93 81 1.300000000000000e+01 +93 82 1.200000000000000e+01 +93 83 1.100000000000000e+01 +93 84 1.000000000000000e+01 +93 85 9.000000000000000e+00 +93 86 8.000000000000000e+00 +93 87 7.000000000000000e+00 +93 88 6.000000000000000e+00 +93 89 5.000000000000000e+00 +93 90 4.000000000000000e+00 +93 91 3.000000000000000e+00 +93 92 2.000000000000000e+00 +93 93 2.000000000000000e+00 +93 94 2.000000000000000e+00 +93 95 3.000000000000000e+00 +93 96 4.000000000000000e+00 +93 97 5.000000000000000e+00 +93 98 6.000000000000000e+00 +93 99 7.000000000000000e+00 +93 100 8.000000000000000e+00 +94 1 9.400000000000000e+01 +94 2 9.300000000000000e+01 +94 3 9.200000000000000e+01 +94 4 9.100000000000000e+01 +94 5 9.000000000000000e+01 +94 6 8.900000000000000e+01 +94 7 8.800000000000000e+01 +94 8 8.700000000000000e+01 +94 9 8.600000000000000e+01 +94 10 8.500000000000000e+01 +94 11 8.400000000000000e+01 +94 12 8.300000000000000e+01 +94 13 8.200000000000000e+01 +94 14 8.100000000000000e+01 +94 15 8.000000000000000e+01 +94 16 7.900000000000000e+01 +94 17 7.800000000000000e+01 +94 18 7.700000000000000e+01 +94 19 7.600000000000000e+01 +94 20 7.500000000000000e+01 +94 21 7.400000000000000e+01 +94 22 7.300000000000000e+01 +94 23 7.200000000000000e+01 +94 24 7.100000000000000e+01 +94 25 7.000000000000000e+01 +94 26 6.900000000000000e+01 +94 27 6.800000000000000e+01 +94 28 6.700000000000000e+01 +94 29 6.600000000000000e+01 +94 30 6.500000000000000e+01 +94 31 6.400000000000000e+01 +94 32 6.300000000000000e+01 +94 33 6.200000000000000e+01 +94 34 6.100000000000000e+01 +94 35 6.000000000000000e+01 +94 36 5.900000000000000e+01 +94 37 5.800000000000000e+01 +94 38 5.700000000000000e+01 +94 39 5.600000000000000e+01 +94 40 5.500000000000000e+01 +94 41 5.400000000000000e+01 +94 42 5.300000000000000e+01 +94 43 5.200000000000000e+01 +94 44 5.100000000000000e+01 +94 45 5.000000000000000e+01 +94 46 4.900000000000000e+01 +94 47 4.800000000000000e+01 +94 48 4.700000000000000e+01 +94 49 4.600000000000000e+01 +94 50 4.500000000000000e+01 +94 51 4.400000000000000e+01 +94 52 4.300000000000000e+01 +94 53 4.200000000000000e+01 +94 54 4.100000000000000e+01 +94 55 4.000000000000000e+01 +94 56 3.900000000000000e+01 +94 57 3.800000000000000e+01 +94 58 3.700000000000000e+01 +94 59 3.600000000000000e+01 +94 60 3.500000000000000e+01 +94 61 3.400000000000000e+01 +94 62 3.300000000000000e+01 +94 63 3.200000000000000e+01 +94 64 3.100000000000000e+01 +94 65 3.000000000000000e+01 +94 66 2.900000000000000e+01 +94 67 2.800000000000000e+01 +94 68 2.700000000000000e+01 +94 69 2.600000000000000e+01 +94 70 2.500000000000000e+01 +94 71 2.400000000000000e+01 +94 72 2.300000000000000e+01 +94 73 2.200000000000000e+01 +94 74 2.100000000000000e+01 +94 75 2.000000000000000e+01 +94 76 1.900000000000000e+01 +94 77 1.800000000000000e+01 +94 78 1.700000000000000e+01 +94 79 1.600000000000000e+01 +94 80 1.500000000000000e+01 +94 81 1.400000000000000e+01 +94 82 1.300000000000000e+01 +94 83 1.200000000000000e+01 +94 84 1.100000000000000e+01 +94 85 1.000000000000000e+01 +94 86 9.000000000000000e+00 +94 87 8.000000000000000e+00 +94 88 7.000000000000000e+00 +94 89 6.000000000000000e+00 +94 90 5.000000000000000e+00 +94 91 4.000000000000000e+00 +94 92 3.000000000000000e+00 +94 93 2.000000000000000e+00 +94 94 2.000000000000000e+00 +94 95 2.000000000000000e+00 +94 96 3.000000000000000e+00 +94 97 4.000000000000000e+00 +94 98 5.000000000000000e+00 +94 99 6.000000000000000e+00 +94 100 7.000000000000000e+00 +95 1 9.500000000000000e+01 +95 2 9.400000000000000e+01 +95 3 9.300000000000000e+01 +95 4 9.200000000000000e+01 +95 5 9.100000000000000e+01 +95 6 9.000000000000000e+01 +95 7 8.900000000000000e+01 +95 8 8.800000000000000e+01 +95 9 8.700000000000000e+01 +95 10 8.600000000000000e+01 +95 11 8.500000000000000e+01 +95 12 8.400000000000000e+01 +95 13 8.300000000000000e+01 +95 14 8.200000000000000e+01 +95 15 8.100000000000000e+01 +95 16 8.000000000000000e+01 +95 17 7.900000000000000e+01 +95 18 7.800000000000000e+01 +95 19 7.700000000000000e+01 +95 20 7.600000000000000e+01 +95 21 7.500000000000000e+01 +95 22 7.400000000000000e+01 +95 23 7.300000000000000e+01 +95 24 7.200000000000000e+01 +95 25 7.100000000000000e+01 +95 26 7.000000000000000e+01 +95 27 6.900000000000000e+01 +95 28 6.800000000000000e+01 +95 29 6.700000000000000e+01 +95 30 6.600000000000000e+01 +95 31 6.500000000000000e+01 +95 32 6.400000000000000e+01 +95 33 6.300000000000000e+01 +95 34 6.200000000000000e+01 +95 35 6.100000000000000e+01 +95 36 6.000000000000000e+01 +95 37 5.900000000000000e+01 +95 38 5.800000000000000e+01 +95 39 5.700000000000000e+01 +95 40 5.600000000000000e+01 +95 41 5.500000000000000e+01 +95 42 5.400000000000000e+01 +95 43 5.300000000000000e+01 +95 44 5.200000000000000e+01 +95 45 5.100000000000000e+01 +95 46 5.000000000000000e+01 +95 47 4.900000000000000e+01 +95 48 4.800000000000000e+01 +95 49 4.700000000000000e+01 +95 50 4.600000000000000e+01 +95 51 4.500000000000000e+01 +95 52 4.400000000000000e+01 +95 53 4.300000000000000e+01 +95 54 4.200000000000000e+01 +95 55 4.100000000000000e+01 +95 56 4.000000000000000e+01 +95 57 3.900000000000000e+01 +95 58 3.800000000000000e+01 +95 59 3.700000000000000e+01 +95 60 3.600000000000000e+01 +95 61 3.500000000000000e+01 +95 62 3.400000000000000e+01 +95 63 3.300000000000000e+01 +95 64 3.200000000000000e+01 +95 65 3.100000000000000e+01 +95 66 3.000000000000000e+01 +95 67 2.900000000000000e+01 +95 68 2.800000000000000e+01 +95 69 2.700000000000000e+01 +95 70 2.600000000000000e+01 +95 71 2.500000000000000e+01 +95 72 2.400000000000000e+01 +95 73 2.300000000000000e+01 +95 74 2.200000000000000e+01 +95 75 2.100000000000000e+01 +95 76 2.000000000000000e+01 +95 77 1.900000000000000e+01 +95 78 1.800000000000000e+01 +95 79 1.700000000000000e+01 +95 80 1.600000000000000e+01 +95 81 1.500000000000000e+01 +95 82 1.400000000000000e+01 +95 83 1.300000000000000e+01 +95 84 1.200000000000000e+01 +95 85 1.100000000000000e+01 +95 86 1.000000000000000e+01 +95 87 9.000000000000000e+00 +95 88 8.000000000000000e+00 +95 89 7.000000000000000e+00 +95 90 6.000000000000000e+00 +95 91 5.000000000000000e+00 +95 92 4.000000000000000e+00 +95 93 3.000000000000000e+00 +95 94 2.000000000000000e+00 +95 95 2.000000000000000e+00 +95 96 2.000000000000000e+00 +95 97 3.000000000000000e+00 +95 98 4.000000000000000e+00 +95 99 5.000000000000000e+00 +95 100 6.000000000000000e+00 +96 1 9.600000000000000e+01 +96 2 9.500000000000000e+01 +96 3 9.400000000000000e+01 +96 4 9.300000000000000e+01 +96 5 9.200000000000000e+01 +96 6 9.100000000000000e+01 +96 7 9.000000000000000e+01 +96 8 8.900000000000000e+01 +96 9 8.800000000000000e+01 +96 10 8.700000000000000e+01 +96 11 8.600000000000000e+01 +96 12 8.500000000000000e+01 +96 13 8.400000000000000e+01 +96 14 8.300000000000000e+01 +96 15 8.200000000000000e+01 +96 16 8.100000000000000e+01 +96 17 8.000000000000000e+01 +96 18 7.900000000000000e+01 +96 19 7.800000000000000e+01 +96 20 7.700000000000000e+01 +96 21 7.600000000000000e+01 +96 22 7.500000000000000e+01 +96 23 7.400000000000000e+01 +96 24 7.300000000000000e+01 +96 25 7.200000000000000e+01 +96 26 7.100000000000000e+01 +96 27 7.000000000000000e+01 +96 28 6.900000000000000e+01 +96 29 6.800000000000000e+01 +96 30 6.700000000000000e+01 +96 31 6.600000000000000e+01 +96 32 6.500000000000000e+01 +96 33 6.400000000000000e+01 +96 34 6.300000000000000e+01 +96 35 6.200000000000000e+01 +96 36 6.100000000000000e+01 +96 37 6.000000000000000e+01 +96 38 5.900000000000000e+01 +96 39 5.800000000000000e+01 +96 40 5.700000000000000e+01 +96 41 5.600000000000000e+01 +96 42 5.500000000000000e+01 +96 43 5.400000000000000e+01 +96 44 5.300000000000000e+01 +96 45 5.200000000000000e+01 +96 46 5.100000000000000e+01 +96 47 5.000000000000000e+01 +96 48 4.900000000000000e+01 +96 49 4.800000000000000e+01 +96 50 4.700000000000000e+01 +96 51 4.600000000000000e+01 +96 52 4.500000000000000e+01 +96 53 4.400000000000000e+01 +96 54 4.300000000000000e+01 +96 55 4.200000000000000e+01 +96 56 4.100000000000000e+01 +96 57 4.000000000000000e+01 +96 58 3.900000000000000e+01 +96 59 3.800000000000000e+01 +96 60 3.700000000000000e+01 +96 61 3.600000000000000e+01 +96 62 3.500000000000000e+01 +96 63 3.400000000000000e+01 +96 64 3.300000000000000e+01 +96 65 3.200000000000000e+01 +96 66 3.100000000000000e+01 +96 67 3.000000000000000e+01 +96 68 2.900000000000000e+01 +96 69 2.800000000000000e+01 +96 70 2.700000000000000e+01 +96 71 2.600000000000000e+01 +96 72 2.500000000000000e+01 +96 73 2.400000000000000e+01 +96 74 2.300000000000000e+01 +96 75 2.200000000000000e+01 +96 76 2.100000000000000e+01 +96 77 2.000000000000000e+01 +96 78 1.900000000000000e+01 +96 79 1.800000000000000e+01 +96 80 1.700000000000000e+01 +96 81 1.600000000000000e+01 +96 82 1.500000000000000e+01 +96 83 1.400000000000000e+01 +96 84 1.300000000000000e+01 +96 85 1.200000000000000e+01 +96 86 1.100000000000000e+01 +96 87 1.000000000000000e+01 +96 88 9.000000000000000e+00 +96 89 8.000000000000000e+00 +96 90 7.000000000000000e+00 +96 91 6.000000000000000e+00 +96 92 5.000000000000000e+00 +96 93 4.000000000000000e+00 +96 94 3.000000000000000e+00 +96 95 2.000000000000000e+00 +96 96 2.000000000000000e+00 +96 97 2.000000000000000e+00 +96 98 3.000000000000000e+00 +96 99 4.000000000000000e+00 +96 100 5.000000000000000e+00 +97 1 9.700000000000000e+01 +97 2 9.600000000000000e+01 +97 3 9.500000000000000e+01 +97 4 9.400000000000000e+01 +97 5 9.300000000000000e+01 +97 6 9.200000000000000e+01 +97 7 9.100000000000000e+01 +97 8 9.000000000000000e+01 +97 9 8.900000000000000e+01 +97 10 8.800000000000000e+01 +97 11 8.700000000000000e+01 +97 12 8.600000000000000e+01 +97 13 8.500000000000000e+01 +97 14 8.400000000000000e+01 +97 15 8.300000000000000e+01 +97 16 8.200000000000000e+01 +97 17 8.100000000000000e+01 +97 18 8.000000000000000e+01 +97 19 7.900000000000000e+01 +97 20 7.800000000000000e+01 +97 21 7.700000000000000e+01 +97 22 7.600000000000000e+01 +97 23 7.500000000000000e+01 +97 24 7.400000000000000e+01 +97 25 7.300000000000000e+01 +97 26 7.200000000000000e+01 +97 27 7.100000000000000e+01 +97 28 7.000000000000000e+01 +97 29 6.900000000000000e+01 +97 30 6.800000000000000e+01 +97 31 6.700000000000000e+01 +97 32 6.600000000000000e+01 +97 33 6.500000000000000e+01 +97 34 6.400000000000000e+01 +97 35 6.300000000000000e+01 +97 36 6.200000000000000e+01 +97 37 6.100000000000000e+01 +97 38 6.000000000000000e+01 +97 39 5.900000000000000e+01 +97 40 5.800000000000000e+01 +97 41 5.700000000000000e+01 +97 42 5.600000000000000e+01 +97 43 5.500000000000000e+01 +97 44 5.400000000000000e+01 +97 45 5.300000000000000e+01 +97 46 5.200000000000000e+01 +97 47 5.100000000000000e+01 +97 48 5.000000000000000e+01 +97 49 4.900000000000000e+01 +97 50 4.800000000000000e+01 +97 51 4.700000000000000e+01 +97 52 4.600000000000000e+01 +97 53 4.500000000000000e+01 +97 54 4.400000000000000e+01 +97 55 4.300000000000000e+01 +97 56 4.200000000000000e+01 +97 57 4.100000000000000e+01 +97 58 4.000000000000000e+01 +97 59 3.900000000000000e+01 +97 60 3.800000000000000e+01 +97 61 3.700000000000000e+01 +97 62 3.600000000000000e+01 +97 63 3.500000000000000e+01 +97 64 3.400000000000000e+01 +97 65 3.300000000000000e+01 +97 66 3.200000000000000e+01 +97 67 3.100000000000000e+01 +97 68 3.000000000000000e+01 +97 69 2.900000000000000e+01 +97 70 2.800000000000000e+01 +97 71 2.700000000000000e+01 +97 72 2.600000000000000e+01 +97 73 2.500000000000000e+01 +97 74 2.400000000000000e+01 +97 75 2.300000000000000e+01 +97 76 2.200000000000000e+01 +97 77 2.100000000000000e+01 +97 78 2.000000000000000e+01 +97 79 1.900000000000000e+01 +97 80 1.800000000000000e+01 +97 81 1.700000000000000e+01 +97 82 1.600000000000000e+01 +97 83 1.500000000000000e+01 +97 84 1.400000000000000e+01 +97 85 1.300000000000000e+01 +97 86 1.200000000000000e+01 +97 87 1.100000000000000e+01 +97 88 1.000000000000000e+01 +97 89 9.000000000000000e+00 +97 90 8.000000000000000e+00 +97 91 7.000000000000000e+00 +97 92 6.000000000000000e+00 +97 93 5.000000000000000e+00 +97 94 4.000000000000000e+00 +97 95 3.000000000000000e+00 +97 96 2.000000000000000e+00 +97 97 2.000000000000000e+00 +97 98 2.000000000000000e+00 +97 99 3.000000000000000e+00 +97 100 4.000000000000000e+00 +98 1 9.800000000000000e+01 +98 2 9.700000000000000e+01 +98 3 9.600000000000000e+01 +98 4 9.500000000000000e+01 +98 5 9.400000000000000e+01 +98 6 9.300000000000000e+01 +98 7 9.200000000000000e+01 +98 8 9.100000000000000e+01 +98 9 9.000000000000000e+01 +98 10 8.900000000000000e+01 +98 11 8.800000000000000e+01 +98 12 8.700000000000000e+01 +98 13 8.600000000000000e+01 +98 14 8.500000000000000e+01 +98 15 8.400000000000000e+01 +98 16 8.300000000000000e+01 +98 17 8.200000000000000e+01 +98 18 8.100000000000000e+01 +98 19 8.000000000000000e+01 +98 20 7.900000000000000e+01 +98 21 7.800000000000000e+01 +98 22 7.700000000000000e+01 +98 23 7.600000000000000e+01 +98 24 7.500000000000000e+01 +98 25 7.400000000000000e+01 +98 26 7.300000000000000e+01 +98 27 7.200000000000000e+01 +98 28 7.100000000000000e+01 +98 29 7.000000000000000e+01 +98 30 6.900000000000000e+01 +98 31 6.800000000000000e+01 +98 32 6.700000000000000e+01 +98 33 6.600000000000000e+01 +98 34 6.500000000000000e+01 +98 35 6.400000000000000e+01 +98 36 6.300000000000000e+01 +98 37 6.200000000000000e+01 +98 38 6.100000000000000e+01 +98 39 6.000000000000000e+01 +98 40 5.900000000000000e+01 +98 41 5.800000000000000e+01 +98 42 5.700000000000000e+01 +98 43 5.600000000000000e+01 +98 44 5.500000000000000e+01 +98 45 5.400000000000000e+01 +98 46 5.300000000000000e+01 +98 47 5.200000000000000e+01 +98 48 5.100000000000000e+01 +98 49 5.000000000000000e+01 +98 50 4.900000000000000e+01 +98 51 4.800000000000000e+01 +98 52 4.700000000000000e+01 +98 53 4.600000000000000e+01 +98 54 4.500000000000000e+01 +98 55 4.400000000000000e+01 +98 56 4.300000000000000e+01 +98 57 4.200000000000000e+01 +98 58 4.100000000000000e+01 +98 59 4.000000000000000e+01 +98 60 3.900000000000000e+01 +98 61 3.800000000000000e+01 +98 62 3.700000000000000e+01 +98 63 3.600000000000000e+01 +98 64 3.500000000000000e+01 +98 65 3.400000000000000e+01 +98 66 3.300000000000000e+01 +98 67 3.200000000000000e+01 +98 68 3.100000000000000e+01 +98 69 3.000000000000000e+01 +98 70 2.900000000000000e+01 +98 71 2.800000000000000e+01 +98 72 2.700000000000000e+01 +98 73 2.600000000000000e+01 +98 74 2.500000000000000e+01 +98 75 2.400000000000000e+01 +98 76 2.300000000000000e+01 +98 77 2.200000000000000e+01 +98 78 2.100000000000000e+01 +98 79 2.000000000000000e+01 +98 80 1.900000000000000e+01 +98 81 1.800000000000000e+01 +98 82 1.700000000000000e+01 +98 83 1.600000000000000e+01 +98 84 1.500000000000000e+01 +98 85 1.400000000000000e+01 +98 86 1.300000000000000e+01 +98 87 1.200000000000000e+01 +98 88 1.100000000000000e+01 +98 89 1.000000000000000e+01 +98 90 9.000000000000000e+00 +98 91 8.000000000000000e+00 +98 92 7.000000000000000e+00 +98 93 6.000000000000000e+00 +98 94 5.000000000000000e+00 +98 95 4.000000000000000e+00 +98 96 3.000000000000000e+00 +98 97 2.000000000000000e+00 +98 98 2.000000000000000e+00 +98 99 2.000000000000000e+00 +98 100 3.000000000000000e+00 +99 1 9.900000000000000e+01 +99 2 9.800000000000000e+01 +99 3 9.700000000000000e+01 +99 4 9.600000000000000e+01 +99 5 9.500000000000000e+01 +99 6 9.400000000000000e+01 +99 7 9.300000000000000e+01 +99 8 9.200000000000000e+01 +99 9 9.100000000000000e+01 +99 10 9.000000000000000e+01 +99 11 8.900000000000000e+01 +99 12 8.800000000000000e+01 +99 13 8.700000000000000e+01 +99 14 8.600000000000000e+01 +99 15 8.500000000000000e+01 +99 16 8.400000000000000e+01 +99 17 8.300000000000000e+01 +99 18 8.200000000000000e+01 +99 19 8.100000000000000e+01 +99 20 8.000000000000000e+01 +99 21 7.900000000000000e+01 +99 22 7.800000000000000e+01 +99 23 7.700000000000000e+01 +99 24 7.600000000000000e+01 +99 25 7.500000000000000e+01 +99 26 7.400000000000000e+01 +99 27 7.300000000000000e+01 +99 28 7.200000000000000e+01 +99 29 7.100000000000000e+01 +99 30 7.000000000000000e+01 +99 31 6.900000000000000e+01 +99 32 6.800000000000000e+01 +99 33 6.700000000000000e+01 +99 34 6.600000000000000e+01 +99 35 6.500000000000000e+01 +99 36 6.400000000000000e+01 +99 37 6.300000000000000e+01 +99 38 6.200000000000000e+01 +99 39 6.100000000000000e+01 +99 40 6.000000000000000e+01 +99 41 5.900000000000000e+01 +99 42 5.800000000000000e+01 +99 43 5.700000000000000e+01 +99 44 5.600000000000000e+01 +99 45 5.500000000000000e+01 +99 46 5.400000000000000e+01 +99 47 5.300000000000000e+01 +99 48 5.200000000000000e+01 +99 49 5.100000000000000e+01 +99 50 5.000000000000000e+01 +99 51 4.900000000000000e+01 +99 52 4.800000000000000e+01 +99 53 4.700000000000000e+01 +99 54 4.600000000000000e+01 +99 55 4.500000000000000e+01 +99 56 4.400000000000000e+01 +99 57 4.300000000000000e+01 +99 58 4.200000000000000e+01 +99 59 4.100000000000000e+01 +99 60 4.000000000000000e+01 +99 61 3.900000000000000e+01 +99 62 3.800000000000000e+01 +99 63 3.700000000000000e+01 +99 64 3.600000000000000e+01 +99 65 3.500000000000000e+01 +99 66 3.400000000000000e+01 +99 67 3.300000000000000e+01 +99 68 3.200000000000000e+01 +99 69 3.100000000000000e+01 +99 70 3.000000000000000e+01 +99 71 2.900000000000000e+01 +99 72 2.800000000000000e+01 +99 73 2.700000000000000e+01 +99 74 2.600000000000000e+01 +99 75 2.500000000000000e+01 +99 76 2.400000000000000e+01 +99 77 2.300000000000000e+01 +99 78 2.200000000000000e+01 +99 79 2.100000000000000e+01 +99 80 2.000000000000000e+01 +99 81 1.900000000000000e+01 +99 82 1.800000000000000e+01 +99 83 1.700000000000000e+01 +99 84 1.600000000000000e+01 +99 85 1.500000000000000e+01 +99 86 1.400000000000000e+01 +99 87 1.300000000000000e+01 +99 88 1.200000000000000e+01 +99 89 1.100000000000000e+01 +99 90 1.000000000000000e+01 +99 91 9.000000000000000e+00 +99 92 8.000000000000000e+00 +99 93 7.000000000000000e+00 +99 94 6.000000000000000e+00 +99 95 5.000000000000000e+00 +99 96 4.000000000000000e+00 +99 97 3.000000000000000e+00 +99 98 2.000000000000000e+00 +99 99 2.000000000000000e+00 +99 100 2.000000000000000e+00 +100 1 1.000000000000000e+02 +100 2 9.900000000000000e+01 +100 3 9.800000000000000e+01 +100 4 9.700000000000000e+01 +100 5 9.600000000000000e+01 +100 6 9.500000000000000e+01 +100 7 9.400000000000000e+01 +100 8 9.300000000000000e+01 +100 9 9.200000000000000e+01 +100 10 9.100000000000000e+01 +100 11 9.000000000000000e+01 +100 12 8.900000000000000e+01 +100 13 8.800000000000000e+01 +100 14 8.700000000000000e+01 +100 15 8.600000000000000e+01 +100 16 8.500000000000000e+01 +100 17 8.400000000000000e+01 +100 18 8.300000000000000e+01 +100 19 8.200000000000000e+01 +100 20 8.100000000000000e+01 +100 21 8.000000000000000e+01 +100 22 7.900000000000000e+01 +100 23 7.800000000000000e+01 +100 24 7.700000000000000e+01 +100 25 7.600000000000000e+01 +100 26 7.500000000000000e+01 +100 27 7.400000000000000e+01 +100 28 7.300000000000000e+01 +100 29 7.200000000000000e+01 +100 30 7.100000000000000e+01 +100 31 7.000000000000000e+01 +100 32 6.900000000000000e+01 +100 33 6.800000000000000e+01 +100 34 6.700000000000000e+01 +100 35 6.600000000000000e+01 +100 36 6.500000000000000e+01 +100 37 6.400000000000000e+01 +100 38 6.300000000000000e+01 +100 39 6.200000000000000e+01 +100 40 6.100000000000000e+01 +100 41 6.000000000000000e+01 +100 42 5.900000000000000e+01 +100 43 5.800000000000000e+01 +100 44 5.700000000000000e+01 +100 45 5.600000000000000e+01 +100 46 5.500000000000000e+01 +100 47 5.400000000000000e+01 +100 48 5.300000000000000e+01 +100 49 5.200000000000000e+01 +100 50 5.100000000000000e+01 +100 51 5.000000000000000e+01 +100 52 4.900000000000000e+01 +100 53 4.800000000000000e+01 +100 54 4.700000000000000e+01 +100 55 4.600000000000000e+01 +100 56 4.500000000000000e+01 +100 57 4.400000000000000e+01 +100 58 4.300000000000000e+01 +100 59 4.200000000000000e+01 +100 60 4.100000000000000e+01 +100 61 4.000000000000000e+01 +100 62 3.900000000000000e+01 +100 63 3.800000000000000e+01 +100 64 3.700000000000000e+01 +100 65 3.600000000000000e+01 +100 66 3.500000000000000e+01 +100 67 3.400000000000000e+01 +100 68 3.300000000000000e+01 +100 69 3.200000000000000e+01 +100 70 3.100000000000000e+01 +100 71 3.000000000000000e+01 +100 72 2.900000000000000e+01 +100 73 2.800000000000000e+01 +100 74 2.700000000000000e+01 +100 75 2.600000000000000e+01 +100 76 2.500000000000000e+01 +100 77 2.400000000000000e+01 +100 78 2.300000000000000e+01 +100 79 2.200000000000000e+01 +100 80 2.100000000000000e+01 +100 81 2.000000000000000e+01 +100 82 1.900000000000000e+01 +100 83 1.800000000000000e+01 +100 84 1.700000000000000e+01 +100 85 1.600000000000000e+01 +100 86 1.500000000000000e+01 +100 87 1.400000000000000e+01 +100 88 1.300000000000000e+01 +100 89 1.200000000000000e+01 +100 90 1.100000000000000e+01 +100 91 1.000000000000000e+01 +100 92 9.000000000000000e+00 +100 93 8.000000000000000e+00 +100 94 7.000000000000000e+00 +100 95 6.000000000000000e+00 +100 96 5.000000000000000e+00 +100 97 4.000000000000000e+00 +100 98 3.000000000000000e+00 +100 99 2.000000000000000e+00 +100 100 2.000000000000000e+00 diff --git a/packages/tpetra/core/test/CrsGraphTransposer/L.mtx b/packages/tpetra/core/test/CrsGraphTransposer/L.mtx new file mode 100644 index 000000000000..ab7d4fe85c26 --- /dev/null +++ b/packages/tpetra/core/test/CrsGraphTransposer/L.mtx @@ -0,0 +1,5053 @@ +%%MatrixMarket matrix coordinate real general +% +100 100 5050 +1 1 1.000000000000000e+00 +2 1 2.000000000000000e+00 +2 2 1.000000000000000e+00 +3 1 3.000000000000000e+00 +3 2 2.000000000000000e+00 +3 3 1.000000000000000e+00 +4 1 4.000000000000000e+00 +4 2 3.000000000000000e+00 +4 3 2.000000000000000e+00 +4 4 1.000000000000000e+00 +5 1 5.000000000000000e+00 +5 2 4.000000000000000e+00 +5 3 3.000000000000000e+00 +5 4 2.000000000000000e+00 +5 5 1.000000000000000e+00 +6 1 6.000000000000000e+00 +6 2 5.000000000000000e+00 +6 3 4.000000000000000e+00 +6 4 3.000000000000000e+00 +6 5 2.000000000000000e+00 +6 6 1.000000000000000e+00 +7 1 7.000000000000000e+00 +7 2 6.000000000000000e+00 +7 3 5.000000000000000e+00 +7 4 4.000000000000000e+00 +7 5 3.000000000000000e+00 +7 6 2.000000000000000e+00 +7 7 1.000000000000000e+00 +8 1 8.000000000000000e+00 +8 2 7.000000000000000e+00 +8 3 6.000000000000000e+00 +8 4 5.000000000000000e+00 +8 5 4.000000000000000e+00 +8 6 3.000000000000000e+00 +8 7 2.000000000000000e+00 +8 8 1.000000000000000e+00 +9 1 9.000000000000000e+00 +9 2 8.000000000000000e+00 +9 3 7.000000000000000e+00 +9 4 6.000000000000000e+00 +9 5 5.000000000000000e+00 +9 6 4.000000000000000e+00 +9 7 3.000000000000000e+00 +9 8 2.000000000000000e+00 +9 9 1.000000000000000e+00 +10 1 1.000000000000000e+01 +10 2 9.000000000000000e+00 +10 3 8.000000000000000e+00 +10 4 7.000000000000000e+00 +10 5 6.000000000000000e+00 +10 6 5.000000000000000e+00 +10 7 4.000000000000000e+00 +10 8 3.000000000000000e+00 +10 9 2.000000000000000e+00 +10 10 1.000000000000000e+00 +11 1 1.100000000000000e+01 +11 2 1.000000000000000e+01 +11 3 9.000000000000000e+00 +11 4 8.000000000000000e+00 +11 5 7.000000000000000e+00 +11 6 6.000000000000000e+00 +11 7 5.000000000000000e+00 +11 8 4.000000000000000e+00 +11 9 3.000000000000000e+00 +11 10 2.000000000000000e+00 +11 11 1.000000000000000e+00 +12 1 1.200000000000000e+01 +12 2 1.100000000000000e+01 +12 3 1.000000000000000e+01 +12 4 9.000000000000000e+00 +12 5 8.000000000000000e+00 +12 6 7.000000000000000e+00 +12 7 6.000000000000000e+00 +12 8 5.000000000000000e+00 +12 9 4.000000000000000e+00 +12 10 3.000000000000000e+00 +12 11 2.000000000000000e+00 +12 12 1.000000000000000e+00 +13 1 1.300000000000000e+01 +13 2 1.200000000000000e+01 +13 3 1.100000000000000e+01 +13 4 1.000000000000000e+01 +13 5 9.000000000000000e+00 +13 6 8.000000000000000e+00 +13 7 7.000000000000000e+00 +13 8 6.000000000000000e+00 +13 9 5.000000000000000e+00 +13 10 4.000000000000000e+00 +13 11 3.000000000000000e+00 +13 12 2.000000000000000e+00 +13 13 1.000000000000000e+00 +14 1 1.400000000000000e+01 +14 2 1.300000000000000e+01 +14 3 1.200000000000000e+01 +14 4 1.100000000000000e+01 +14 5 1.000000000000000e+01 +14 6 9.000000000000000e+00 +14 7 8.000000000000000e+00 +14 8 7.000000000000000e+00 +14 9 6.000000000000000e+00 +14 10 5.000000000000000e+00 +14 11 4.000000000000000e+00 +14 12 3.000000000000000e+00 +14 13 2.000000000000000e+00 +14 14 1.000000000000000e+00 +15 1 1.500000000000000e+01 +15 2 1.400000000000000e+01 +15 3 1.300000000000000e+01 +15 4 1.200000000000000e+01 +15 5 1.100000000000000e+01 +15 6 1.000000000000000e+01 +15 7 9.000000000000000e+00 +15 8 8.000000000000000e+00 +15 9 7.000000000000000e+00 +15 10 6.000000000000000e+00 +15 11 5.000000000000000e+00 +15 12 4.000000000000000e+00 +15 13 3.000000000000000e+00 +15 14 2.000000000000000e+00 +15 15 1.000000000000000e+00 +16 1 1.600000000000000e+01 +16 2 1.500000000000000e+01 +16 3 1.400000000000000e+01 +16 4 1.300000000000000e+01 +16 5 1.200000000000000e+01 +16 6 1.100000000000000e+01 +16 7 1.000000000000000e+01 +16 8 9.000000000000000e+00 +16 9 8.000000000000000e+00 +16 10 7.000000000000000e+00 +16 11 6.000000000000000e+00 +16 12 5.000000000000000e+00 +16 13 4.000000000000000e+00 +16 14 3.000000000000000e+00 +16 15 2.000000000000000e+00 +16 16 1.000000000000000e+00 +17 1 1.700000000000000e+01 +17 2 1.600000000000000e+01 +17 3 1.500000000000000e+01 +17 4 1.400000000000000e+01 +17 5 1.300000000000000e+01 +17 6 1.200000000000000e+01 +17 7 1.100000000000000e+01 +17 8 1.000000000000000e+01 +17 9 9.000000000000000e+00 +17 10 8.000000000000000e+00 +17 11 7.000000000000000e+00 +17 12 6.000000000000000e+00 +17 13 5.000000000000000e+00 +17 14 4.000000000000000e+00 +17 15 3.000000000000000e+00 +17 16 2.000000000000000e+00 +17 17 1.000000000000000e+00 +18 1 1.800000000000000e+01 +18 2 1.700000000000000e+01 +18 3 1.600000000000000e+01 +18 4 1.500000000000000e+01 +18 5 1.400000000000000e+01 +18 6 1.300000000000000e+01 +18 7 1.200000000000000e+01 +18 8 1.100000000000000e+01 +18 9 1.000000000000000e+01 +18 10 9.000000000000000e+00 +18 11 8.000000000000000e+00 +18 12 7.000000000000000e+00 +18 13 6.000000000000000e+00 +18 14 5.000000000000000e+00 +18 15 4.000000000000000e+00 +18 16 3.000000000000000e+00 +18 17 2.000000000000000e+00 +18 18 1.000000000000000e+00 +19 1 1.900000000000000e+01 +19 2 1.800000000000000e+01 +19 3 1.700000000000000e+01 +19 4 1.600000000000000e+01 +19 5 1.500000000000000e+01 +19 6 1.400000000000000e+01 +19 7 1.300000000000000e+01 +19 8 1.200000000000000e+01 +19 9 1.100000000000000e+01 +19 10 1.000000000000000e+01 +19 11 9.000000000000000e+00 +19 12 8.000000000000000e+00 +19 13 7.000000000000000e+00 +19 14 6.000000000000000e+00 +19 15 5.000000000000000e+00 +19 16 4.000000000000000e+00 +19 17 3.000000000000000e+00 +19 18 2.000000000000000e+00 +19 19 1.000000000000000e+00 +20 1 2.000000000000000e+01 +20 2 1.900000000000000e+01 +20 3 1.800000000000000e+01 +20 4 1.700000000000000e+01 +20 5 1.600000000000000e+01 +20 6 1.500000000000000e+01 +20 7 1.400000000000000e+01 +20 8 1.300000000000000e+01 +20 9 1.200000000000000e+01 +20 10 1.100000000000000e+01 +20 11 1.000000000000000e+01 +20 12 9.000000000000000e+00 +20 13 8.000000000000000e+00 +20 14 7.000000000000000e+00 +20 15 6.000000000000000e+00 +20 16 5.000000000000000e+00 +20 17 4.000000000000000e+00 +20 18 3.000000000000000e+00 +20 19 2.000000000000000e+00 +20 20 1.000000000000000e+00 +21 1 2.100000000000000e+01 +21 2 2.000000000000000e+01 +21 3 1.900000000000000e+01 +21 4 1.800000000000000e+01 +21 5 1.700000000000000e+01 +21 6 1.600000000000000e+01 +21 7 1.500000000000000e+01 +21 8 1.400000000000000e+01 +21 9 1.300000000000000e+01 +21 10 1.200000000000000e+01 +21 11 1.100000000000000e+01 +21 12 1.000000000000000e+01 +21 13 9.000000000000000e+00 +21 14 8.000000000000000e+00 +21 15 7.000000000000000e+00 +21 16 6.000000000000000e+00 +21 17 5.000000000000000e+00 +21 18 4.000000000000000e+00 +21 19 3.000000000000000e+00 +21 20 2.000000000000000e+00 +21 21 1.000000000000000e+00 +22 1 2.200000000000000e+01 +22 2 2.100000000000000e+01 +22 3 2.000000000000000e+01 +22 4 1.900000000000000e+01 +22 5 1.800000000000000e+01 +22 6 1.700000000000000e+01 +22 7 1.600000000000000e+01 +22 8 1.500000000000000e+01 +22 9 1.400000000000000e+01 +22 10 1.300000000000000e+01 +22 11 1.200000000000000e+01 +22 12 1.100000000000000e+01 +22 13 1.000000000000000e+01 +22 14 9.000000000000000e+00 +22 15 8.000000000000000e+00 +22 16 7.000000000000000e+00 +22 17 6.000000000000000e+00 +22 18 5.000000000000000e+00 +22 19 4.000000000000000e+00 +22 20 3.000000000000000e+00 +22 21 2.000000000000000e+00 +22 22 1.000000000000000e+00 +23 1 2.300000000000000e+01 +23 2 2.200000000000000e+01 +23 3 2.100000000000000e+01 +23 4 2.000000000000000e+01 +23 5 1.900000000000000e+01 +23 6 1.800000000000000e+01 +23 7 1.700000000000000e+01 +23 8 1.600000000000000e+01 +23 9 1.500000000000000e+01 +23 10 1.400000000000000e+01 +23 11 1.300000000000000e+01 +23 12 1.200000000000000e+01 +23 13 1.100000000000000e+01 +23 14 1.000000000000000e+01 +23 15 9.000000000000000e+00 +23 16 8.000000000000000e+00 +23 17 7.000000000000000e+00 +23 18 6.000000000000000e+00 +23 19 5.000000000000000e+00 +23 20 4.000000000000000e+00 +23 21 3.000000000000000e+00 +23 22 2.000000000000000e+00 +23 23 1.000000000000000e+00 +24 1 2.400000000000000e+01 +24 2 2.300000000000000e+01 +24 3 2.200000000000000e+01 +24 4 2.100000000000000e+01 +24 5 2.000000000000000e+01 +24 6 1.900000000000000e+01 +24 7 1.800000000000000e+01 +24 8 1.700000000000000e+01 +24 9 1.600000000000000e+01 +24 10 1.500000000000000e+01 +24 11 1.400000000000000e+01 +24 12 1.300000000000000e+01 +24 13 1.200000000000000e+01 +24 14 1.100000000000000e+01 +24 15 1.000000000000000e+01 +24 16 9.000000000000000e+00 +24 17 8.000000000000000e+00 +24 18 7.000000000000000e+00 +24 19 6.000000000000000e+00 +24 20 5.000000000000000e+00 +24 21 4.000000000000000e+00 +24 22 3.000000000000000e+00 +24 23 2.000000000000000e+00 +24 24 1.000000000000000e+00 +25 1 2.500000000000000e+01 +25 2 2.400000000000000e+01 +25 3 2.300000000000000e+01 +25 4 2.200000000000000e+01 +25 5 2.100000000000000e+01 +25 6 2.000000000000000e+01 +25 7 1.900000000000000e+01 +25 8 1.800000000000000e+01 +25 9 1.700000000000000e+01 +25 10 1.600000000000000e+01 +25 11 1.500000000000000e+01 +25 12 1.400000000000000e+01 +25 13 1.300000000000000e+01 +25 14 1.200000000000000e+01 +25 15 1.100000000000000e+01 +25 16 1.000000000000000e+01 +25 17 9.000000000000000e+00 +25 18 8.000000000000000e+00 +25 19 7.000000000000000e+00 +25 20 6.000000000000000e+00 +25 21 5.000000000000000e+00 +25 22 4.000000000000000e+00 +25 23 3.000000000000000e+00 +25 24 2.000000000000000e+00 +25 25 1.000000000000000e+00 +26 1 2.600000000000000e+01 +26 2 2.500000000000000e+01 +26 3 2.400000000000000e+01 +26 4 2.300000000000000e+01 +26 5 2.200000000000000e+01 +26 6 2.100000000000000e+01 +26 7 2.000000000000000e+01 +26 8 1.900000000000000e+01 +26 9 1.800000000000000e+01 +26 10 1.700000000000000e+01 +26 11 1.600000000000000e+01 +26 12 1.500000000000000e+01 +26 13 1.400000000000000e+01 +26 14 1.300000000000000e+01 +26 15 1.200000000000000e+01 +26 16 1.100000000000000e+01 +26 17 1.000000000000000e+01 +26 18 9.000000000000000e+00 +26 19 8.000000000000000e+00 +26 20 7.000000000000000e+00 +26 21 6.000000000000000e+00 +26 22 5.000000000000000e+00 +26 23 4.000000000000000e+00 +26 24 3.000000000000000e+00 +26 25 2.000000000000000e+00 +26 26 1.000000000000000e+00 +27 1 2.700000000000000e+01 +27 2 2.600000000000000e+01 +27 3 2.500000000000000e+01 +27 4 2.400000000000000e+01 +27 5 2.300000000000000e+01 +27 6 2.200000000000000e+01 +27 7 2.100000000000000e+01 +27 8 2.000000000000000e+01 +27 9 1.900000000000000e+01 +27 10 1.800000000000000e+01 +27 11 1.700000000000000e+01 +27 12 1.600000000000000e+01 +27 13 1.500000000000000e+01 +27 14 1.400000000000000e+01 +27 15 1.300000000000000e+01 +27 16 1.200000000000000e+01 +27 17 1.100000000000000e+01 +27 18 1.000000000000000e+01 +27 19 9.000000000000000e+00 +27 20 8.000000000000000e+00 +27 21 7.000000000000000e+00 +27 22 6.000000000000000e+00 +27 23 5.000000000000000e+00 +27 24 4.000000000000000e+00 +27 25 3.000000000000000e+00 +27 26 2.000000000000000e+00 +27 27 1.000000000000000e+00 +28 1 2.800000000000000e+01 +28 2 2.700000000000000e+01 +28 3 2.600000000000000e+01 +28 4 2.500000000000000e+01 +28 5 2.400000000000000e+01 +28 6 2.300000000000000e+01 +28 7 2.200000000000000e+01 +28 8 2.100000000000000e+01 +28 9 2.000000000000000e+01 +28 10 1.900000000000000e+01 +28 11 1.800000000000000e+01 +28 12 1.700000000000000e+01 +28 13 1.600000000000000e+01 +28 14 1.500000000000000e+01 +28 15 1.400000000000000e+01 +28 16 1.300000000000000e+01 +28 17 1.200000000000000e+01 +28 18 1.100000000000000e+01 +28 19 1.000000000000000e+01 +28 20 9.000000000000000e+00 +28 21 8.000000000000000e+00 +28 22 7.000000000000000e+00 +28 23 6.000000000000000e+00 +28 24 5.000000000000000e+00 +28 25 4.000000000000000e+00 +28 26 3.000000000000000e+00 +28 27 2.000000000000000e+00 +28 28 1.000000000000000e+00 +29 1 2.900000000000000e+01 +29 2 2.800000000000000e+01 +29 3 2.700000000000000e+01 +29 4 2.600000000000000e+01 +29 5 2.500000000000000e+01 +29 6 2.400000000000000e+01 +29 7 2.300000000000000e+01 +29 8 2.200000000000000e+01 +29 9 2.100000000000000e+01 +29 10 2.000000000000000e+01 +29 11 1.900000000000000e+01 +29 12 1.800000000000000e+01 +29 13 1.700000000000000e+01 +29 14 1.600000000000000e+01 +29 15 1.500000000000000e+01 +29 16 1.400000000000000e+01 +29 17 1.300000000000000e+01 +29 18 1.200000000000000e+01 +29 19 1.100000000000000e+01 +29 20 1.000000000000000e+01 +29 21 9.000000000000000e+00 +29 22 8.000000000000000e+00 +29 23 7.000000000000000e+00 +29 24 6.000000000000000e+00 +29 25 5.000000000000000e+00 +29 26 4.000000000000000e+00 +29 27 3.000000000000000e+00 +29 28 2.000000000000000e+00 +29 29 1.000000000000000e+00 +30 1 3.000000000000000e+01 +30 2 2.900000000000000e+01 +30 3 2.800000000000000e+01 +30 4 2.700000000000000e+01 +30 5 2.600000000000000e+01 +30 6 2.500000000000000e+01 +30 7 2.400000000000000e+01 +30 8 2.300000000000000e+01 +30 9 2.200000000000000e+01 +30 10 2.100000000000000e+01 +30 11 2.000000000000000e+01 +30 12 1.900000000000000e+01 +30 13 1.800000000000000e+01 +30 14 1.700000000000000e+01 +30 15 1.600000000000000e+01 +30 16 1.500000000000000e+01 +30 17 1.400000000000000e+01 +30 18 1.300000000000000e+01 +30 19 1.200000000000000e+01 +30 20 1.100000000000000e+01 +30 21 1.000000000000000e+01 +30 22 9.000000000000000e+00 +30 23 8.000000000000000e+00 +30 24 7.000000000000000e+00 +30 25 6.000000000000000e+00 +30 26 5.000000000000000e+00 +30 27 4.000000000000000e+00 +30 28 3.000000000000000e+00 +30 29 2.000000000000000e+00 +30 30 1.000000000000000e+00 +31 1 3.100000000000000e+01 +31 2 3.000000000000000e+01 +31 3 2.900000000000000e+01 +31 4 2.800000000000000e+01 +31 5 2.700000000000000e+01 +31 6 2.600000000000000e+01 +31 7 2.500000000000000e+01 +31 8 2.400000000000000e+01 +31 9 2.300000000000000e+01 +31 10 2.200000000000000e+01 +31 11 2.100000000000000e+01 +31 12 2.000000000000000e+01 +31 13 1.900000000000000e+01 +31 14 1.800000000000000e+01 +31 15 1.700000000000000e+01 +31 16 1.600000000000000e+01 +31 17 1.500000000000000e+01 +31 18 1.400000000000000e+01 +31 19 1.300000000000000e+01 +31 20 1.200000000000000e+01 +31 21 1.100000000000000e+01 +31 22 1.000000000000000e+01 +31 23 9.000000000000000e+00 +31 24 8.000000000000000e+00 +31 25 7.000000000000000e+00 +31 26 6.000000000000000e+00 +31 27 5.000000000000000e+00 +31 28 4.000000000000000e+00 +31 29 3.000000000000000e+00 +31 30 2.000000000000000e+00 +31 31 1.000000000000000e+00 +32 1 3.200000000000000e+01 +32 2 3.100000000000000e+01 +32 3 3.000000000000000e+01 +32 4 2.900000000000000e+01 +32 5 2.800000000000000e+01 +32 6 2.700000000000000e+01 +32 7 2.600000000000000e+01 +32 8 2.500000000000000e+01 +32 9 2.400000000000000e+01 +32 10 2.300000000000000e+01 +32 11 2.200000000000000e+01 +32 12 2.100000000000000e+01 +32 13 2.000000000000000e+01 +32 14 1.900000000000000e+01 +32 15 1.800000000000000e+01 +32 16 1.700000000000000e+01 +32 17 1.600000000000000e+01 +32 18 1.500000000000000e+01 +32 19 1.400000000000000e+01 +32 20 1.300000000000000e+01 +32 21 1.200000000000000e+01 +32 22 1.100000000000000e+01 +32 23 1.000000000000000e+01 +32 24 9.000000000000000e+00 +32 25 8.000000000000000e+00 +32 26 7.000000000000000e+00 +32 27 6.000000000000000e+00 +32 28 5.000000000000000e+00 +32 29 4.000000000000000e+00 +32 30 3.000000000000000e+00 +32 31 2.000000000000000e+00 +32 32 1.000000000000000e+00 +33 1 3.300000000000000e+01 +33 2 3.200000000000000e+01 +33 3 3.100000000000000e+01 +33 4 3.000000000000000e+01 +33 5 2.900000000000000e+01 +33 6 2.800000000000000e+01 +33 7 2.700000000000000e+01 +33 8 2.600000000000000e+01 +33 9 2.500000000000000e+01 +33 10 2.400000000000000e+01 +33 11 2.300000000000000e+01 +33 12 2.200000000000000e+01 +33 13 2.100000000000000e+01 +33 14 2.000000000000000e+01 +33 15 1.900000000000000e+01 +33 16 1.800000000000000e+01 +33 17 1.700000000000000e+01 +33 18 1.600000000000000e+01 +33 19 1.500000000000000e+01 +33 20 1.400000000000000e+01 +33 21 1.300000000000000e+01 +33 22 1.200000000000000e+01 +33 23 1.100000000000000e+01 +33 24 1.000000000000000e+01 +33 25 9.000000000000000e+00 +33 26 8.000000000000000e+00 +33 27 7.000000000000000e+00 +33 28 6.000000000000000e+00 +33 29 5.000000000000000e+00 +33 30 4.000000000000000e+00 +33 31 3.000000000000000e+00 +33 32 2.000000000000000e+00 +33 33 1.000000000000000e+00 +34 1 3.400000000000000e+01 +34 2 3.300000000000000e+01 +34 3 3.200000000000000e+01 +34 4 3.100000000000000e+01 +34 5 3.000000000000000e+01 +34 6 2.900000000000000e+01 +34 7 2.800000000000000e+01 +34 8 2.700000000000000e+01 +34 9 2.600000000000000e+01 +34 10 2.500000000000000e+01 +34 11 2.400000000000000e+01 +34 12 2.300000000000000e+01 +34 13 2.200000000000000e+01 +34 14 2.100000000000000e+01 +34 15 2.000000000000000e+01 +34 16 1.900000000000000e+01 +34 17 1.800000000000000e+01 +34 18 1.700000000000000e+01 +34 19 1.600000000000000e+01 +34 20 1.500000000000000e+01 +34 21 1.400000000000000e+01 +34 22 1.300000000000000e+01 +34 23 1.200000000000000e+01 +34 24 1.100000000000000e+01 +34 25 1.000000000000000e+01 +34 26 9.000000000000000e+00 +34 27 8.000000000000000e+00 +34 28 7.000000000000000e+00 +34 29 6.000000000000000e+00 +34 30 5.000000000000000e+00 +34 31 4.000000000000000e+00 +34 32 3.000000000000000e+00 +34 33 2.000000000000000e+00 +34 34 1.000000000000000e+00 +35 1 3.500000000000000e+01 +35 2 3.400000000000000e+01 +35 3 3.300000000000000e+01 +35 4 3.200000000000000e+01 +35 5 3.100000000000000e+01 +35 6 3.000000000000000e+01 +35 7 2.900000000000000e+01 +35 8 2.800000000000000e+01 +35 9 2.700000000000000e+01 +35 10 2.600000000000000e+01 +35 11 2.500000000000000e+01 +35 12 2.400000000000000e+01 +35 13 2.300000000000000e+01 +35 14 2.200000000000000e+01 +35 15 2.100000000000000e+01 +35 16 2.000000000000000e+01 +35 17 1.900000000000000e+01 +35 18 1.800000000000000e+01 +35 19 1.700000000000000e+01 +35 20 1.600000000000000e+01 +35 21 1.500000000000000e+01 +35 22 1.400000000000000e+01 +35 23 1.300000000000000e+01 +35 24 1.200000000000000e+01 +35 25 1.100000000000000e+01 +35 26 1.000000000000000e+01 +35 27 9.000000000000000e+00 +35 28 8.000000000000000e+00 +35 29 7.000000000000000e+00 +35 30 6.000000000000000e+00 +35 31 5.000000000000000e+00 +35 32 4.000000000000000e+00 +35 33 3.000000000000000e+00 +35 34 2.000000000000000e+00 +35 35 1.000000000000000e+00 +36 1 3.600000000000000e+01 +36 2 3.500000000000000e+01 +36 3 3.400000000000000e+01 +36 4 3.300000000000000e+01 +36 5 3.200000000000000e+01 +36 6 3.100000000000000e+01 +36 7 3.000000000000000e+01 +36 8 2.900000000000000e+01 +36 9 2.800000000000000e+01 +36 10 2.700000000000000e+01 +36 11 2.600000000000000e+01 +36 12 2.500000000000000e+01 +36 13 2.400000000000000e+01 +36 14 2.300000000000000e+01 +36 15 2.200000000000000e+01 +36 16 2.100000000000000e+01 +36 17 2.000000000000000e+01 +36 18 1.900000000000000e+01 +36 19 1.800000000000000e+01 +36 20 1.700000000000000e+01 +36 21 1.600000000000000e+01 +36 22 1.500000000000000e+01 +36 23 1.400000000000000e+01 +36 24 1.300000000000000e+01 +36 25 1.200000000000000e+01 +36 26 1.100000000000000e+01 +36 27 1.000000000000000e+01 +36 28 9.000000000000000e+00 +36 29 8.000000000000000e+00 +36 30 7.000000000000000e+00 +36 31 6.000000000000000e+00 +36 32 5.000000000000000e+00 +36 33 4.000000000000000e+00 +36 34 3.000000000000000e+00 +36 35 2.000000000000000e+00 +36 36 1.000000000000000e+00 +37 1 3.700000000000000e+01 +37 2 3.600000000000000e+01 +37 3 3.500000000000000e+01 +37 4 3.400000000000000e+01 +37 5 3.300000000000000e+01 +37 6 3.200000000000000e+01 +37 7 3.100000000000000e+01 +37 8 3.000000000000000e+01 +37 9 2.900000000000000e+01 +37 10 2.800000000000000e+01 +37 11 2.700000000000000e+01 +37 12 2.600000000000000e+01 +37 13 2.500000000000000e+01 +37 14 2.400000000000000e+01 +37 15 2.300000000000000e+01 +37 16 2.200000000000000e+01 +37 17 2.100000000000000e+01 +37 18 2.000000000000000e+01 +37 19 1.900000000000000e+01 +37 20 1.800000000000000e+01 +37 21 1.700000000000000e+01 +37 22 1.600000000000000e+01 +37 23 1.500000000000000e+01 +37 24 1.400000000000000e+01 +37 25 1.300000000000000e+01 +37 26 1.200000000000000e+01 +37 27 1.100000000000000e+01 +37 28 1.000000000000000e+01 +37 29 9.000000000000000e+00 +37 30 8.000000000000000e+00 +37 31 7.000000000000000e+00 +37 32 6.000000000000000e+00 +37 33 5.000000000000000e+00 +37 34 4.000000000000000e+00 +37 35 3.000000000000000e+00 +37 36 2.000000000000000e+00 +37 37 1.000000000000000e+00 +38 1 3.800000000000000e+01 +38 2 3.700000000000000e+01 +38 3 3.600000000000000e+01 +38 4 3.500000000000000e+01 +38 5 3.400000000000000e+01 +38 6 3.300000000000000e+01 +38 7 3.200000000000000e+01 +38 8 3.100000000000000e+01 +38 9 3.000000000000000e+01 +38 10 2.900000000000000e+01 +38 11 2.800000000000000e+01 +38 12 2.700000000000000e+01 +38 13 2.600000000000000e+01 +38 14 2.500000000000000e+01 +38 15 2.400000000000000e+01 +38 16 2.300000000000000e+01 +38 17 2.200000000000000e+01 +38 18 2.100000000000000e+01 +38 19 2.000000000000000e+01 +38 20 1.900000000000000e+01 +38 21 1.800000000000000e+01 +38 22 1.700000000000000e+01 +38 23 1.600000000000000e+01 +38 24 1.500000000000000e+01 +38 25 1.400000000000000e+01 +38 26 1.300000000000000e+01 +38 27 1.200000000000000e+01 +38 28 1.100000000000000e+01 +38 29 1.000000000000000e+01 +38 30 9.000000000000000e+00 +38 31 8.000000000000000e+00 +38 32 7.000000000000000e+00 +38 33 6.000000000000000e+00 +38 34 5.000000000000000e+00 +38 35 4.000000000000000e+00 +38 36 3.000000000000000e+00 +38 37 2.000000000000000e+00 +38 38 1.000000000000000e+00 +39 1 3.900000000000000e+01 +39 2 3.800000000000000e+01 +39 3 3.700000000000000e+01 +39 4 3.600000000000000e+01 +39 5 3.500000000000000e+01 +39 6 3.400000000000000e+01 +39 7 3.300000000000000e+01 +39 8 3.200000000000000e+01 +39 9 3.100000000000000e+01 +39 10 3.000000000000000e+01 +39 11 2.900000000000000e+01 +39 12 2.800000000000000e+01 +39 13 2.700000000000000e+01 +39 14 2.600000000000000e+01 +39 15 2.500000000000000e+01 +39 16 2.400000000000000e+01 +39 17 2.300000000000000e+01 +39 18 2.200000000000000e+01 +39 19 2.100000000000000e+01 +39 20 2.000000000000000e+01 +39 21 1.900000000000000e+01 +39 22 1.800000000000000e+01 +39 23 1.700000000000000e+01 +39 24 1.600000000000000e+01 +39 25 1.500000000000000e+01 +39 26 1.400000000000000e+01 +39 27 1.300000000000000e+01 +39 28 1.200000000000000e+01 +39 29 1.100000000000000e+01 +39 30 1.000000000000000e+01 +39 31 9.000000000000000e+00 +39 32 8.000000000000000e+00 +39 33 7.000000000000000e+00 +39 34 6.000000000000000e+00 +39 35 5.000000000000000e+00 +39 36 4.000000000000000e+00 +39 37 3.000000000000000e+00 +39 38 2.000000000000000e+00 +39 39 1.000000000000000e+00 +40 1 4.000000000000000e+01 +40 2 3.900000000000000e+01 +40 3 3.800000000000000e+01 +40 4 3.700000000000000e+01 +40 5 3.600000000000000e+01 +40 6 3.500000000000000e+01 +40 7 3.400000000000000e+01 +40 8 3.300000000000000e+01 +40 9 3.200000000000000e+01 +40 10 3.100000000000000e+01 +40 11 3.000000000000000e+01 +40 12 2.900000000000000e+01 +40 13 2.800000000000000e+01 +40 14 2.700000000000000e+01 +40 15 2.600000000000000e+01 +40 16 2.500000000000000e+01 +40 17 2.400000000000000e+01 +40 18 2.300000000000000e+01 +40 19 2.200000000000000e+01 +40 20 2.100000000000000e+01 +40 21 2.000000000000000e+01 +40 22 1.900000000000000e+01 +40 23 1.800000000000000e+01 +40 24 1.700000000000000e+01 +40 25 1.600000000000000e+01 +40 26 1.500000000000000e+01 +40 27 1.400000000000000e+01 +40 28 1.300000000000000e+01 +40 29 1.200000000000000e+01 +40 30 1.100000000000000e+01 +40 31 1.000000000000000e+01 +40 32 9.000000000000000e+00 +40 33 8.000000000000000e+00 +40 34 7.000000000000000e+00 +40 35 6.000000000000000e+00 +40 36 5.000000000000000e+00 +40 37 4.000000000000000e+00 +40 38 3.000000000000000e+00 +40 39 2.000000000000000e+00 +40 40 1.000000000000000e+00 +41 1 4.100000000000000e+01 +41 2 4.000000000000000e+01 +41 3 3.900000000000000e+01 +41 4 3.800000000000000e+01 +41 5 3.700000000000000e+01 +41 6 3.600000000000000e+01 +41 7 3.500000000000000e+01 +41 8 3.400000000000000e+01 +41 9 3.300000000000000e+01 +41 10 3.200000000000000e+01 +41 11 3.100000000000000e+01 +41 12 3.000000000000000e+01 +41 13 2.900000000000000e+01 +41 14 2.800000000000000e+01 +41 15 2.700000000000000e+01 +41 16 2.600000000000000e+01 +41 17 2.500000000000000e+01 +41 18 2.400000000000000e+01 +41 19 2.300000000000000e+01 +41 20 2.200000000000000e+01 +41 21 2.100000000000000e+01 +41 22 2.000000000000000e+01 +41 23 1.900000000000000e+01 +41 24 1.800000000000000e+01 +41 25 1.700000000000000e+01 +41 26 1.600000000000000e+01 +41 27 1.500000000000000e+01 +41 28 1.400000000000000e+01 +41 29 1.300000000000000e+01 +41 30 1.200000000000000e+01 +41 31 1.100000000000000e+01 +41 32 1.000000000000000e+01 +41 33 9.000000000000000e+00 +41 34 8.000000000000000e+00 +41 35 7.000000000000000e+00 +41 36 6.000000000000000e+00 +41 37 5.000000000000000e+00 +41 38 4.000000000000000e+00 +41 39 3.000000000000000e+00 +41 40 2.000000000000000e+00 +41 41 1.000000000000000e+00 +42 1 4.200000000000000e+01 +42 2 4.100000000000000e+01 +42 3 4.000000000000000e+01 +42 4 3.900000000000000e+01 +42 5 3.800000000000000e+01 +42 6 3.700000000000000e+01 +42 7 3.600000000000000e+01 +42 8 3.500000000000000e+01 +42 9 3.400000000000000e+01 +42 10 3.300000000000000e+01 +42 11 3.200000000000000e+01 +42 12 3.100000000000000e+01 +42 13 3.000000000000000e+01 +42 14 2.900000000000000e+01 +42 15 2.800000000000000e+01 +42 16 2.700000000000000e+01 +42 17 2.600000000000000e+01 +42 18 2.500000000000000e+01 +42 19 2.400000000000000e+01 +42 20 2.300000000000000e+01 +42 21 2.200000000000000e+01 +42 22 2.100000000000000e+01 +42 23 2.000000000000000e+01 +42 24 1.900000000000000e+01 +42 25 1.800000000000000e+01 +42 26 1.700000000000000e+01 +42 27 1.600000000000000e+01 +42 28 1.500000000000000e+01 +42 29 1.400000000000000e+01 +42 30 1.300000000000000e+01 +42 31 1.200000000000000e+01 +42 32 1.100000000000000e+01 +42 33 1.000000000000000e+01 +42 34 9.000000000000000e+00 +42 35 8.000000000000000e+00 +42 36 7.000000000000000e+00 +42 37 6.000000000000000e+00 +42 38 5.000000000000000e+00 +42 39 4.000000000000000e+00 +42 40 3.000000000000000e+00 +42 41 2.000000000000000e+00 +42 42 1.000000000000000e+00 +43 1 4.300000000000000e+01 +43 2 4.200000000000000e+01 +43 3 4.100000000000000e+01 +43 4 4.000000000000000e+01 +43 5 3.900000000000000e+01 +43 6 3.800000000000000e+01 +43 7 3.700000000000000e+01 +43 8 3.600000000000000e+01 +43 9 3.500000000000000e+01 +43 10 3.400000000000000e+01 +43 11 3.300000000000000e+01 +43 12 3.200000000000000e+01 +43 13 3.100000000000000e+01 +43 14 3.000000000000000e+01 +43 15 2.900000000000000e+01 +43 16 2.800000000000000e+01 +43 17 2.700000000000000e+01 +43 18 2.600000000000000e+01 +43 19 2.500000000000000e+01 +43 20 2.400000000000000e+01 +43 21 2.300000000000000e+01 +43 22 2.200000000000000e+01 +43 23 2.100000000000000e+01 +43 24 2.000000000000000e+01 +43 25 1.900000000000000e+01 +43 26 1.800000000000000e+01 +43 27 1.700000000000000e+01 +43 28 1.600000000000000e+01 +43 29 1.500000000000000e+01 +43 30 1.400000000000000e+01 +43 31 1.300000000000000e+01 +43 32 1.200000000000000e+01 +43 33 1.100000000000000e+01 +43 34 1.000000000000000e+01 +43 35 9.000000000000000e+00 +43 36 8.000000000000000e+00 +43 37 7.000000000000000e+00 +43 38 6.000000000000000e+00 +43 39 5.000000000000000e+00 +43 40 4.000000000000000e+00 +43 41 3.000000000000000e+00 +43 42 2.000000000000000e+00 +43 43 1.000000000000000e+00 +44 1 4.400000000000000e+01 +44 2 4.300000000000000e+01 +44 3 4.200000000000000e+01 +44 4 4.100000000000000e+01 +44 5 4.000000000000000e+01 +44 6 3.900000000000000e+01 +44 7 3.800000000000000e+01 +44 8 3.700000000000000e+01 +44 9 3.600000000000000e+01 +44 10 3.500000000000000e+01 +44 11 3.400000000000000e+01 +44 12 3.300000000000000e+01 +44 13 3.200000000000000e+01 +44 14 3.100000000000000e+01 +44 15 3.000000000000000e+01 +44 16 2.900000000000000e+01 +44 17 2.800000000000000e+01 +44 18 2.700000000000000e+01 +44 19 2.600000000000000e+01 +44 20 2.500000000000000e+01 +44 21 2.400000000000000e+01 +44 22 2.300000000000000e+01 +44 23 2.200000000000000e+01 +44 24 2.100000000000000e+01 +44 25 2.000000000000000e+01 +44 26 1.900000000000000e+01 +44 27 1.800000000000000e+01 +44 28 1.700000000000000e+01 +44 29 1.600000000000000e+01 +44 30 1.500000000000000e+01 +44 31 1.400000000000000e+01 +44 32 1.300000000000000e+01 +44 33 1.200000000000000e+01 +44 34 1.100000000000000e+01 +44 35 1.000000000000000e+01 +44 36 9.000000000000000e+00 +44 37 8.000000000000000e+00 +44 38 7.000000000000000e+00 +44 39 6.000000000000000e+00 +44 40 5.000000000000000e+00 +44 41 4.000000000000000e+00 +44 42 3.000000000000000e+00 +44 43 2.000000000000000e+00 +44 44 1.000000000000000e+00 +45 1 4.500000000000000e+01 +45 2 4.400000000000000e+01 +45 3 4.300000000000000e+01 +45 4 4.200000000000000e+01 +45 5 4.100000000000000e+01 +45 6 4.000000000000000e+01 +45 7 3.900000000000000e+01 +45 8 3.800000000000000e+01 +45 9 3.700000000000000e+01 +45 10 3.600000000000000e+01 +45 11 3.500000000000000e+01 +45 12 3.400000000000000e+01 +45 13 3.300000000000000e+01 +45 14 3.200000000000000e+01 +45 15 3.100000000000000e+01 +45 16 3.000000000000000e+01 +45 17 2.900000000000000e+01 +45 18 2.800000000000000e+01 +45 19 2.700000000000000e+01 +45 20 2.600000000000000e+01 +45 21 2.500000000000000e+01 +45 22 2.400000000000000e+01 +45 23 2.300000000000000e+01 +45 24 2.200000000000000e+01 +45 25 2.100000000000000e+01 +45 26 2.000000000000000e+01 +45 27 1.900000000000000e+01 +45 28 1.800000000000000e+01 +45 29 1.700000000000000e+01 +45 30 1.600000000000000e+01 +45 31 1.500000000000000e+01 +45 32 1.400000000000000e+01 +45 33 1.300000000000000e+01 +45 34 1.200000000000000e+01 +45 35 1.100000000000000e+01 +45 36 1.000000000000000e+01 +45 37 9.000000000000000e+00 +45 38 8.000000000000000e+00 +45 39 7.000000000000000e+00 +45 40 6.000000000000000e+00 +45 41 5.000000000000000e+00 +45 42 4.000000000000000e+00 +45 43 3.000000000000000e+00 +45 44 2.000000000000000e+00 +45 45 1.000000000000000e+00 +46 1 4.600000000000000e+01 +46 2 4.500000000000000e+01 +46 3 4.400000000000000e+01 +46 4 4.300000000000000e+01 +46 5 4.200000000000000e+01 +46 6 4.100000000000000e+01 +46 7 4.000000000000000e+01 +46 8 3.900000000000000e+01 +46 9 3.800000000000000e+01 +46 10 3.700000000000000e+01 +46 11 3.600000000000000e+01 +46 12 3.500000000000000e+01 +46 13 3.400000000000000e+01 +46 14 3.300000000000000e+01 +46 15 3.200000000000000e+01 +46 16 3.100000000000000e+01 +46 17 3.000000000000000e+01 +46 18 2.900000000000000e+01 +46 19 2.800000000000000e+01 +46 20 2.700000000000000e+01 +46 21 2.600000000000000e+01 +46 22 2.500000000000000e+01 +46 23 2.400000000000000e+01 +46 24 2.300000000000000e+01 +46 25 2.200000000000000e+01 +46 26 2.100000000000000e+01 +46 27 2.000000000000000e+01 +46 28 1.900000000000000e+01 +46 29 1.800000000000000e+01 +46 30 1.700000000000000e+01 +46 31 1.600000000000000e+01 +46 32 1.500000000000000e+01 +46 33 1.400000000000000e+01 +46 34 1.300000000000000e+01 +46 35 1.200000000000000e+01 +46 36 1.100000000000000e+01 +46 37 1.000000000000000e+01 +46 38 9.000000000000000e+00 +46 39 8.000000000000000e+00 +46 40 7.000000000000000e+00 +46 41 6.000000000000000e+00 +46 42 5.000000000000000e+00 +46 43 4.000000000000000e+00 +46 44 3.000000000000000e+00 +46 45 2.000000000000000e+00 +46 46 1.000000000000000e+00 +47 1 4.700000000000000e+01 +47 2 4.600000000000000e+01 +47 3 4.500000000000000e+01 +47 4 4.400000000000000e+01 +47 5 4.300000000000000e+01 +47 6 4.200000000000000e+01 +47 7 4.100000000000000e+01 +47 8 4.000000000000000e+01 +47 9 3.900000000000000e+01 +47 10 3.800000000000000e+01 +47 11 3.700000000000000e+01 +47 12 3.600000000000000e+01 +47 13 3.500000000000000e+01 +47 14 3.400000000000000e+01 +47 15 3.300000000000000e+01 +47 16 3.200000000000000e+01 +47 17 3.100000000000000e+01 +47 18 3.000000000000000e+01 +47 19 2.900000000000000e+01 +47 20 2.800000000000000e+01 +47 21 2.700000000000000e+01 +47 22 2.600000000000000e+01 +47 23 2.500000000000000e+01 +47 24 2.400000000000000e+01 +47 25 2.300000000000000e+01 +47 26 2.200000000000000e+01 +47 27 2.100000000000000e+01 +47 28 2.000000000000000e+01 +47 29 1.900000000000000e+01 +47 30 1.800000000000000e+01 +47 31 1.700000000000000e+01 +47 32 1.600000000000000e+01 +47 33 1.500000000000000e+01 +47 34 1.400000000000000e+01 +47 35 1.300000000000000e+01 +47 36 1.200000000000000e+01 +47 37 1.100000000000000e+01 +47 38 1.000000000000000e+01 +47 39 9.000000000000000e+00 +47 40 8.000000000000000e+00 +47 41 7.000000000000000e+00 +47 42 6.000000000000000e+00 +47 43 5.000000000000000e+00 +47 44 4.000000000000000e+00 +47 45 3.000000000000000e+00 +47 46 2.000000000000000e+00 +47 47 1.000000000000000e+00 +48 1 4.800000000000000e+01 +48 2 4.700000000000000e+01 +48 3 4.600000000000000e+01 +48 4 4.500000000000000e+01 +48 5 4.400000000000000e+01 +48 6 4.300000000000000e+01 +48 7 4.200000000000000e+01 +48 8 4.100000000000000e+01 +48 9 4.000000000000000e+01 +48 10 3.900000000000000e+01 +48 11 3.800000000000000e+01 +48 12 3.700000000000000e+01 +48 13 3.600000000000000e+01 +48 14 3.500000000000000e+01 +48 15 3.400000000000000e+01 +48 16 3.300000000000000e+01 +48 17 3.200000000000000e+01 +48 18 3.100000000000000e+01 +48 19 3.000000000000000e+01 +48 20 2.900000000000000e+01 +48 21 2.800000000000000e+01 +48 22 2.700000000000000e+01 +48 23 2.600000000000000e+01 +48 24 2.500000000000000e+01 +48 25 2.400000000000000e+01 +48 26 2.300000000000000e+01 +48 27 2.200000000000000e+01 +48 28 2.100000000000000e+01 +48 29 2.000000000000000e+01 +48 30 1.900000000000000e+01 +48 31 1.800000000000000e+01 +48 32 1.700000000000000e+01 +48 33 1.600000000000000e+01 +48 34 1.500000000000000e+01 +48 35 1.400000000000000e+01 +48 36 1.300000000000000e+01 +48 37 1.200000000000000e+01 +48 38 1.100000000000000e+01 +48 39 1.000000000000000e+01 +48 40 9.000000000000000e+00 +48 41 8.000000000000000e+00 +48 42 7.000000000000000e+00 +48 43 6.000000000000000e+00 +48 44 5.000000000000000e+00 +48 45 4.000000000000000e+00 +48 46 3.000000000000000e+00 +48 47 2.000000000000000e+00 +48 48 1.000000000000000e+00 +49 1 4.900000000000000e+01 +49 2 4.800000000000000e+01 +49 3 4.700000000000000e+01 +49 4 4.600000000000000e+01 +49 5 4.500000000000000e+01 +49 6 4.400000000000000e+01 +49 7 4.300000000000000e+01 +49 8 4.200000000000000e+01 +49 9 4.100000000000000e+01 +49 10 4.000000000000000e+01 +49 11 3.900000000000000e+01 +49 12 3.800000000000000e+01 +49 13 3.700000000000000e+01 +49 14 3.600000000000000e+01 +49 15 3.500000000000000e+01 +49 16 3.400000000000000e+01 +49 17 3.300000000000000e+01 +49 18 3.200000000000000e+01 +49 19 3.100000000000000e+01 +49 20 3.000000000000000e+01 +49 21 2.900000000000000e+01 +49 22 2.800000000000000e+01 +49 23 2.700000000000000e+01 +49 24 2.600000000000000e+01 +49 25 2.500000000000000e+01 +49 26 2.400000000000000e+01 +49 27 2.300000000000000e+01 +49 28 2.200000000000000e+01 +49 29 2.100000000000000e+01 +49 30 2.000000000000000e+01 +49 31 1.900000000000000e+01 +49 32 1.800000000000000e+01 +49 33 1.700000000000000e+01 +49 34 1.600000000000000e+01 +49 35 1.500000000000000e+01 +49 36 1.400000000000000e+01 +49 37 1.300000000000000e+01 +49 38 1.200000000000000e+01 +49 39 1.100000000000000e+01 +49 40 1.000000000000000e+01 +49 41 9.000000000000000e+00 +49 42 8.000000000000000e+00 +49 43 7.000000000000000e+00 +49 44 6.000000000000000e+00 +49 45 5.000000000000000e+00 +49 46 4.000000000000000e+00 +49 47 3.000000000000000e+00 +49 48 2.000000000000000e+00 +49 49 1.000000000000000e+00 +50 1 5.000000000000000e+01 +50 2 4.900000000000000e+01 +50 3 4.800000000000000e+01 +50 4 4.700000000000000e+01 +50 5 4.600000000000000e+01 +50 6 4.500000000000000e+01 +50 7 4.400000000000000e+01 +50 8 4.300000000000000e+01 +50 9 4.200000000000000e+01 +50 10 4.100000000000000e+01 +50 11 4.000000000000000e+01 +50 12 3.900000000000000e+01 +50 13 3.800000000000000e+01 +50 14 3.700000000000000e+01 +50 15 3.600000000000000e+01 +50 16 3.500000000000000e+01 +50 17 3.400000000000000e+01 +50 18 3.300000000000000e+01 +50 19 3.200000000000000e+01 +50 20 3.100000000000000e+01 +50 21 3.000000000000000e+01 +50 22 2.900000000000000e+01 +50 23 2.800000000000000e+01 +50 24 2.700000000000000e+01 +50 25 2.600000000000000e+01 +50 26 2.500000000000000e+01 +50 27 2.400000000000000e+01 +50 28 2.300000000000000e+01 +50 29 2.200000000000000e+01 +50 30 2.100000000000000e+01 +50 31 2.000000000000000e+01 +50 32 1.900000000000000e+01 +50 33 1.800000000000000e+01 +50 34 1.700000000000000e+01 +50 35 1.600000000000000e+01 +50 36 1.500000000000000e+01 +50 37 1.400000000000000e+01 +50 38 1.300000000000000e+01 +50 39 1.200000000000000e+01 +50 40 1.100000000000000e+01 +50 41 1.000000000000000e+01 +50 42 9.000000000000000e+00 +50 43 8.000000000000000e+00 +50 44 7.000000000000000e+00 +50 45 6.000000000000000e+00 +50 46 5.000000000000000e+00 +50 47 4.000000000000000e+00 +50 48 3.000000000000000e+00 +50 49 2.000000000000000e+00 +50 50 1.000000000000000e+00 +51 1 5.100000000000000e+01 +51 2 5.000000000000000e+01 +51 3 4.900000000000000e+01 +51 4 4.800000000000000e+01 +51 5 4.700000000000000e+01 +51 6 4.600000000000000e+01 +51 7 4.500000000000000e+01 +51 8 4.400000000000000e+01 +51 9 4.300000000000000e+01 +51 10 4.200000000000000e+01 +51 11 4.100000000000000e+01 +51 12 4.000000000000000e+01 +51 13 3.900000000000000e+01 +51 14 3.800000000000000e+01 +51 15 3.700000000000000e+01 +51 16 3.600000000000000e+01 +51 17 3.500000000000000e+01 +51 18 3.400000000000000e+01 +51 19 3.300000000000000e+01 +51 20 3.200000000000000e+01 +51 21 3.100000000000000e+01 +51 22 3.000000000000000e+01 +51 23 2.900000000000000e+01 +51 24 2.800000000000000e+01 +51 25 2.700000000000000e+01 +51 26 2.600000000000000e+01 +51 27 2.500000000000000e+01 +51 28 2.400000000000000e+01 +51 29 2.300000000000000e+01 +51 30 2.200000000000000e+01 +51 31 2.100000000000000e+01 +51 32 2.000000000000000e+01 +51 33 1.900000000000000e+01 +51 34 1.800000000000000e+01 +51 35 1.700000000000000e+01 +51 36 1.600000000000000e+01 +51 37 1.500000000000000e+01 +51 38 1.400000000000000e+01 +51 39 1.300000000000000e+01 +51 40 1.200000000000000e+01 +51 41 1.100000000000000e+01 +51 42 1.000000000000000e+01 +51 43 9.000000000000000e+00 +51 44 8.000000000000000e+00 +51 45 7.000000000000000e+00 +51 46 6.000000000000000e+00 +51 47 5.000000000000000e+00 +51 48 4.000000000000000e+00 +51 49 3.000000000000000e+00 +51 50 2.000000000000000e+00 +51 51 1.000000000000000e+00 +52 1 5.200000000000000e+01 +52 2 5.100000000000000e+01 +52 3 5.000000000000000e+01 +52 4 4.900000000000000e+01 +52 5 4.800000000000000e+01 +52 6 4.700000000000000e+01 +52 7 4.600000000000000e+01 +52 8 4.500000000000000e+01 +52 9 4.400000000000000e+01 +52 10 4.300000000000000e+01 +52 11 4.200000000000000e+01 +52 12 4.100000000000000e+01 +52 13 4.000000000000000e+01 +52 14 3.900000000000000e+01 +52 15 3.800000000000000e+01 +52 16 3.700000000000000e+01 +52 17 3.600000000000000e+01 +52 18 3.500000000000000e+01 +52 19 3.400000000000000e+01 +52 20 3.300000000000000e+01 +52 21 3.200000000000000e+01 +52 22 3.100000000000000e+01 +52 23 3.000000000000000e+01 +52 24 2.900000000000000e+01 +52 25 2.800000000000000e+01 +52 26 2.700000000000000e+01 +52 27 2.600000000000000e+01 +52 28 2.500000000000000e+01 +52 29 2.400000000000000e+01 +52 30 2.300000000000000e+01 +52 31 2.200000000000000e+01 +52 32 2.100000000000000e+01 +52 33 2.000000000000000e+01 +52 34 1.900000000000000e+01 +52 35 1.800000000000000e+01 +52 36 1.700000000000000e+01 +52 37 1.600000000000000e+01 +52 38 1.500000000000000e+01 +52 39 1.400000000000000e+01 +52 40 1.300000000000000e+01 +52 41 1.200000000000000e+01 +52 42 1.100000000000000e+01 +52 43 1.000000000000000e+01 +52 44 9.000000000000000e+00 +52 45 8.000000000000000e+00 +52 46 7.000000000000000e+00 +52 47 6.000000000000000e+00 +52 48 5.000000000000000e+00 +52 49 4.000000000000000e+00 +52 50 3.000000000000000e+00 +52 51 2.000000000000000e+00 +52 52 1.000000000000000e+00 +53 1 5.300000000000000e+01 +53 2 5.200000000000000e+01 +53 3 5.100000000000000e+01 +53 4 5.000000000000000e+01 +53 5 4.900000000000000e+01 +53 6 4.800000000000000e+01 +53 7 4.700000000000000e+01 +53 8 4.600000000000000e+01 +53 9 4.500000000000000e+01 +53 10 4.400000000000000e+01 +53 11 4.300000000000000e+01 +53 12 4.200000000000000e+01 +53 13 4.100000000000000e+01 +53 14 4.000000000000000e+01 +53 15 3.900000000000000e+01 +53 16 3.800000000000000e+01 +53 17 3.700000000000000e+01 +53 18 3.600000000000000e+01 +53 19 3.500000000000000e+01 +53 20 3.400000000000000e+01 +53 21 3.300000000000000e+01 +53 22 3.200000000000000e+01 +53 23 3.100000000000000e+01 +53 24 3.000000000000000e+01 +53 25 2.900000000000000e+01 +53 26 2.800000000000000e+01 +53 27 2.700000000000000e+01 +53 28 2.600000000000000e+01 +53 29 2.500000000000000e+01 +53 30 2.400000000000000e+01 +53 31 2.300000000000000e+01 +53 32 2.200000000000000e+01 +53 33 2.100000000000000e+01 +53 34 2.000000000000000e+01 +53 35 1.900000000000000e+01 +53 36 1.800000000000000e+01 +53 37 1.700000000000000e+01 +53 38 1.600000000000000e+01 +53 39 1.500000000000000e+01 +53 40 1.400000000000000e+01 +53 41 1.300000000000000e+01 +53 42 1.200000000000000e+01 +53 43 1.100000000000000e+01 +53 44 1.000000000000000e+01 +53 45 9.000000000000000e+00 +53 46 8.000000000000000e+00 +53 47 7.000000000000000e+00 +53 48 6.000000000000000e+00 +53 49 5.000000000000000e+00 +53 50 4.000000000000000e+00 +53 51 3.000000000000000e+00 +53 52 2.000000000000000e+00 +53 53 1.000000000000000e+00 +54 1 5.400000000000000e+01 +54 2 5.300000000000000e+01 +54 3 5.200000000000000e+01 +54 4 5.100000000000000e+01 +54 5 5.000000000000000e+01 +54 6 4.900000000000000e+01 +54 7 4.800000000000000e+01 +54 8 4.700000000000000e+01 +54 9 4.600000000000000e+01 +54 10 4.500000000000000e+01 +54 11 4.400000000000000e+01 +54 12 4.300000000000000e+01 +54 13 4.200000000000000e+01 +54 14 4.100000000000000e+01 +54 15 4.000000000000000e+01 +54 16 3.900000000000000e+01 +54 17 3.800000000000000e+01 +54 18 3.700000000000000e+01 +54 19 3.600000000000000e+01 +54 20 3.500000000000000e+01 +54 21 3.400000000000000e+01 +54 22 3.300000000000000e+01 +54 23 3.200000000000000e+01 +54 24 3.100000000000000e+01 +54 25 3.000000000000000e+01 +54 26 2.900000000000000e+01 +54 27 2.800000000000000e+01 +54 28 2.700000000000000e+01 +54 29 2.600000000000000e+01 +54 30 2.500000000000000e+01 +54 31 2.400000000000000e+01 +54 32 2.300000000000000e+01 +54 33 2.200000000000000e+01 +54 34 2.100000000000000e+01 +54 35 2.000000000000000e+01 +54 36 1.900000000000000e+01 +54 37 1.800000000000000e+01 +54 38 1.700000000000000e+01 +54 39 1.600000000000000e+01 +54 40 1.500000000000000e+01 +54 41 1.400000000000000e+01 +54 42 1.300000000000000e+01 +54 43 1.200000000000000e+01 +54 44 1.100000000000000e+01 +54 45 1.000000000000000e+01 +54 46 9.000000000000000e+00 +54 47 8.000000000000000e+00 +54 48 7.000000000000000e+00 +54 49 6.000000000000000e+00 +54 50 5.000000000000000e+00 +54 51 4.000000000000000e+00 +54 52 3.000000000000000e+00 +54 53 2.000000000000000e+00 +54 54 1.000000000000000e+00 +55 1 5.500000000000000e+01 +55 2 5.400000000000000e+01 +55 3 5.300000000000000e+01 +55 4 5.200000000000000e+01 +55 5 5.100000000000000e+01 +55 6 5.000000000000000e+01 +55 7 4.900000000000000e+01 +55 8 4.800000000000000e+01 +55 9 4.700000000000000e+01 +55 10 4.600000000000000e+01 +55 11 4.500000000000000e+01 +55 12 4.400000000000000e+01 +55 13 4.300000000000000e+01 +55 14 4.200000000000000e+01 +55 15 4.100000000000000e+01 +55 16 4.000000000000000e+01 +55 17 3.900000000000000e+01 +55 18 3.800000000000000e+01 +55 19 3.700000000000000e+01 +55 20 3.600000000000000e+01 +55 21 3.500000000000000e+01 +55 22 3.400000000000000e+01 +55 23 3.300000000000000e+01 +55 24 3.200000000000000e+01 +55 25 3.100000000000000e+01 +55 26 3.000000000000000e+01 +55 27 2.900000000000000e+01 +55 28 2.800000000000000e+01 +55 29 2.700000000000000e+01 +55 30 2.600000000000000e+01 +55 31 2.500000000000000e+01 +55 32 2.400000000000000e+01 +55 33 2.300000000000000e+01 +55 34 2.200000000000000e+01 +55 35 2.100000000000000e+01 +55 36 2.000000000000000e+01 +55 37 1.900000000000000e+01 +55 38 1.800000000000000e+01 +55 39 1.700000000000000e+01 +55 40 1.600000000000000e+01 +55 41 1.500000000000000e+01 +55 42 1.400000000000000e+01 +55 43 1.300000000000000e+01 +55 44 1.200000000000000e+01 +55 45 1.100000000000000e+01 +55 46 1.000000000000000e+01 +55 47 9.000000000000000e+00 +55 48 8.000000000000000e+00 +55 49 7.000000000000000e+00 +55 50 6.000000000000000e+00 +55 51 5.000000000000000e+00 +55 52 4.000000000000000e+00 +55 53 3.000000000000000e+00 +55 54 2.000000000000000e+00 +55 55 1.000000000000000e+00 +56 1 5.600000000000000e+01 +56 2 5.500000000000000e+01 +56 3 5.400000000000000e+01 +56 4 5.300000000000000e+01 +56 5 5.200000000000000e+01 +56 6 5.100000000000000e+01 +56 7 5.000000000000000e+01 +56 8 4.900000000000000e+01 +56 9 4.800000000000000e+01 +56 10 4.700000000000000e+01 +56 11 4.600000000000000e+01 +56 12 4.500000000000000e+01 +56 13 4.400000000000000e+01 +56 14 4.300000000000000e+01 +56 15 4.200000000000000e+01 +56 16 4.100000000000000e+01 +56 17 4.000000000000000e+01 +56 18 3.900000000000000e+01 +56 19 3.800000000000000e+01 +56 20 3.700000000000000e+01 +56 21 3.600000000000000e+01 +56 22 3.500000000000000e+01 +56 23 3.400000000000000e+01 +56 24 3.300000000000000e+01 +56 25 3.200000000000000e+01 +56 26 3.100000000000000e+01 +56 27 3.000000000000000e+01 +56 28 2.900000000000000e+01 +56 29 2.800000000000000e+01 +56 30 2.700000000000000e+01 +56 31 2.600000000000000e+01 +56 32 2.500000000000000e+01 +56 33 2.400000000000000e+01 +56 34 2.300000000000000e+01 +56 35 2.200000000000000e+01 +56 36 2.100000000000000e+01 +56 37 2.000000000000000e+01 +56 38 1.900000000000000e+01 +56 39 1.800000000000000e+01 +56 40 1.700000000000000e+01 +56 41 1.600000000000000e+01 +56 42 1.500000000000000e+01 +56 43 1.400000000000000e+01 +56 44 1.300000000000000e+01 +56 45 1.200000000000000e+01 +56 46 1.100000000000000e+01 +56 47 1.000000000000000e+01 +56 48 9.000000000000000e+00 +56 49 8.000000000000000e+00 +56 50 7.000000000000000e+00 +56 51 6.000000000000000e+00 +56 52 5.000000000000000e+00 +56 53 4.000000000000000e+00 +56 54 3.000000000000000e+00 +56 55 2.000000000000000e+00 +56 56 1.000000000000000e+00 +57 1 5.700000000000000e+01 +57 2 5.600000000000000e+01 +57 3 5.500000000000000e+01 +57 4 5.400000000000000e+01 +57 5 5.300000000000000e+01 +57 6 5.200000000000000e+01 +57 7 5.100000000000000e+01 +57 8 5.000000000000000e+01 +57 9 4.900000000000000e+01 +57 10 4.800000000000000e+01 +57 11 4.700000000000000e+01 +57 12 4.600000000000000e+01 +57 13 4.500000000000000e+01 +57 14 4.400000000000000e+01 +57 15 4.300000000000000e+01 +57 16 4.200000000000000e+01 +57 17 4.100000000000000e+01 +57 18 4.000000000000000e+01 +57 19 3.900000000000000e+01 +57 20 3.800000000000000e+01 +57 21 3.700000000000000e+01 +57 22 3.600000000000000e+01 +57 23 3.500000000000000e+01 +57 24 3.400000000000000e+01 +57 25 3.300000000000000e+01 +57 26 3.200000000000000e+01 +57 27 3.100000000000000e+01 +57 28 3.000000000000000e+01 +57 29 2.900000000000000e+01 +57 30 2.800000000000000e+01 +57 31 2.700000000000000e+01 +57 32 2.600000000000000e+01 +57 33 2.500000000000000e+01 +57 34 2.400000000000000e+01 +57 35 2.300000000000000e+01 +57 36 2.200000000000000e+01 +57 37 2.100000000000000e+01 +57 38 2.000000000000000e+01 +57 39 1.900000000000000e+01 +57 40 1.800000000000000e+01 +57 41 1.700000000000000e+01 +57 42 1.600000000000000e+01 +57 43 1.500000000000000e+01 +57 44 1.400000000000000e+01 +57 45 1.300000000000000e+01 +57 46 1.200000000000000e+01 +57 47 1.100000000000000e+01 +57 48 1.000000000000000e+01 +57 49 9.000000000000000e+00 +57 50 8.000000000000000e+00 +57 51 7.000000000000000e+00 +57 52 6.000000000000000e+00 +57 53 5.000000000000000e+00 +57 54 4.000000000000000e+00 +57 55 3.000000000000000e+00 +57 56 2.000000000000000e+00 +57 57 1.000000000000000e+00 +58 1 5.800000000000000e+01 +58 2 5.700000000000000e+01 +58 3 5.600000000000000e+01 +58 4 5.500000000000000e+01 +58 5 5.400000000000000e+01 +58 6 5.300000000000000e+01 +58 7 5.200000000000000e+01 +58 8 5.100000000000000e+01 +58 9 5.000000000000000e+01 +58 10 4.900000000000000e+01 +58 11 4.800000000000000e+01 +58 12 4.700000000000000e+01 +58 13 4.600000000000000e+01 +58 14 4.500000000000000e+01 +58 15 4.400000000000000e+01 +58 16 4.300000000000000e+01 +58 17 4.200000000000000e+01 +58 18 4.100000000000000e+01 +58 19 4.000000000000000e+01 +58 20 3.900000000000000e+01 +58 21 3.800000000000000e+01 +58 22 3.700000000000000e+01 +58 23 3.600000000000000e+01 +58 24 3.500000000000000e+01 +58 25 3.400000000000000e+01 +58 26 3.300000000000000e+01 +58 27 3.200000000000000e+01 +58 28 3.100000000000000e+01 +58 29 3.000000000000000e+01 +58 30 2.900000000000000e+01 +58 31 2.800000000000000e+01 +58 32 2.700000000000000e+01 +58 33 2.600000000000000e+01 +58 34 2.500000000000000e+01 +58 35 2.400000000000000e+01 +58 36 2.300000000000000e+01 +58 37 2.200000000000000e+01 +58 38 2.100000000000000e+01 +58 39 2.000000000000000e+01 +58 40 1.900000000000000e+01 +58 41 1.800000000000000e+01 +58 42 1.700000000000000e+01 +58 43 1.600000000000000e+01 +58 44 1.500000000000000e+01 +58 45 1.400000000000000e+01 +58 46 1.300000000000000e+01 +58 47 1.200000000000000e+01 +58 48 1.100000000000000e+01 +58 49 1.000000000000000e+01 +58 50 9.000000000000000e+00 +58 51 8.000000000000000e+00 +58 52 7.000000000000000e+00 +58 53 6.000000000000000e+00 +58 54 5.000000000000000e+00 +58 55 4.000000000000000e+00 +58 56 3.000000000000000e+00 +58 57 2.000000000000000e+00 +58 58 1.000000000000000e+00 +59 1 5.900000000000000e+01 +59 2 5.800000000000000e+01 +59 3 5.700000000000000e+01 +59 4 5.600000000000000e+01 +59 5 5.500000000000000e+01 +59 6 5.400000000000000e+01 +59 7 5.300000000000000e+01 +59 8 5.200000000000000e+01 +59 9 5.100000000000000e+01 +59 10 5.000000000000000e+01 +59 11 4.900000000000000e+01 +59 12 4.800000000000000e+01 +59 13 4.700000000000000e+01 +59 14 4.600000000000000e+01 +59 15 4.500000000000000e+01 +59 16 4.400000000000000e+01 +59 17 4.300000000000000e+01 +59 18 4.200000000000000e+01 +59 19 4.100000000000000e+01 +59 20 4.000000000000000e+01 +59 21 3.900000000000000e+01 +59 22 3.800000000000000e+01 +59 23 3.700000000000000e+01 +59 24 3.600000000000000e+01 +59 25 3.500000000000000e+01 +59 26 3.400000000000000e+01 +59 27 3.300000000000000e+01 +59 28 3.200000000000000e+01 +59 29 3.100000000000000e+01 +59 30 3.000000000000000e+01 +59 31 2.900000000000000e+01 +59 32 2.800000000000000e+01 +59 33 2.700000000000000e+01 +59 34 2.600000000000000e+01 +59 35 2.500000000000000e+01 +59 36 2.400000000000000e+01 +59 37 2.300000000000000e+01 +59 38 2.200000000000000e+01 +59 39 2.100000000000000e+01 +59 40 2.000000000000000e+01 +59 41 1.900000000000000e+01 +59 42 1.800000000000000e+01 +59 43 1.700000000000000e+01 +59 44 1.600000000000000e+01 +59 45 1.500000000000000e+01 +59 46 1.400000000000000e+01 +59 47 1.300000000000000e+01 +59 48 1.200000000000000e+01 +59 49 1.100000000000000e+01 +59 50 1.000000000000000e+01 +59 51 9.000000000000000e+00 +59 52 8.000000000000000e+00 +59 53 7.000000000000000e+00 +59 54 6.000000000000000e+00 +59 55 5.000000000000000e+00 +59 56 4.000000000000000e+00 +59 57 3.000000000000000e+00 +59 58 2.000000000000000e+00 +59 59 1.000000000000000e+00 +60 1 6.000000000000000e+01 +60 2 5.900000000000000e+01 +60 3 5.800000000000000e+01 +60 4 5.700000000000000e+01 +60 5 5.600000000000000e+01 +60 6 5.500000000000000e+01 +60 7 5.400000000000000e+01 +60 8 5.300000000000000e+01 +60 9 5.200000000000000e+01 +60 10 5.100000000000000e+01 +60 11 5.000000000000000e+01 +60 12 4.900000000000000e+01 +60 13 4.800000000000000e+01 +60 14 4.700000000000000e+01 +60 15 4.600000000000000e+01 +60 16 4.500000000000000e+01 +60 17 4.400000000000000e+01 +60 18 4.300000000000000e+01 +60 19 4.200000000000000e+01 +60 20 4.100000000000000e+01 +60 21 4.000000000000000e+01 +60 22 3.900000000000000e+01 +60 23 3.800000000000000e+01 +60 24 3.700000000000000e+01 +60 25 3.600000000000000e+01 +60 26 3.500000000000000e+01 +60 27 3.400000000000000e+01 +60 28 3.300000000000000e+01 +60 29 3.200000000000000e+01 +60 30 3.100000000000000e+01 +60 31 3.000000000000000e+01 +60 32 2.900000000000000e+01 +60 33 2.800000000000000e+01 +60 34 2.700000000000000e+01 +60 35 2.600000000000000e+01 +60 36 2.500000000000000e+01 +60 37 2.400000000000000e+01 +60 38 2.300000000000000e+01 +60 39 2.200000000000000e+01 +60 40 2.100000000000000e+01 +60 41 2.000000000000000e+01 +60 42 1.900000000000000e+01 +60 43 1.800000000000000e+01 +60 44 1.700000000000000e+01 +60 45 1.600000000000000e+01 +60 46 1.500000000000000e+01 +60 47 1.400000000000000e+01 +60 48 1.300000000000000e+01 +60 49 1.200000000000000e+01 +60 50 1.100000000000000e+01 +60 51 1.000000000000000e+01 +60 52 9.000000000000000e+00 +60 53 8.000000000000000e+00 +60 54 7.000000000000000e+00 +60 55 6.000000000000000e+00 +60 56 5.000000000000000e+00 +60 57 4.000000000000000e+00 +60 58 3.000000000000000e+00 +60 59 2.000000000000000e+00 +60 60 1.000000000000000e+00 +61 1 6.100000000000000e+01 +61 2 6.000000000000000e+01 +61 3 5.900000000000000e+01 +61 4 5.800000000000000e+01 +61 5 5.700000000000000e+01 +61 6 5.600000000000000e+01 +61 7 5.500000000000000e+01 +61 8 5.400000000000000e+01 +61 9 5.300000000000000e+01 +61 10 5.200000000000000e+01 +61 11 5.100000000000000e+01 +61 12 5.000000000000000e+01 +61 13 4.900000000000000e+01 +61 14 4.800000000000000e+01 +61 15 4.700000000000000e+01 +61 16 4.600000000000000e+01 +61 17 4.500000000000000e+01 +61 18 4.400000000000000e+01 +61 19 4.300000000000000e+01 +61 20 4.200000000000000e+01 +61 21 4.100000000000000e+01 +61 22 4.000000000000000e+01 +61 23 3.900000000000000e+01 +61 24 3.800000000000000e+01 +61 25 3.700000000000000e+01 +61 26 3.600000000000000e+01 +61 27 3.500000000000000e+01 +61 28 3.400000000000000e+01 +61 29 3.300000000000000e+01 +61 30 3.200000000000000e+01 +61 31 3.100000000000000e+01 +61 32 3.000000000000000e+01 +61 33 2.900000000000000e+01 +61 34 2.800000000000000e+01 +61 35 2.700000000000000e+01 +61 36 2.600000000000000e+01 +61 37 2.500000000000000e+01 +61 38 2.400000000000000e+01 +61 39 2.300000000000000e+01 +61 40 2.200000000000000e+01 +61 41 2.100000000000000e+01 +61 42 2.000000000000000e+01 +61 43 1.900000000000000e+01 +61 44 1.800000000000000e+01 +61 45 1.700000000000000e+01 +61 46 1.600000000000000e+01 +61 47 1.500000000000000e+01 +61 48 1.400000000000000e+01 +61 49 1.300000000000000e+01 +61 50 1.200000000000000e+01 +61 51 1.100000000000000e+01 +61 52 1.000000000000000e+01 +61 53 9.000000000000000e+00 +61 54 8.000000000000000e+00 +61 55 7.000000000000000e+00 +61 56 6.000000000000000e+00 +61 57 5.000000000000000e+00 +61 58 4.000000000000000e+00 +61 59 3.000000000000000e+00 +61 60 2.000000000000000e+00 +61 61 1.000000000000000e+00 +62 1 6.200000000000000e+01 +62 2 6.100000000000000e+01 +62 3 6.000000000000000e+01 +62 4 5.900000000000000e+01 +62 5 5.800000000000000e+01 +62 6 5.700000000000000e+01 +62 7 5.600000000000000e+01 +62 8 5.500000000000000e+01 +62 9 5.400000000000000e+01 +62 10 5.300000000000000e+01 +62 11 5.200000000000000e+01 +62 12 5.100000000000000e+01 +62 13 5.000000000000000e+01 +62 14 4.900000000000000e+01 +62 15 4.800000000000000e+01 +62 16 4.700000000000000e+01 +62 17 4.600000000000000e+01 +62 18 4.500000000000000e+01 +62 19 4.400000000000000e+01 +62 20 4.300000000000000e+01 +62 21 4.200000000000000e+01 +62 22 4.100000000000000e+01 +62 23 4.000000000000000e+01 +62 24 3.900000000000000e+01 +62 25 3.800000000000000e+01 +62 26 3.700000000000000e+01 +62 27 3.600000000000000e+01 +62 28 3.500000000000000e+01 +62 29 3.400000000000000e+01 +62 30 3.300000000000000e+01 +62 31 3.200000000000000e+01 +62 32 3.100000000000000e+01 +62 33 3.000000000000000e+01 +62 34 2.900000000000000e+01 +62 35 2.800000000000000e+01 +62 36 2.700000000000000e+01 +62 37 2.600000000000000e+01 +62 38 2.500000000000000e+01 +62 39 2.400000000000000e+01 +62 40 2.300000000000000e+01 +62 41 2.200000000000000e+01 +62 42 2.100000000000000e+01 +62 43 2.000000000000000e+01 +62 44 1.900000000000000e+01 +62 45 1.800000000000000e+01 +62 46 1.700000000000000e+01 +62 47 1.600000000000000e+01 +62 48 1.500000000000000e+01 +62 49 1.400000000000000e+01 +62 50 1.300000000000000e+01 +62 51 1.200000000000000e+01 +62 52 1.100000000000000e+01 +62 53 1.000000000000000e+01 +62 54 9.000000000000000e+00 +62 55 8.000000000000000e+00 +62 56 7.000000000000000e+00 +62 57 6.000000000000000e+00 +62 58 5.000000000000000e+00 +62 59 4.000000000000000e+00 +62 60 3.000000000000000e+00 +62 61 2.000000000000000e+00 +62 62 1.000000000000000e+00 +63 1 6.300000000000000e+01 +63 2 6.200000000000000e+01 +63 3 6.100000000000000e+01 +63 4 6.000000000000000e+01 +63 5 5.900000000000000e+01 +63 6 5.800000000000000e+01 +63 7 5.700000000000000e+01 +63 8 5.600000000000000e+01 +63 9 5.500000000000000e+01 +63 10 5.400000000000000e+01 +63 11 5.300000000000000e+01 +63 12 5.200000000000000e+01 +63 13 5.100000000000000e+01 +63 14 5.000000000000000e+01 +63 15 4.900000000000000e+01 +63 16 4.800000000000000e+01 +63 17 4.700000000000000e+01 +63 18 4.600000000000000e+01 +63 19 4.500000000000000e+01 +63 20 4.400000000000000e+01 +63 21 4.300000000000000e+01 +63 22 4.200000000000000e+01 +63 23 4.100000000000000e+01 +63 24 4.000000000000000e+01 +63 25 3.900000000000000e+01 +63 26 3.800000000000000e+01 +63 27 3.700000000000000e+01 +63 28 3.600000000000000e+01 +63 29 3.500000000000000e+01 +63 30 3.400000000000000e+01 +63 31 3.300000000000000e+01 +63 32 3.200000000000000e+01 +63 33 3.100000000000000e+01 +63 34 3.000000000000000e+01 +63 35 2.900000000000000e+01 +63 36 2.800000000000000e+01 +63 37 2.700000000000000e+01 +63 38 2.600000000000000e+01 +63 39 2.500000000000000e+01 +63 40 2.400000000000000e+01 +63 41 2.300000000000000e+01 +63 42 2.200000000000000e+01 +63 43 2.100000000000000e+01 +63 44 2.000000000000000e+01 +63 45 1.900000000000000e+01 +63 46 1.800000000000000e+01 +63 47 1.700000000000000e+01 +63 48 1.600000000000000e+01 +63 49 1.500000000000000e+01 +63 50 1.400000000000000e+01 +63 51 1.300000000000000e+01 +63 52 1.200000000000000e+01 +63 53 1.100000000000000e+01 +63 54 1.000000000000000e+01 +63 55 9.000000000000000e+00 +63 56 8.000000000000000e+00 +63 57 7.000000000000000e+00 +63 58 6.000000000000000e+00 +63 59 5.000000000000000e+00 +63 60 4.000000000000000e+00 +63 61 3.000000000000000e+00 +63 62 2.000000000000000e+00 +63 63 1.000000000000000e+00 +64 1 6.400000000000000e+01 +64 2 6.300000000000000e+01 +64 3 6.200000000000000e+01 +64 4 6.100000000000000e+01 +64 5 6.000000000000000e+01 +64 6 5.900000000000000e+01 +64 7 5.800000000000000e+01 +64 8 5.700000000000000e+01 +64 9 5.600000000000000e+01 +64 10 5.500000000000000e+01 +64 11 5.400000000000000e+01 +64 12 5.300000000000000e+01 +64 13 5.200000000000000e+01 +64 14 5.100000000000000e+01 +64 15 5.000000000000000e+01 +64 16 4.900000000000000e+01 +64 17 4.800000000000000e+01 +64 18 4.700000000000000e+01 +64 19 4.600000000000000e+01 +64 20 4.500000000000000e+01 +64 21 4.400000000000000e+01 +64 22 4.300000000000000e+01 +64 23 4.200000000000000e+01 +64 24 4.100000000000000e+01 +64 25 4.000000000000000e+01 +64 26 3.900000000000000e+01 +64 27 3.800000000000000e+01 +64 28 3.700000000000000e+01 +64 29 3.600000000000000e+01 +64 30 3.500000000000000e+01 +64 31 3.400000000000000e+01 +64 32 3.300000000000000e+01 +64 33 3.200000000000000e+01 +64 34 3.100000000000000e+01 +64 35 3.000000000000000e+01 +64 36 2.900000000000000e+01 +64 37 2.800000000000000e+01 +64 38 2.700000000000000e+01 +64 39 2.600000000000000e+01 +64 40 2.500000000000000e+01 +64 41 2.400000000000000e+01 +64 42 2.300000000000000e+01 +64 43 2.200000000000000e+01 +64 44 2.100000000000000e+01 +64 45 2.000000000000000e+01 +64 46 1.900000000000000e+01 +64 47 1.800000000000000e+01 +64 48 1.700000000000000e+01 +64 49 1.600000000000000e+01 +64 50 1.500000000000000e+01 +64 51 1.400000000000000e+01 +64 52 1.300000000000000e+01 +64 53 1.200000000000000e+01 +64 54 1.100000000000000e+01 +64 55 1.000000000000000e+01 +64 56 9.000000000000000e+00 +64 57 8.000000000000000e+00 +64 58 7.000000000000000e+00 +64 59 6.000000000000000e+00 +64 60 5.000000000000000e+00 +64 61 4.000000000000000e+00 +64 62 3.000000000000000e+00 +64 63 2.000000000000000e+00 +64 64 1.000000000000000e+00 +65 1 6.500000000000000e+01 +65 2 6.400000000000000e+01 +65 3 6.300000000000000e+01 +65 4 6.200000000000000e+01 +65 5 6.100000000000000e+01 +65 6 6.000000000000000e+01 +65 7 5.900000000000000e+01 +65 8 5.800000000000000e+01 +65 9 5.700000000000000e+01 +65 10 5.600000000000000e+01 +65 11 5.500000000000000e+01 +65 12 5.400000000000000e+01 +65 13 5.300000000000000e+01 +65 14 5.200000000000000e+01 +65 15 5.100000000000000e+01 +65 16 5.000000000000000e+01 +65 17 4.900000000000000e+01 +65 18 4.800000000000000e+01 +65 19 4.700000000000000e+01 +65 20 4.600000000000000e+01 +65 21 4.500000000000000e+01 +65 22 4.400000000000000e+01 +65 23 4.300000000000000e+01 +65 24 4.200000000000000e+01 +65 25 4.100000000000000e+01 +65 26 4.000000000000000e+01 +65 27 3.900000000000000e+01 +65 28 3.800000000000000e+01 +65 29 3.700000000000000e+01 +65 30 3.600000000000000e+01 +65 31 3.500000000000000e+01 +65 32 3.400000000000000e+01 +65 33 3.300000000000000e+01 +65 34 3.200000000000000e+01 +65 35 3.100000000000000e+01 +65 36 3.000000000000000e+01 +65 37 2.900000000000000e+01 +65 38 2.800000000000000e+01 +65 39 2.700000000000000e+01 +65 40 2.600000000000000e+01 +65 41 2.500000000000000e+01 +65 42 2.400000000000000e+01 +65 43 2.300000000000000e+01 +65 44 2.200000000000000e+01 +65 45 2.100000000000000e+01 +65 46 2.000000000000000e+01 +65 47 1.900000000000000e+01 +65 48 1.800000000000000e+01 +65 49 1.700000000000000e+01 +65 50 1.600000000000000e+01 +65 51 1.500000000000000e+01 +65 52 1.400000000000000e+01 +65 53 1.300000000000000e+01 +65 54 1.200000000000000e+01 +65 55 1.100000000000000e+01 +65 56 1.000000000000000e+01 +65 57 9.000000000000000e+00 +65 58 8.000000000000000e+00 +65 59 7.000000000000000e+00 +65 60 6.000000000000000e+00 +65 61 5.000000000000000e+00 +65 62 4.000000000000000e+00 +65 63 3.000000000000000e+00 +65 64 2.000000000000000e+00 +65 65 1.000000000000000e+00 +66 1 6.600000000000000e+01 +66 2 6.500000000000000e+01 +66 3 6.400000000000000e+01 +66 4 6.300000000000000e+01 +66 5 6.200000000000000e+01 +66 6 6.100000000000000e+01 +66 7 6.000000000000000e+01 +66 8 5.900000000000000e+01 +66 9 5.800000000000000e+01 +66 10 5.700000000000000e+01 +66 11 5.600000000000000e+01 +66 12 5.500000000000000e+01 +66 13 5.400000000000000e+01 +66 14 5.300000000000000e+01 +66 15 5.200000000000000e+01 +66 16 5.100000000000000e+01 +66 17 5.000000000000000e+01 +66 18 4.900000000000000e+01 +66 19 4.800000000000000e+01 +66 20 4.700000000000000e+01 +66 21 4.600000000000000e+01 +66 22 4.500000000000000e+01 +66 23 4.400000000000000e+01 +66 24 4.300000000000000e+01 +66 25 4.200000000000000e+01 +66 26 4.100000000000000e+01 +66 27 4.000000000000000e+01 +66 28 3.900000000000000e+01 +66 29 3.800000000000000e+01 +66 30 3.700000000000000e+01 +66 31 3.600000000000000e+01 +66 32 3.500000000000000e+01 +66 33 3.400000000000000e+01 +66 34 3.300000000000000e+01 +66 35 3.200000000000000e+01 +66 36 3.100000000000000e+01 +66 37 3.000000000000000e+01 +66 38 2.900000000000000e+01 +66 39 2.800000000000000e+01 +66 40 2.700000000000000e+01 +66 41 2.600000000000000e+01 +66 42 2.500000000000000e+01 +66 43 2.400000000000000e+01 +66 44 2.300000000000000e+01 +66 45 2.200000000000000e+01 +66 46 2.100000000000000e+01 +66 47 2.000000000000000e+01 +66 48 1.900000000000000e+01 +66 49 1.800000000000000e+01 +66 50 1.700000000000000e+01 +66 51 1.600000000000000e+01 +66 52 1.500000000000000e+01 +66 53 1.400000000000000e+01 +66 54 1.300000000000000e+01 +66 55 1.200000000000000e+01 +66 56 1.100000000000000e+01 +66 57 1.000000000000000e+01 +66 58 9.000000000000000e+00 +66 59 8.000000000000000e+00 +66 60 7.000000000000000e+00 +66 61 6.000000000000000e+00 +66 62 5.000000000000000e+00 +66 63 4.000000000000000e+00 +66 64 3.000000000000000e+00 +66 65 2.000000000000000e+00 +66 66 1.000000000000000e+00 +67 1 6.700000000000000e+01 +67 2 6.600000000000000e+01 +67 3 6.500000000000000e+01 +67 4 6.400000000000000e+01 +67 5 6.300000000000000e+01 +67 6 6.200000000000000e+01 +67 7 6.100000000000000e+01 +67 8 6.000000000000000e+01 +67 9 5.900000000000000e+01 +67 10 5.800000000000000e+01 +67 11 5.700000000000000e+01 +67 12 5.600000000000000e+01 +67 13 5.500000000000000e+01 +67 14 5.400000000000000e+01 +67 15 5.300000000000000e+01 +67 16 5.200000000000000e+01 +67 17 5.100000000000000e+01 +67 18 5.000000000000000e+01 +67 19 4.900000000000000e+01 +67 20 4.800000000000000e+01 +67 21 4.700000000000000e+01 +67 22 4.600000000000000e+01 +67 23 4.500000000000000e+01 +67 24 4.400000000000000e+01 +67 25 4.300000000000000e+01 +67 26 4.200000000000000e+01 +67 27 4.100000000000000e+01 +67 28 4.000000000000000e+01 +67 29 3.900000000000000e+01 +67 30 3.800000000000000e+01 +67 31 3.700000000000000e+01 +67 32 3.600000000000000e+01 +67 33 3.500000000000000e+01 +67 34 3.400000000000000e+01 +67 35 3.300000000000000e+01 +67 36 3.200000000000000e+01 +67 37 3.100000000000000e+01 +67 38 3.000000000000000e+01 +67 39 2.900000000000000e+01 +67 40 2.800000000000000e+01 +67 41 2.700000000000000e+01 +67 42 2.600000000000000e+01 +67 43 2.500000000000000e+01 +67 44 2.400000000000000e+01 +67 45 2.300000000000000e+01 +67 46 2.200000000000000e+01 +67 47 2.100000000000000e+01 +67 48 2.000000000000000e+01 +67 49 1.900000000000000e+01 +67 50 1.800000000000000e+01 +67 51 1.700000000000000e+01 +67 52 1.600000000000000e+01 +67 53 1.500000000000000e+01 +67 54 1.400000000000000e+01 +67 55 1.300000000000000e+01 +67 56 1.200000000000000e+01 +67 57 1.100000000000000e+01 +67 58 1.000000000000000e+01 +67 59 9.000000000000000e+00 +67 60 8.000000000000000e+00 +67 61 7.000000000000000e+00 +67 62 6.000000000000000e+00 +67 63 5.000000000000000e+00 +67 64 4.000000000000000e+00 +67 65 3.000000000000000e+00 +67 66 2.000000000000000e+00 +67 67 1.000000000000000e+00 +68 1 6.800000000000000e+01 +68 2 6.700000000000000e+01 +68 3 6.600000000000000e+01 +68 4 6.500000000000000e+01 +68 5 6.400000000000000e+01 +68 6 6.300000000000000e+01 +68 7 6.200000000000000e+01 +68 8 6.100000000000000e+01 +68 9 6.000000000000000e+01 +68 10 5.900000000000000e+01 +68 11 5.800000000000000e+01 +68 12 5.700000000000000e+01 +68 13 5.600000000000000e+01 +68 14 5.500000000000000e+01 +68 15 5.400000000000000e+01 +68 16 5.300000000000000e+01 +68 17 5.200000000000000e+01 +68 18 5.100000000000000e+01 +68 19 5.000000000000000e+01 +68 20 4.900000000000000e+01 +68 21 4.800000000000000e+01 +68 22 4.700000000000000e+01 +68 23 4.600000000000000e+01 +68 24 4.500000000000000e+01 +68 25 4.400000000000000e+01 +68 26 4.300000000000000e+01 +68 27 4.200000000000000e+01 +68 28 4.100000000000000e+01 +68 29 4.000000000000000e+01 +68 30 3.900000000000000e+01 +68 31 3.800000000000000e+01 +68 32 3.700000000000000e+01 +68 33 3.600000000000000e+01 +68 34 3.500000000000000e+01 +68 35 3.400000000000000e+01 +68 36 3.300000000000000e+01 +68 37 3.200000000000000e+01 +68 38 3.100000000000000e+01 +68 39 3.000000000000000e+01 +68 40 2.900000000000000e+01 +68 41 2.800000000000000e+01 +68 42 2.700000000000000e+01 +68 43 2.600000000000000e+01 +68 44 2.500000000000000e+01 +68 45 2.400000000000000e+01 +68 46 2.300000000000000e+01 +68 47 2.200000000000000e+01 +68 48 2.100000000000000e+01 +68 49 2.000000000000000e+01 +68 50 1.900000000000000e+01 +68 51 1.800000000000000e+01 +68 52 1.700000000000000e+01 +68 53 1.600000000000000e+01 +68 54 1.500000000000000e+01 +68 55 1.400000000000000e+01 +68 56 1.300000000000000e+01 +68 57 1.200000000000000e+01 +68 58 1.100000000000000e+01 +68 59 1.000000000000000e+01 +68 60 9.000000000000000e+00 +68 61 8.000000000000000e+00 +68 62 7.000000000000000e+00 +68 63 6.000000000000000e+00 +68 64 5.000000000000000e+00 +68 65 4.000000000000000e+00 +68 66 3.000000000000000e+00 +68 67 2.000000000000000e+00 +68 68 1.000000000000000e+00 +69 1 6.900000000000000e+01 +69 2 6.800000000000000e+01 +69 3 6.700000000000000e+01 +69 4 6.600000000000000e+01 +69 5 6.500000000000000e+01 +69 6 6.400000000000000e+01 +69 7 6.300000000000000e+01 +69 8 6.200000000000000e+01 +69 9 6.100000000000000e+01 +69 10 6.000000000000000e+01 +69 11 5.900000000000000e+01 +69 12 5.800000000000000e+01 +69 13 5.700000000000000e+01 +69 14 5.600000000000000e+01 +69 15 5.500000000000000e+01 +69 16 5.400000000000000e+01 +69 17 5.300000000000000e+01 +69 18 5.200000000000000e+01 +69 19 5.100000000000000e+01 +69 20 5.000000000000000e+01 +69 21 4.900000000000000e+01 +69 22 4.800000000000000e+01 +69 23 4.700000000000000e+01 +69 24 4.600000000000000e+01 +69 25 4.500000000000000e+01 +69 26 4.400000000000000e+01 +69 27 4.300000000000000e+01 +69 28 4.200000000000000e+01 +69 29 4.100000000000000e+01 +69 30 4.000000000000000e+01 +69 31 3.900000000000000e+01 +69 32 3.800000000000000e+01 +69 33 3.700000000000000e+01 +69 34 3.600000000000000e+01 +69 35 3.500000000000000e+01 +69 36 3.400000000000000e+01 +69 37 3.300000000000000e+01 +69 38 3.200000000000000e+01 +69 39 3.100000000000000e+01 +69 40 3.000000000000000e+01 +69 41 2.900000000000000e+01 +69 42 2.800000000000000e+01 +69 43 2.700000000000000e+01 +69 44 2.600000000000000e+01 +69 45 2.500000000000000e+01 +69 46 2.400000000000000e+01 +69 47 2.300000000000000e+01 +69 48 2.200000000000000e+01 +69 49 2.100000000000000e+01 +69 50 2.000000000000000e+01 +69 51 1.900000000000000e+01 +69 52 1.800000000000000e+01 +69 53 1.700000000000000e+01 +69 54 1.600000000000000e+01 +69 55 1.500000000000000e+01 +69 56 1.400000000000000e+01 +69 57 1.300000000000000e+01 +69 58 1.200000000000000e+01 +69 59 1.100000000000000e+01 +69 60 1.000000000000000e+01 +69 61 9.000000000000000e+00 +69 62 8.000000000000000e+00 +69 63 7.000000000000000e+00 +69 64 6.000000000000000e+00 +69 65 5.000000000000000e+00 +69 66 4.000000000000000e+00 +69 67 3.000000000000000e+00 +69 68 2.000000000000000e+00 +69 69 1.000000000000000e+00 +70 1 7.000000000000000e+01 +70 2 6.900000000000000e+01 +70 3 6.800000000000000e+01 +70 4 6.700000000000000e+01 +70 5 6.600000000000000e+01 +70 6 6.500000000000000e+01 +70 7 6.400000000000000e+01 +70 8 6.300000000000000e+01 +70 9 6.200000000000000e+01 +70 10 6.100000000000000e+01 +70 11 6.000000000000000e+01 +70 12 5.900000000000000e+01 +70 13 5.800000000000000e+01 +70 14 5.700000000000000e+01 +70 15 5.600000000000000e+01 +70 16 5.500000000000000e+01 +70 17 5.400000000000000e+01 +70 18 5.300000000000000e+01 +70 19 5.200000000000000e+01 +70 20 5.100000000000000e+01 +70 21 5.000000000000000e+01 +70 22 4.900000000000000e+01 +70 23 4.800000000000000e+01 +70 24 4.700000000000000e+01 +70 25 4.600000000000000e+01 +70 26 4.500000000000000e+01 +70 27 4.400000000000000e+01 +70 28 4.300000000000000e+01 +70 29 4.200000000000000e+01 +70 30 4.100000000000000e+01 +70 31 4.000000000000000e+01 +70 32 3.900000000000000e+01 +70 33 3.800000000000000e+01 +70 34 3.700000000000000e+01 +70 35 3.600000000000000e+01 +70 36 3.500000000000000e+01 +70 37 3.400000000000000e+01 +70 38 3.300000000000000e+01 +70 39 3.200000000000000e+01 +70 40 3.100000000000000e+01 +70 41 3.000000000000000e+01 +70 42 2.900000000000000e+01 +70 43 2.800000000000000e+01 +70 44 2.700000000000000e+01 +70 45 2.600000000000000e+01 +70 46 2.500000000000000e+01 +70 47 2.400000000000000e+01 +70 48 2.300000000000000e+01 +70 49 2.200000000000000e+01 +70 50 2.100000000000000e+01 +70 51 2.000000000000000e+01 +70 52 1.900000000000000e+01 +70 53 1.800000000000000e+01 +70 54 1.700000000000000e+01 +70 55 1.600000000000000e+01 +70 56 1.500000000000000e+01 +70 57 1.400000000000000e+01 +70 58 1.300000000000000e+01 +70 59 1.200000000000000e+01 +70 60 1.100000000000000e+01 +70 61 1.000000000000000e+01 +70 62 9.000000000000000e+00 +70 63 8.000000000000000e+00 +70 64 7.000000000000000e+00 +70 65 6.000000000000000e+00 +70 66 5.000000000000000e+00 +70 67 4.000000000000000e+00 +70 68 3.000000000000000e+00 +70 69 2.000000000000000e+00 +70 70 1.000000000000000e+00 +71 1 7.100000000000000e+01 +71 2 7.000000000000000e+01 +71 3 6.900000000000000e+01 +71 4 6.800000000000000e+01 +71 5 6.700000000000000e+01 +71 6 6.600000000000000e+01 +71 7 6.500000000000000e+01 +71 8 6.400000000000000e+01 +71 9 6.300000000000000e+01 +71 10 6.200000000000000e+01 +71 11 6.100000000000000e+01 +71 12 6.000000000000000e+01 +71 13 5.900000000000000e+01 +71 14 5.800000000000000e+01 +71 15 5.700000000000000e+01 +71 16 5.600000000000000e+01 +71 17 5.500000000000000e+01 +71 18 5.400000000000000e+01 +71 19 5.300000000000000e+01 +71 20 5.200000000000000e+01 +71 21 5.100000000000000e+01 +71 22 5.000000000000000e+01 +71 23 4.900000000000000e+01 +71 24 4.800000000000000e+01 +71 25 4.700000000000000e+01 +71 26 4.600000000000000e+01 +71 27 4.500000000000000e+01 +71 28 4.400000000000000e+01 +71 29 4.300000000000000e+01 +71 30 4.200000000000000e+01 +71 31 4.100000000000000e+01 +71 32 4.000000000000000e+01 +71 33 3.900000000000000e+01 +71 34 3.800000000000000e+01 +71 35 3.700000000000000e+01 +71 36 3.600000000000000e+01 +71 37 3.500000000000000e+01 +71 38 3.400000000000000e+01 +71 39 3.300000000000000e+01 +71 40 3.200000000000000e+01 +71 41 3.100000000000000e+01 +71 42 3.000000000000000e+01 +71 43 2.900000000000000e+01 +71 44 2.800000000000000e+01 +71 45 2.700000000000000e+01 +71 46 2.600000000000000e+01 +71 47 2.500000000000000e+01 +71 48 2.400000000000000e+01 +71 49 2.300000000000000e+01 +71 50 2.200000000000000e+01 +71 51 2.100000000000000e+01 +71 52 2.000000000000000e+01 +71 53 1.900000000000000e+01 +71 54 1.800000000000000e+01 +71 55 1.700000000000000e+01 +71 56 1.600000000000000e+01 +71 57 1.500000000000000e+01 +71 58 1.400000000000000e+01 +71 59 1.300000000000000e+01 +71 60 1.200000000000000e+01 +71 61 1.100000000000000e+01 +71 62 1.000000000000000e+01 +71 63 9.000000000000000e+00 +71 64 8.000000000000000e+00 +71 65 7.000000000000000e+00 +71 66 6.000000000000000e+00 +71 67 5.000000000000000e+00 +71 68 4.000000000000000e+00 +71 69 3.000000000000000e+00 +71 70 2.000000000000000e+00 +71 71 1.000000000000000e+00 +72 1 7.200000000000000e+01 +72 2 7.100000000000000e+01 +72 3 7.000000000000000e+01 +72 4 6.900000000000000e+01 +72 5 6.800000000000000e+01 +72 6 6.700000000000000e+01 +72 7 6.600000000000000e+01 +72 8 6.500000000000000e+01 +72 9 6.400000000000000e+01 +72 10 6.300000000000000e+01 +72 11 6.200000000000000e+01 +72 12 6.100000000000000e+01 +72 13 6.000000000000000e+01 +72 14 5.900000000000000e+01 +72 15 5.800000000000000e+01 +72 16 5.700000000000000e+01 +72 17 5.600000000000000e+01 +72 18 5.500000000000000e+01 +72 19 5.400000000000000e+01 +72 20 5.300000000000000e+01 +72 21 5.200000000000000e+01 +72 22 5.100000000000000e+01 +72 23 5.000000000000000e+01 +72 24 4.900000000000000e+01 +72 25 4.800000000000000e+01 +72 26 4.700000000000000e+01 +72 27 4.600000000000000e+01 +72 28 4.500000000000000e+01 +72 29 4.400000000000000e+01 +72 30 4.300000000000000e+01 +72 31 4.200000000000000e+01 +72 32 4.100000000000000e+01 +72 33 4.000000000000000e+01 +72 34 3.900000000000000e+01 +72 35 3.800000000000000e+01 +72 36 3.700000000000000e+01 +72 37 3.600000000000000e+01 +72 38 3.500000000000000e+01 +72 39 3.400000000000000e+01 +72 40 3.300000000000000e+01 +72 41 3.200000000000000e+01 +72 42 3.100000000000000e+01 +72 43 3.000000000000000e+01 +72 44 2.900000000000000e+01 +72 45 2.800000000000000e+01 +72 46 2.700000000000000e+01 +72 47 2.600000000000000e+01 +72 48 2.500000000000000e+01 +72 49 2.400000000000000e+01 +72 50 2.300000000000000e+01 +72 51 2.200000000000000e+01 +72 52 2.100000000000000e+01 +72 53 2.000000000000000e+01 +72 54 1.900000000000000e+01 +72 55 1.800000000000000e+01 +72 56 1.700000000000000e+01 +72 57 1.600000000000000e+01 +72 58 1.500000000000000e+01 +72 59 1.400000000000000e+01 +72 60 1.300000000000000e+01 +72 61 1.200000000000000e+01 +72 62 1.100000000000000e+01 +72 63 1.000000000000000e+01 +72 64 9.000000000000000e+00 +72 65 8.000000000000000e+00 +72 66 7.000000000000000e+00 +72 67 6.000000000000000e+00 +72 68 5.000000000000000e+00 +72 69 4.000000000000000e+00 +72 70 3.000000000000000e+00 +72 71 2.000000000000000e+00 +72 72 1.000000000000000e+00 +73 1 7.300000000000000e+01 +73 2 7.200000000000000e+01 +73 3 7.100000000000000e+01 +73 4 7.000000000000000e+01 +73 5 6.900000000000000e+01 +73 6 6.800000000000000e+01 +73 7 6.700000000000000e+01 +73 8 6.600000000000000e+01 +73 9 6.500000000000000e+01 +73 10 6.400000000000000e+01 +73 11 6.300000000000000e+01 +73 12 6.200000000000000e+01 +73 13 6.100000000000000e+01 +73 14 6.000000000000000e+01 +73 15 5.900000000000000e+01 +73 16 5.800000000000000e+01 +73 17 5.700000000000000e+01 +73 18 5.600000000000000e+01 +73 19 5.500000000000000e+01 +73 20 5.400000000000000e+01 +73 21 5.300000000000000e+01 +73 22 5.200000000000000e+01 +73 23 5.100000000000000e+01 +73 24 5.000000000000000e+01 +73 25 4.900000000000000e+01 +73 26 4.800000000000000e+01 +73 27 4.700000000000000e+01 +73 28 4.600000000000000e+01 +73 29 4.500000000000000e+01 +73 30 4.400000000000000e+01 +73 31 4.300000000000000e+01 +73 32 4.200000000000000e+01 +73 33 4.100000000000000e+01 +73 34 4.000000000000000e+01 +73 35 3.900000000000000e+01 +73 36 3.800000000000000e+01 +73 37 3.700000000000000e+01 +73 38 3.600000000000000e+01 +73 39 3.500000000000000e+01 +73 40 3.400000000000000e+01 +73 41 3.300000000000000e+01 +73 42 3.200000000000000e+01 +73 43 3.100000000000000e+01 +73 44 3.000000000000000e+01 +73 45 2.900000000000000e+01 +73 46 2.800000000000000e+01 +73 47 2.700000000000000e+01 +73 48 2.600000000000000e+01 +73 49 2.500000000000000e+01 +73 50 2.400000000000000e+01 +73 51 2.300000000000000e+01 +73 52 2.200000000000000e+01 +73 53 2.100000000000000e+01 +73 54 2.000000000000000e+01 +73 55 1.900000000000000e+01 +73 56 1.800000000000000e+01 +73 57 1.700000000000000e+01 +73 58 1.600000000000000e+01 +73 59 1.500000000000000e+01 +73 60 1.400000000000000e+01 +73 61 1.300000000000000e+01 +73 62 1.200000000000000e+01 +73 63 1.100000000000000e+01 +73 64 1.000000000000000e+01 +73 65 9.000000000000000e+00 +73 66 8.000000000000000e+00 +73 67 7.000000000000000e+00 +73 68 6.000000000000000e+00 +73 69 5.000000000000000e+00 +73 70 4.000000000000000e+00 +73 71 3.000000000000000e+00 +73 72 2.000000000000000e+00 +73 73 1.000000000000000e+00 +74 1 7.400000000000000e+01 +74 2 7.300000000000000e+01 +74 3 7.200000000000000e+01 +74 4 7.100000000000000e+01 +74 5 7.000000000000000e+01 +74 6 6.900000000000000e+01 +74 7 6.800000000000000e+01 +74 8 6.700000000000000e+01 +74 9 6.600000000000000e+01 +74 10 6.500000000000000e+01 +74 11 6.400000000000000e+01 +74 12 6.300000000000000e+01 +74 13 6.200000000000000e+01 +74 14 6.100000000000000e+01 +74 15 6.000000000000000e+01 +74 16 5.900000000000000e+01 +74 17 5.800000000000000e+01 +74 18 5.700000000000000e+01 +74 19 5.600000000000000e+01 +74 20 5.500000000000000e+01 +74 21 5.400000000000000e+01 +74 22 5.300000000000000e+01 +74 23 5.200000000000000e+01 +74 24 5.100000000000000e+01 +74 25 5.000000000000000e+01 +74 26 4.900000000000000e+01 +74 27 4.800000000000000e+01 +74 28 4.700000000000000e+01 +74 29 4.600000000000000e+01 +74 30 4.500000000000000e+01 +74 31 4.400000000000000e+01 +74 32 4.300000000000000e+01 +74 33 4.200000000000000e+01 +74 34 4.100000000000000e+01 +74 35 4.000000000000000e+01 +74 36 3.900000000000000e+01 +74 37 3.800000000000000e+01 +74 38 3.700000000000000e+01 +74 39 3.600000000000000e+01 +74 40 3.500000000000000e+01 +74 41 3.400000000000000e+01 +74 42 3.300000000000000e+01 +74 43 3.200000000000000e+01 +74 44 3.100000000000000e+01 +74 45 3.000000000000000e+01 +74 46 2.900000000000000e+01 +74 47 2.800000000000000e+01 +74 48 2.700000000000000e+01 +74 49 2.600000000000000e+01 +74 50 2.500000000000000e+01 +74 51 2.400000000000000e+01 +74 52 2.300000000000000e+01 +74 53 2.200000000000000e+01 +74 54 2.100000000000000e+01 +74 55 2.000000000000000e+01 +74 56 1.900000000000000e+01 +74 57 1.800000000000000e+01 +74 58 1.700000000000000e+01 +74 59 1.600000000000000e+01 +74 60 1.500000000000000e+01 +74 61 1.400000000000000e+01 +74 62 1.300000000000000e+01 +74 63 1.200000000000000e+01 +74 64 1.100000000000000e+01 +74 65 1.000000000000000e+01 +74 66 9.000000000000000e+00 +74 67 8.000000000000000e+00 +74 68 7.000000000000000e+00 +74 69 6.000000000000000e+00 +74 70 5.000000000000000e+00 +74 71 4.000000000000000e+00 +74 72 3.000000000000000e+00 +74 73 2.000000000000000e+00 +74 74 1.000000000000000e+00 +75 1 7.500000000000000e+01 +75 2 7.400000000000000e+01 +75 3 7.300000000000000e+01 +75 4 7.200000000000000e+01 +75 5 7.100000000000000e+01 +75 6 7.000000000000000e+01 +75 7 6.900000000000000e+01 +75 8 6.800000000000000e+01 +75 9 6.700000000000000e+01 +75 10 6.600000000000000e+01 +75 11 6.500000000000000e+01 +75 12 6.400000000000000e+01 +75 13 6.300000000000000e+01 +75 14 6.200000000000000e+01 +75 15 6.100000000000000e+01 +75 16 6.000000000000000e+01 +75 17 5.900000000000000e+01 +75 18 5.800000000000000e+01 +75 19 5.700000000000000e+01 +75 20 5.600000000000000e+01 +75 21 5.500000000000000e+01 +75 22 5.400000000000000e+01 +75 23 5.300000000000000e+01 +75 24 5.200000000000000e+01 +75 25 5.100000000000000e+01 +75 26 5.000000000000000e+01 +75 27 4.900000000000000e+01 +75 28 4.800000000000000e+01 +75 29 4.700000000000000e+01 +75 30 4.600000000000000e+01 +75 31 4.500000000000000e+01 +75 32 4.400000000000000e+01 +75 33 4.300000000000000e+01 +75 34 4.200000000000000e+01 +75 35 4.100000000000000e+01 +75 36 4.000000000000000e+01 +75 37 3.900000000000000e+01 +75 38 3.800000000000000e+01 +75 39 3.700000000000000e+01 +75 40 3.600000000000000e+01 +75 41 3.500000000000000e+01 +75 42 3.400000000000000e+01 +75 43 3.300000000000000e+01 +75 44 3.200000000000000e+01 +75 45 3.100000000000000e+01 +75 46 3.000000000000000e+01 +75 47 2.900000000000000e+01 +75 48 2.800000000000000e+01 +75 49 2.700000000000000e+01 +75 50 2.600000000000000e+01 +75 51 2.500000000000000e+01 +75 52 2.400000000000000e+01 +75 53 2.300000000000000e+01 +75 54 2.200000000000000e+01 +75 55 2.100000000000000e+01 +75 56 2.000000000000000e+01 +75 57 1.900000000000000e+01 +75 58 1.800000000000000e+01 +75 59 1.700000000000000e+01 +75 60 1.600000000000000e+01 +75 61 1.500000000000000e+01 +75 62 1.400000000000000e+01 +75 63 1.300000000000000e+01 +75 64 1.200000000000000e+01 +75 65 1.100000000000000e+01 +75 66 1.000000000000000e+01 +75 67 9.000000000000000e+00 +75 68 8.000000000000000e+00 +75 69 7.000000000000000e+00 +75 70 6.000000000000000e+00 +75 71 5.000000000000000e+00 +75 72 4.000000000000000e+00 +75 73 3.000000000000000e+00 +75 74 2.000000000000000e+00 +75 75 1.000000000000000e+00 +76 1 7.600000000000000e+01 +76 2 7.500000000000000e+01 +76 3 7.400000000000000e+01 +76 4 7.300000000000000e+01 +76 5 7.200000000000000e+01 +76 6 7.100000000000000e+01 +76 7 7.000000000000000e+01 +76 8 6.900000000000000e+01 +76 9 6.800000000000000e+01 +76 10 6.700000000000000e+01 +76 11 6.600000000000000e+01 +76 12 6.500000000000000e+01 +76 13 6.400000000000000e+01 +76 14 6.300000000000000e+01 +76 15 6.200000000000000e+01 +76 16 6.100000000000000e+01 +76 17 6.000000000000000e+01 +76 18 5.900000000000000e+01 +76 19 5.800000000000000e+01 +76 20 5.700000000000000e+01 +76 21 5.600000000000000e+01 +76 22 5.500000000000000e+01 +76 23 5.400000000000000e+01 +76 24 5.300000000000000e+01 +76 25 5.200000000000000e+01 +76 26 5.100000000000000e+01 +76 27 5.000000000000000e+01 +76 28 4.900000000000000e+01 +76 29 4.800000000000000e+01 +76 30 4.700000000000000e+01 +76 31 4.600000000000000e+01 +76 32 4.500000000000000e+01 +76 33 4.400000000000000e+01 +76 34 4.300000000000000e+01 +76 35 4.200000000000000e+01 +76 36 4.100000000000000e+01 +76 37 4.000000000000000e+01 +76 38 3.900000000000000e+01 +76 39 3.800000000000000e+01 +76 40 3.700000000000000e+01 +76 41 3.600000000000000e+01 +76 42 3.500000000000000e+01 +76 43 3.400000000000000e+01 +76 44 3.300000000000000e+01 +76 45 3.200000000000000e+01 +76 46 3.100000000000000e+01 +76 47 3.000000000000000e+01 +76 48 2.900000000000000e+01 +76 49 2.800000000000000e+01 +76 50 2.700000000000000e+01 +76 51 2.600000000000000e+01 +76 52 2.500000000000000e+01 +76 53 2.400000000000000e+01 +76 54 2.300000000000000e+01 +76 55 2.200000000000000e+01 +76 56 2.100000000000000e+01 +76 57 2.000000000000000e+01 +76 58 1.900000000000000e+01 +76 59 1.800000000000000e+01 +76 60 1.700000000000000e+01 +76 61 1.600000000000000e+01 +76 62 1.500000000000000e+01 +76 63 1.400000000000000e+01 +76 64 1.300000000000000e+01 +76 65 1.200000000000000e+01 +76 66 1.100000000000000e+01 +76 67 1.000000000000000e+01 +76 68 9.000000000000000e+00 +76 69 8.000000000000000e+00 +76 70 7.000000000000000e+00 +76 71 6.000000000000000e+00 +76 72 5.000000000000000e+00 +76 73 4.000000000000000e+00 +76 74 3.000000000000000e+00 +76 75 2.000000000000000e+00 +76 76 1.000000000000000e+00 +77 1 7.700000000000000e+01 +77 2 7.600000000000000e+01 +77 3 7.500000000000000e+01 +77 4 7.400000000000000e+01 +77 5 7.300000000000000e+01 +77 6 7.200000000000000e+01 +77 7 7.100000000000000e+01 +77 8 7.000000000000000e+01 +77 9 6.900000000000000e+01 +77 10 6.800000000000000e+01 +77 11 6.700000000000000e+01 +77 12 6.600000000000000e+01 +77 13 6.500000000000000e+01 +77 14 6.400000000000000e+01 +77 15 6.300000000000000e+01 +77 16 6.200000000000000e+01 +77 17 6.100000000000000e+01 +77 18 6.000000000000000e+01 +77 19 5.900000000000000e+01 +77 20 5.800000000000000e+01 +77 21 5.700000000000000e+01 +77 22 5.600000000000000e+01 +77 23 5.500000000000000e+01 +77 24 5.400000000000000e+01 +77 25 5.300000000000000e+01 +77 26 5.200000000000000e+01 +77 27 5.100000000000000e+01 +77 28 5.000000000000000e+01 +77 29 4.900000000000000e+01 +77 30 4.800000000000000e+01 +77 31 4.700000000000000e+01 +77 32 4.600000000000000e+01 +77 33 4.500000000000000e+01 +77 34 4.400000000000000e+01 +77 35 4.300000000000000e+01 +77 36 4.200000000000000e+01 +77 37 4.100000000000000e+01 +77 38 4.000000000000000e+01 +77 39 3.900000000000000e+01 +77 40 3.800000000000000e+01 +77 41 3.700000000000000e+01 +77 42 3.600000000000000e+01 +77 43 3.500000000000000e+01 +77 44 3.400000000000000e+01 +77 45 3.300000000000000e+01 +77 46 3.200000000000000e+01 +77 47 3.100000000000000e+01 +77 48 3.000000000000000e+01 +77 49 2.900000000000000e+01 +77 50 2.800000000000000e+01 +77 51 2.700000000000000e+01 +77 52 2.600000000000000e+01 +77 53 2.500000000000000e+01 +77 54 2.400000000000000e+01 +77 55 2.300000000000000e+01 +77 56 2.200000000000000e+01 +77 57 2.100000000000000e+01 +77 58 2.000000000000000e+01 +77 59 1.900000000000000e+01 +77 60 1.800000000000000e+01 +77 61 1.700000000000000e+01 +77 62 1.600000000000000e+01 +77 63 1.500000000000000e+01 +77 64 1.400000000000000e+01 +77 65 1.300000000000000e+01 +77 66 1.200000000000000e+01 +77 67 1.100000000000000e+01 +77 68 1.000000000000000e+01 +77 69 9.000000000000000e+00 +77 70 8.000000000000000e+00 +77 71 7.000000000000000e+00 +77 72 6.000000000000000e+00 +77 73 5.000000000000000e+00 +77 74 4.000000000000000e+00 +77 75 3.000000000000000e+00 +77 76 2.000000000000000e+00 +77 77 1.000000000000000e+00 +78 1 7.800000000000000e+01 +78 2 7.700000000000000e+01 +78 3 7.600000000000000e+01 +78 4 7.500000000000000e+01 +78 5 7.400000000000000e+01 +78 6 7.300000000000000e+01 +78 7 7.200000000000000e+01 +78 8 7.100000000000000e+01 +78 9 7.000000000000000e+01 +78 10 6.900000000000000e+01 +78 11 6.800000000000000e+01 +78 12 6.700000000000000e+01 +78 13 6.600000000000000e+01 +78 14 6.500000000000000e+01 +78 15 6.400000000000000e+01 +78 16 6.300000000000000e+01 +78 17 6.200000000000000e+01 +78 18 6.100000000000000e+01 +78 19 6.000000000000000e+01 +78 20 5.900000000000000e+01 +78 21 5.800000000000000e+01 +78 22 5.700000000000000e+01 +78 23 5.600000000000000e+01 +78 24 5.500000000000000e+01 +78 25 5.400000000000000e+01 +78 26 5.300000000000000e+01 +78 27 5.200000000000000e+01 +78 28 5.100000000000000e+01 +78 29 5.000000000000000e+01 +78 30 4.900000000000000e+01 +78 31 4.800000000000000e+01 +78 32 4.700000000000000e+01 +78 33 4.600000000000000e+01 +78 34 4.500000000000000e+01 +78 35 4.400000000000000e+01 +78 36 4.300000000000000e+01 +78 37 4.200000000000000e+01 +78 38 4.100000000000000e+01 +78 39 4.000000000000000e+01 +78 40 3.900000000000000e+01 +78 41 3.800000000000000e+01 +78 42 3.700000000000000e+01 +78 43 3.600000000000000e+01 +78 44 3.500000000000000e+01 +78 45 3.400000000000000e+01 +78 46 3.300000000000000e+01 +78 47 3.200000000000000e+01 +78 48 3.100000000000000e+01 +78 49 3.000000000000000e+01 +78 50 2.900000000000000e+01 +78 51 2.800000000000000e+01 +78 52 2.700000000000000e+01 +78 53 2.600000000000000e+01 +78 54 2.500000000000000e+01 +78 55 2.400000000000000e+01 +78 56 2.300000000000000e+01 +78 57 2.200000000000000e+01 +78 58 2.100000000000000e+01 +78 59 2.000000000000000e+01 +78 60 1.900000000000000e+01 +78 61 1.800000000000000e+01 +78 62 1.700000000000000e+01 +78 63 1.600000000000000e+01 +78 64 1.500000000000000e+01 +78 65 1.400000000000000e+01 +78 66 1.300000000000000e+01 +78 67 1.200000000000000e+01 +78 68 1.100000000000000e+01 +78 69 1.000000000000000e+01 +78 70 9.000000000000000e+00 +78 71 8.000000000000000e+00 +78 72 7.000000000000000e+00 +78 73 6.000000000000000e+00 +78 74 5.000000000000000e+00 +78 75 4.000000000000000e+00 +78 76 3.000000000000000e+00 +78 77 2.000000000000000e+00 +78 78 1.000000000000000e+00 +79 1 7.900000000000000e+01 +79 2 7.800000000000000e+01 +79 3 7.700000000000000e+01 +79 4 7.600000000000000e+01 +79 5 7.500000000000000e+01 +79 6 7.400000000000000e+01 +79 7 7.300000000000000e+01 +79 8 7.200000000000000e+01 +79 9 7.100000000000000e+01 +79 10 7.000000000000000e+01 +79 11 6.900000000000000e+01 +79 12 6.800000000000000e+01 +79 13 6.700000000000000e+01 +79 14 6.600000000000000e+01 +79 15 6.500000000000000e+01 +79 16 6.400000000000000e+01 +79 17 6.300000000000000e+01 +79 18 6.200000000000000e+01 +79 19 6.100000000000000e+01 +79 20 6.000000000000000e+01 +79 21 5.900000000000000e+01 +79 22 5.800000000000000e+01 +79 23 5.700000000000000e+01 +79 24 5.600000000000000e+01 +79 25 5.500000000000000e+01 +79 26 5.400000000000000e+01 +79 27 5.300000000000000e+01 +79 28 5.200000000000000e+01 +79 29 5.100000000000000e+01 +79 30 5.000000000000000e+01 +79 31 4.900000000000000e+01 +79 32 4.800000000000000e+01 +79 33 4.700000000000000e+01 +79 34 4.600000000000000e+01 +79 35 4.500000000000000e+01 +79 36 4.400000000000000e+01 +79 37 4.300000000000000e+01 +79 38 4.200000000000000e+01 +79 39 4.100000000000000e+01 +79 40 4.000000000000000e+01 +79 41 3.900000000000000e+01 +79 42 3.800000000000000e+01 +79 43 3.700000000000000e+01 +79 44 3.600000000000000e+01 +79 45 3.500000000000000e+01 +79 46 3.400000000000000e+01 +79 47 3.300000000000000e+01 +79 48 3.200000000000000e+01 +79 49 3.100000000000000e+01 +79 50 3.000000000000000e+01 +79 51 2.900000000000000e+01 +79 52 2.800000000000000e+01 +79 53 2.700000000000000e+01 +79 54 2.600000000000000e+01 +79 55 2.500000000000000e+01 +79 56 2.400000000000000e+01 +79 57 2.300000000000000e+01 +79 58 2.200000000000000e+01 +79 59 2.100000000000000e+01 +79 60 2.000000000000000e+01 +79 61 1.900000000000000e+01 +79 62 1.800000000000000e+01 +79 63 1.700000000000000e+01 +79 64 1.600000000000000e+01 +79 65 1.500000000000000e+01 +79 66 1.400000000000000e+01 +79 67 1.300000000000000e+01 +79 68 1.200000000000000e+01 +79 69 1.100000000000000e+01 +79 70 1.000000000000000e+01 +79 71 9.000000000000000e+00 +79 72 8.000000000000000e+00 +79 73 7.000000000000000e+00 +79 74 6.000000000000000e+00 +79 75 5.000000000000000e+00 +79 76 4.000000000000000e+00 +79 77 3.000000000000000e+00 +79 78 2.000000000000000e+00 +79 79 1.000000000000000e+00 +80 1 8.000000000000000e+01 +80 2 7.900000000000000e+01 +80 3 7.800000000000000e+01 +80 4 7.700000000000000e+01 +80 5 7.600000000000000e+01 +80 6 7.500000000000000e+01 +80 7 7.400000000000000e+01 +80 8 7.300000000000000e+01 +80 9 7.200000000000000e+01 +80 10 7.100000000000000e+01 +80 11 7.000000000000000e+01 +80 12 6.900000000000000e+01 +80 13 6.800000000000000e+01 +80 14 6.700000000000000e+01 +80 15 6.600000000000000e+01 +80 16 6.500000000000000e+01 +80 17 6.400000000000000e+01 +80 18 6.300000000000000e+01 +80 19 6.200000000000000e+01 +80 20 6.100000000000000e+01 +80 21 6.000000000000000e+01 +80 22 5.900000000000000e+01 +80 23 5.800000000000000e+01 +80 24 5.700000000000000e+01 +80 25 5.600000000000000e+01 +80 26 5.500000000000000e+01 +80 27 5.400000000000000e+01 +80 28 5.300000000000000e+01 +80 29 5.200000000000000e+01 +80 30 5.100000000000000e+01 +80 31 5.000000000000000e+01 +80 32 4.900000000000000e+01 +80 33 4.800000000000000e+01 +80 34 4.700000000000000e+01 +80 35 4.600000000000000e+01 +80 36 4.500000000000000e+01 +80 37 4.400000000000000e+01 +80 38 4.300000000000000e+01 +80 39 4.200000000000000e+01 +80 40 4.100000000000000e+01 +80 41 4.000000000000000e+01 +80 42 3.900000000000000e+01 +80 43 3.800000000000000e+01 +80 44 3.700000000000000e+01 +80 45 3.600000000000000e+01 +80 46 3.500000000000000e+01 +80 47 3.400000000000000e+01 +80 48 3.300000000000000e+01 +80 49 3.200000000000000e+01 +80 50 3.100000000000000e+01 +80 51 3.000000000000000e+01 +80 52 2.900000000000000e+01 +80 53 2.800000000000000e+01 +80 54 2.700000000000000e+01 +80 55 2.600000000000000e+01 +80 56 2.500000000000000e+01 +80 57 2.400000000000000e+01 +80 58 2.300000000000000e+01 +80 59 2.200000000000000e+01 +80 60 2.100000000000000e+01 +80 61 2.000000000000000e+01 +80 62 1.900000000000000e+01 +80 63 1.800000000000000e+01 +80 64 1.700000000000000e+01 +80 65 1.600000000000000e+01 +80 66 1.500000000000000e+01 +80 67 1.400000000000000e+01 +80 68 1.300000000000000e+01 +80 69 1.200000000000000e+01 +80 70 1.100000000000000e+01 +80 71 1.000000000000000e+01 +80 72 9.000000000000000e+00 +80 73 8.000000000000000e+00 +80 74 7.000000000000000e+00 +80 75 6.000000000000000e+00 +80 76 5.000000000000000e+00 +80 77 4.000000000000000e+00 +80 78 3.000000000000000e+00 +80 79 2.000000000000000e+00 +80 80 1.000000000000000e+00 +81 1 8.100000000000000e+01 +81 2 8.000000000000000e+01 +81 3 7.900000000000000e+01 +81 4 7.800000000000000e+01 +81 5 7.700000000000000e+01 +81 6 7.600000000000000e+01 +81 7 7.500000000000000e+01 +81 8 7.400000000000000e+01 +81 9 7.300000000000000e+01 +81 10 7.200000000000000e+01 +81 11 7.100000000000000e+01 +81 12 7.000000000000000e+01 +81 13 6.900000000000000e+01 +81 14 6.800000000000000e+01 +81 15 6.700000000000000e+01 +81 16 6.600000000000000e+01 +81 17 6.500000000000000e+01 +81 18 6.400000000000000e+01 +81 19 6.300000000000000e+01 +81 20 6.200000000000000e+01 +81 21 6.100000000000000e+01 +81 22 6.000000000000000e+01 +81 23 5.900000000000000e+01 +81 24 5.800000000000000e+01 +81 25 5.700000000000000e+01 +81 26 5.600000000000000e+01 +81 27 5.500000000000000e+01 +81 28 5.400000000000000e+01 +81 29 5.300000000000000e+01 +81 30 5.200000000000000e+01 +81 31 5.100000000000000e+01 +81 32 5.000000000000000e+01 +81 33 4.900000000000000e+01 +81 34 4.800000000000000e+01 +81 35 4.700000000000000e+01 +81 36 4.600000000000000e+01 +81 37 4.500000000000000e+01 +81 38 4.400000000000000e+01 +81 39 4.300000000000000e+01 +81 40 4.200000000000000e+01 +81 41 4.100000000000000e+01 +81 42 4.000000000000000e+01 +81 43 3.900000000000000e+01 +81 44 3.800000000000000e+01 +81 45 3.700000000000000e+01 +81 46 3.600000000000000e+01 +81 47 3.500000000000000e+01 +81 48 3.400000000000000e+01 +81 49 3.300000000000000e+01 +81 50 3.200000000000000e+01 +81 51 3.100000000000000e+01 +81 52 3.000000000000000e+01 +81 53 2.900000000000000e+01 +81 54 2.800000000000000e+01 +81 55 2.700000000000000e+01 +81 56 2.600000000000000e+01 +81 57 2.500000000000000e+01 +81 58 2.400000000000000e+01 +81 59 2.300000000000000e+01 +81 60 2.200000000000000e+01 +81 61 2.100000000000000e+01 +81 62 2.000000000000000e+01 +81 63 1.900000000000000e+01 +81 64 1.800000000000000e+01 +81 65 1.700000000000000e+01 +81 66 1.600000000000000e+01 +81 67 1.500000000000000e+01 +81 68 1.400000000000000e+01 +81 69 1.300000000000000e+01 +81 70 1.200000000000000e+01 +81 71 1.100000000000000e+01 +81 72 1.000000000000000e+01 +81 73 9.000000000000000e+00 +81 74 8.000000000000000e+00 +81 75 7.000000000000000e+00 +81 76 6.000000000000000e+00 +81 77 5.000000000000000e+00 +81 78 4.000000000000000e+00 +81 79 3.000000000000000e+00 +81 80 2.000000000000000e+00 +81 81 1.000000000000000e+00 +82 1 8.200000000000000e+01 +82 2 8.100000000000000e+01 +82 3 8.000000000000000e+01 +82 4 7.900000000000000e+01 +82 5 7.800000000000000e+01 +82 6 7.700000000000000e+01 +82 7 7.600000000000000e+01 +82 8 7.500000000000000e+01 +82 9 7.400000000000000e+01 +82 10 7.300000000000000e+01 +82 11 7.200000000000000e+01 +82 12 7.100000000000000e+01 +82 13 7.000000000000000e+01 +82 14 6.900000000000000e+01 +82 15 6.800000000000000e+01 +82 16 6.700000000000000e+01 +82 17 6.600000000000000e+01 +82 18 6.500000000000000e+01 +82 19 6.400000000000000e+01 +82 20 6.300000000000000e+01 +82 21 6.200000000000000e+01 +82 22 6.100000000000000e+01 +82 23 6.000000000000000e+01 +82 24 5.900000000000000e+01 +82 25 5.800000000000000e+01 +82 26 5.700000000000000e+01 +82 27 5.600000000000000e+01 +82 28 5.500000000000000e+01 +82 29 5.400000000000000e+01 +82 30 5.300000000000000e+01 +82 31 5.200000000000000e+01 +82 32 5.100000000000000e+01 +82 33 5.000000000000000e+01 +82 34 4.900000000000000e+01 +82 35 4.800000000000000e+01 +82 36 4.700000000000000e+01 +82 37 4.600000000000000e+01 +82 38 4.500000000000000e+01 +82 39 4.400000000000000e+01 +82 40 4.300000000000000e+01 +82 41 4.200000000000000e+01 +82 42 4.100000000000000e+01 +82 43 4.000000000000000e+01 +82 44 3.900000000000000e+01 +82 45 3.800000000000000e+01 +82 46 3.700000000000000e+01 +82 47 3.600000000000000e+01 +82 48 3.500000000000000e+01 +82 49 3.400000000000000e+01 +82 50 3.300000000000000e+01 +82 51 3.200000000000000e+01 +82 52 3.100000000000000e+01 +82 53 3.000000000000000e+01 +82 54 2.900000000000000e+01 +82 55 2.800000000000000e+01 +82 56 2.700000000000000e+01 +82 57 2.600000000000000e+01 +82 58 2.500000000000000e+01 +82 59 2.400000000000000e+01 +82 60 2.300000000000000e+01 +82 61 2.200000000000000e+01 +82 62 2.100000000000000e+01 +82 63 2.000000000000000e+01 +82 64 1.900000000000000e+01 +82 65 1.800000000000000e+01 +82 66 1.700000000000000e+01 +82 67 1.600000000000000e+01 +82 68 1.500000000000000e+01 +82 69 1.400000000000000e+01 +82 70 1.300000000000000e+01 +82 71 1.200000000000000e+01 +82 72 1.100000000000000e+01 +82 73 1.000000000000000e+01 +82 74 9.000000000000000e+00 +82 75 8.000000000000000e+00 +82 76 7.000000000000000e+00 +82 77 6.000000000000000e+00 +82 78 5.000000000000000e+00 +82 79 4.000000000000000e+00 +82 80 3.000000000000000e+00 +82 81 2.000000000000000e+00 +82 82 1.000000000000000e+00 +83 1 8.300000000000000e+01 +83 2 8.200000000000000e+01 +83 3 8.100000000000000e+01 +83 4 8.000000000000000e+01 +83 5 7.900000000000000e+01 +83 6 7.800000000000000e+01 +83 7 7.700000000000000e+01 +83 8 7.600000000000000e+01 +83 9 7.500000000000000e+01 +83 10 7.400000000000000e+01 +83 11 7.300000000000000e+01 +83 12 7.200000000000000e+01 +83 13 7.100000000000000e+01 +83 14 7.000000000000000e+01 +83 15 6.900000000000000e+01 +83 16 6.800000000000000e+01 +83 17 6.700000000000000e+01 +83 18 6.600000000000000e+01 +83 19 6.500000000000000e+01 +83 20 6.400000000000000e+01 +83 21 6.300000000000000e+01 +83 22 6.200000000000000e+01 +83 23 6.100000000000000e+01 +83 24 6.000000000000000e+01 +83 25 5.900000000000000e+01 +83 26 5.800000000000000e+01 +83 27 5.700000000000000e+01 +83 28 5.600000000000000e+01 +83 29 5.500000000000000e+01 +83 30 5.400000000000000e+01 +83 31 5.300000000000000e+01 +83 32 5.200000000000000e+01 +83 33 5.100000000000000e+01 +83 34 5.000000000000000e+01 +83 35 4.900000000000000e+01 +83 36 4.800000000000000e+01 +83 37 4.700000000000000e+01 +83 38 4.600000000000000e+01 +83 39 4.500000000000000e+01 +83 40 4.400000000000000e+01 +83 41 4.300000000000000e+01 +83 42 4.200000000000000e+01 +83 43 4.100000000000000e+01 +83 44 4.000000000000000e+01 +83 45 3.900000000000000e+01 +83 46 3.800000000000000e+01 +83 47 3.700000000000000e+01 +83 48 3.600000000000000e+01 +83 49 3.500000000000000e+01 +83 50 3.400000000000000e+01 +83 51 3.300000000000000e+01 +83 52 3.200000000000000e+01 +83 53 3.100000000000000e+01 +83 54 3.000000000000000e+01 +83 55 2.900000000000000e+01 +83 56 2.800000000000000e+01 +83 57 2.700000000000000e+01 +83 58 2.600000000000000e+01 +83 59 2.500000000000000e+01 +83 60 2.400000000000000e+01 +83 61 2.300000000000000e+01 +83 62 2.200000000000000e+01 +83 63 2.100000000000000e+01 +83 64 2.000000000000000e+01 +83 65 1.900000000000000e+01 +83 66 1.800000000000000e+01 +83 67 1.700000000000000e+01 +83 68 1.600000000000000e+01 +83 69 1.500000000000000e+01 +83 70 1.400000000000000e+01 +83 71 1.300000000000000e+01 +83 72 1.200000000000000e+01 +83 73 1.100000000000000e+01 +83 74 1.000000000000000e+01 +83 75 9.000000000000000e+00 +83 76 8.000000000000000e+00 +83 77 7.000000000000000e+00 +83 78 6.000000000000000e+00 +83 79 5.000000000000000e+00 +83 80 4.000000000000000e+00 +83 81 3.000000000000000e+00 +83 82 2.000000000000000e+00 +83 83 1.000000000000000e+00 +84 1 8.400000000000000e+01 +84 2 8.300000000000000e+01 +84 3 8.200000000000000e+01 +84 4 8.100000000000000e+01 +84 5 8.000000000000000e+01 +84 6 7.900000000000000e+01 +84 7 7.800000000000000e+01 +84 8 7.700000000000000e+01 +84 9 7.600000000000000e+01 +84 10 7.500000000000000e+01 +84 11 7.400000000000000e+01 +84 12 7.300000000000000e+01 +84 13 7.200000000000000e+01 +84 14 7.100000000000000e+01 +84 15 7.000000000000000e+01 +84 16 6.900000000000000e+01 +84 17 6.800000000000000e+01 +84 18 6.700000000000000e+01 +84 19 6.600000000000000e+01 +84 20 6.500000000000000e+01 +84 21 6.400000000000000e+01 +84 22 6.300000000000000e+01 +84 23 6.200000000000000e+01 +84 24 6.100000000000000e+01 +84 25 6.000000000000000e+01 +84 26 5.900000000000000e+01 +84 27 5.800000000000000e+01 +84 28 5.700000000000000e+01 +84 29 5.600000000000000e+01 +84 30 5.500000000000000e+01 +84 31 5.400000000000000e+01 +84 32 5.300000000000000e+01 +84 33 5.200000000000000e+01 +84 34 5.100000000000000e+01 +84 35 5.000000000000000e+01 +84 36 4.900000000000000e+01 +84 37 4.800000000000000e+01 +84 38 4.700000000000000e+01 +84 39 4.600000000000000e+01 +84 40 4.500000000000000e+01 +84 41 4.400000000000000e+01 +84 42 4.300000000000000e+01 +84 43 4.200000000000000e+01 +84 44 4.100000000000000e+01 +84 45 4.000000000000000e+01 +84 46 3.900000000000000e+01 +84 47 3.800000000000000e+01 +84 48 3.700000000000000e+01 +84 49 3.600000000000000e+01 +84 50 3.500000000000000e+01 +84 51 3.400000000000000e+01 +84 52 3.300000000000000e+01 +84 53 3.200000000000000e+01 +84 54 3.100000000000000e+01 +84 55 3.000000000000000e+01 +84 56 2.900000000000000e+01 +84 57 2.800000000000000e+01 +84 58 2.700000000000000e+01 +84 59 2.600000000000000e+01 +84 60 2.500000000000000e+01 +84 61 2.400000000000000e+01 +84 62 2.300000000000000e+01 +84 63 2.200000000000000e+01 +84 64 2.100000000000000e+01 +84 65 2.000000000000000e+01 +84 66 1.900000000000000e+01 +84 67 1.800000000000000e+01 +84 68 1.700000000000000e+01 +84 69 1.600000000000000e+01 +84 70 1.500000000000000e+01 +84 71 1.400000000000000e+01 +84 72 1.300000000000000e+01 +84 73 1.200000000000000e+01 +84 74 1.100000000000000e+01 +84 75 1.000000000000000e+01 +84 76 9.000000000000000e+00 +84 77 8.000000000000000e+00 +84 78 7.000000000000000e+00 +84 79 6.000000000000000e+00 +84 80 5.000000000000000e+00 +84 81 4.000000000000000e+00 +84 82 3.000000000000000e+00 +84 83 2.000000000000000e+00 +84 84 1.000000000000000e+00 +85 1 8.500000000000000e+01 +85 2 8.400000000000000e+01 +85 3 8.300000000000000e+01 +85 4 8.200000000000000e+01 +85 5 8.100000000000000e+01 +85 6 8.000000000000000e+01 +85 7 7.900000000000000e+01 +85 8 7.800000000000000e+01 +85 9 7.700000000000000e+01 +85 10 7.600000000000000e+01 +85 11 7.500000000000000e+01 +85 12 7.400000000000000e+01 +85 13 7.300000000000000e+01 +85 14 7.200000000000000e+01 +85 15 7.100000000000000e+01 +85 16 7.000000000000000e+01 +85 17 6.900000000000000e+01 +85 18 6.800000000000000e+01 +85 19 6.700000000000000e+01 +85 20 6.600000000000000e+01 +85 21 6.500000000000000e+01 +85 22 6.400000000000000e+01 +85 23 6.300000000000000e+01 +85 24 6.200000000000000e+01 +85 25 6.100000000000000e+01 +85 26 6.000000000000000e+01 +85 27 5.900000000000000e+01 +85 28 5.800000000000000e+01 +85 29 5.700000000000000e+01 +85 30 5.600000000000000e+01 +85 31 5.500000000000000e+01 +85 32 5.400000000000000e+01 +85 33 5.300000000000000e+01 +85 34 5.200000000000000e+01 +85 35 5.100000000000000e+01 +85 36 5.000000000000000e+01 +85 37 4.900000000000000e+01 +85 38 4.800000000000000e+01 +85 39 4.700000000000000e+01 +85 40 4.600000000000000e+01 +85 41 4.500000000000000e+01 +85 42 4.400000000000000e+01 +85 43 4.300000000000000e+01 +85 44 4.200000000000000e+01 +85 45 4.100000000000000e+01 +85 46 4.000000000000000e+01 +85 47 3.900000000000000e+01 +85 48 3.800000000000000e+01 +85 49 3.700000000000000e+01 +85 50 3.600000000000000e+01 +85 51 3.500000000000000e+01 +85 52 3.400000000000000e+01 +85 53 3.300000000000000e+01 +85 54 3.200000000000000e+01 +85 55 3.100000000000000e+01 +85 56 3.000000000000000e+01 +85 57 2.900000000000000e+01 +85 58 2.800000000000000e+01 +85 59 2.700000000000000e+01 +85 60 2.600000000000000e+01 +85 61 2.500000000000000e+01 +85 62 2.400000000000000e+01 +85 63 2.300000000000000e+01 +85 64 2.200000000000000e+01 +85 65 2.100000000000000e+01 +85 66 2.000000000000000e+01 +85 67 1.900000000000000e+01 +85 68 1.800000000000000e+01 +85 69 1.700000000000000e+01 +85 70 1.600000000000000e+01 +85 71 1.500000000000000e+01 +85 72 1.400000000000000e+01 +85 73 1.300000000000000e+01 +85 74 1.200000000000000e+01 +85 75 1.100000000000000e+01 +85 76 1.000000000000000e+01 +85 77 9.000000000000000e+00 +85 78 8.000000000000000e+00 +85 79 7.000000000000000e+00 +85 80 6.000000000000000e+00 +85 81 5.000000000000000e+00 +85 82 4.000000000000000e+00 +85 83 3.000000000000000e+00 +85 84 2.000000000000000e+00 +85 85 1.000000000000000e+00 +86 1 8.600000000000000e+01 +86 2 8.500000000000000e+01 +86 3 8.400000000000000e+01 +86 4 8.300000000000000e+01 +86 5 8.200000000000000e+01 +86 6 8.100000000000000e+01 +86 7 8.000000000000000e+01 +86 8 7.900000000000000e+01 +86 9 7.800000000000000e+01 +86 10 7.700000000000000e+01 +86 11 7.600000000000000e+01 +86 12 7.500000000000000e+01 +86 13 7.400000000000000e+01 +86 14 7.300000000000000e+01 +86 15 7.200000000000000e+01 +86 16 7.100000000000000e+01 +86 17 7.000000000000000e+01 +86 18 6.900000000000000e+01 +86 19 6.800000000000000e+01 +86 20 6.700000000000000e+01 +86 21 6.600000000000000e+01 +86 22 6.500000000000000e+01 +86 23 6.400000000000000e+01 +86 24 6.300000000000000e+01 +86 25 6.200000000000000e+01 +86 26 6.100000000000000e+01 +86 27 6.000000000000000e+01 +86 28 5.900000000000000e+01 +86 29 5.800000000000000e+01 +86 30 5.700000000000000e+01 +86 31 5.600000000000000e+01 +86 32 5.500000000000000e+01 +86 33 5.400000000000000e+01 +86 34 5.300000000000000e+01 +86 35 5.200000000000000e+01 +86 36 5.100000000000000e+01 +86 37 5.000000000000000e+01 +86 38 4.900000000000000e+01 +86 39 4.800000000000000e+01 +86 40 4.700000000000000e+01 +86 41 4.600000000000000e+01 +86 42 4.500000000000000e+01 +86 43 4.400000000000000e+01 +86 44 4.300000000000000e+01 +86 45 4.200000000000000e+01 +86 46 4.100000000000000e+01 +86 47 4.000000000000000e+01 +86 48 3.900000000000000e+01 +86 49 3.800000000000000e+01 +86 50 3.700000000000000e+01 +86 51 3.600000000000000e+01 +86 52 3.500000000000000e+01 +86 53 3.400000000000000e+01 +86 54 3.300000000000000e+01 +86 55 3.200000000000000e+01 +86 56 3.100000000000000e+01 +86 57 3.000000000000000e+01 +86 58 2.900000000000000e+01 +86 59 2.800000000000000e+01 +86 60 2.700000000000000e+01 +86 61 2.600000000000000e+01 +86 62 2.500000000000000e+01 +86 63 2.400000000000000e+01 +86 64 2.300000000000000e+01 +86 65 2.200000000000000e+01 +86 66 2.100000000000000e+01 +86 67 2.000000000000000e+01 +86 68 1.900000000000000e+01 +86 69 1.800000000000000e+01 +86 70 1.700000000000000e+01 +86 71 1.600000000000000e+01 +86 72 1.500000000000000e+01 +86 73 1.400000000000000e+01 +86 74 1.300000000000000e+01 +86 75 1.200000000000000e+01 +86 76 1.100000000000000e+01 +86 77 1.000000000000000e+01 +86 78 9.000000000000000e+00 +86 79 8.000000000000000e+00 +86 80 7.000000000000000e+00 +86 81 6.000000000000000e+00 +86 82 5.000000000000000e+00 +86 83 4.000000000000000e+00 +86 84 3.000000000000000e+00 +86 85 2.000000000000000e+00 +86 86 1.000000000000000e+00 +87 1 8.700000000000000e+01 +87 2 8.600000000000000e+01 +87 3 8.500000000000000e+01 +87 4 8.400000000000000e+01 +87 5 8.300000000000000e+01 +87 6 8.200000000000000e+01 +87 7 8.100000000000000e+01 +87 8 8.000000000000000e+01 +87 9 7.900000000000000e+01 +87 10 7.800000000000000e+01 +87 11 7.700000000000000e+01 +87 12 7.600000000000000e+01 +87 13 7.500000000000000e+01 +87 14 7.400000000000000e+01 +87 15 7.300000000000000e+01 +87 16 7.200000000000000e+01 +87 17 7.100000000000000e+01 +87 18 7.000000000000000e+01 +87 19 6.900000000000000e+01 +87 20 6.800000000000000e+01 +87 21 6.700000000000000e+01 +87 22 6.600000000000000e+01 +87 23 6.500000000000000e+01 +87 24 6.400000000000000e+01 +87 25 6.300000000000000e+01 +87 26 6.200000000000000e+01 +87 27 6.100000000000000e+01 +87 28 6.000000000000000e+01 +87 29 5.900000000000000e+01 +87 30 5.800000000000000e+01 +87 31 5.700000000000000e+01 +87 32 5.600000000000000e+01 +87 33 5.500000000000000e+01 +87 34 5.400000000000000e+01 +87 35 5.300000000000000e+01 +87 36 5.200000000000000e+01 +87 37 5.100000000000000e+01 +87 38 5.000000000000000e+01 +87 39 4.900000000000000e+01 +87 40 4.800000000000000e+01 +87 41 4.700000000000000e+01 +87 42 4.600000000000000e+01 +87 43 4.500000000000000e+01 +87 44 4.400000000000000e+01 +87 45 4.300000000000000e+01 +87 46 4.200000000000000e+01 +87 47 4.100000000000000e+01 +87 48 4.000000000000000e+01 +87 49 3.900000000000000e+01 +87 50 3.800000000000000e+01 +87 51 3.700000000000000e+01 +87 52 3.600000000000000e+01 +87 53 3.500000000000000e+01 +87 54 3.400000000000000e+01 +87 55 3.300000000000000e+01 +87 56 3.200000000000000e+01 +87 57 3.100000000000000e+01 +87 58 3.000000000000000e+01 +87 59 2.900000000000000e+01 +87 60 2.800000000000000e+01 +87 61 2.700000000000000e+01 +87 62 2.600000000000000e+01 +87 63 2.500000000000000e+01 +87 64 2.400000000000000e+01 +87 65 2.300000000000000e+01 +87 66 2.200000000000000e+01 +87 67 2.100000000000000e+01 +87 68 2.000000000000000e+01 +87 69 1.900000000000000e+01 +87 70 1.800000000000000e+01 +87 71 1.700000000000000e+01 +87 72 1.600000000000000e+01 +87 73 1.500000000000000e+01 +87 74 1.400000000000000e+01 +87 75 1.300000000000000e+01 +87 76 1.200000000000000e+01 +87 77 1.100000000000000e+01 +87 78 1.000000000000000e+01 +87 79 9.000000000000000e+00 +87 80 8.000000000000000e+00 +87 81 7.000000000000000e+00 +87 82 6.000000000000000e+00 +87 83 5.000000000000000e+00 +87 84 4.000000000000000e+00 +87 85 3.000000000000000e+00 +87 86 2.000000000000000e+00 +87 87 1.000000000000000e+00 +88 1 8.800000000000000e+01 +88 2 8.700000000000000e+01 +88 3 8.600000000000000e+01 +88 4 8.500000000000000e+01 +88 5 8.400000000000000e+01 +88 6 8.300000000000000e+01 +88 7 8.200000000000000e+01 +88 8 8.100000000000000e+01 +88 9 8.000000000000000e+01 +88 10 7.900000000000000e+01 +88 11 7.800000000000000e+01 +88 12 7.700000000000000e+01 +88 13 7.600000000000000e+01 +88 14 7.500000000000000e+01 +88 15 7.400000000000000e+01 +88 16 7.300000000000000e+01 +88 17 7.200000000000000e+01 +88 18 7.100000000000000e+01 +88 19 7.000000000000000e+01 +88 20 6.900000000000000e+01 +88 21 6.800000000000000e+01 +88 22 6.700000000000000e+01 +88 23 6.600000000000000e+01 +88 24 6.500000000000000e+01 +88 25 6.400000000000000e+01 +88 26 6.300000000000000e+01 +88 27 6.200000000000000e+01 +88 28 6.100000000000000e+01 +88 29 6.000000000000000e+01 +88 30 5.900000000000000e+01 +88 31 5.800000000000000e+01 +88 32 5.700000000000000e+01 +88 33 5.600000000000000e+01 +88 34 5.500000000000000e+01 +88 35 5.400000000000000e+01 +88 36 5.300000000000000e+01 +88 37 5.200000000000000e+01 +88 38 5.100000000000000e+01 +88 39 5.000000000000000e+01 +88 40 4.900000000000000e+01 +88 41 4.800000000000000e+01 +88 42 4.700000000000000e+01 +88 43 4.600000000000000e+01 +88 44 4.500000000000000e+01 +88 45 4.400000000000000e+01 +88 46 4.300000000000000e+01 +88 47 4.200000000000000e+01 +88 48 4.100000000000000e+01 +88 49 4.000000000000000e+01 +88 50 3.900000000000000e+01 +88 51 3.800000000000000e+01 +88 52 3.700000000000000e+01 +88 53 3.600000000000000e+01 +88 54 3.500000000000000e+01 +88 55 3.400000000000000e+01 +88 56 3.300000000000000e+01 +88 57 3.200000000000000e+01 +88 58 3.100000000000000e+01 +88 59 3.000000000000000e+01 +88 60 2.900000000000000e+01 +88 61 2.800000000000000e+01 +88 62 2.700000000000000e+01 +88 63 2.600000000000000e+01 +88 64 2.500000000000000e+01 +88 65 2.400000000000000e+01 +88 66 2.300000000000000e+01 +88 67 2.200000000000000e+01 +88 68 2.100000000000000e+01 +88 69 2.000000000000000e+01 +88 70 1.900000000000000e+01 +88 71 1.800000000000000e+01 +88 72 1.700000000000000e+01 +88 73 1.600000000000000e+01 +88 74 1.500000000000000e+01 +88 75 1.400000000000000e+01 +88 76 1.300000000000000e+01 +88 77 1.200000000000000e+01 +88 78 1.100000000000000e+01 +88 79 1.000000000000000e+01 +88 80 9.000000000000000e+00 +88 81 8.000000000000000e+00 +88 82 7.000000000000000e+00 +88 83 6.000000000000000e+00 +88 84 5.000000000000000e+00 +88 85 4.000000000000000e+00 +88 86 3.000000000000000e+00 +88 87 2.000000000000000e+00 +88 88 1.000000000000000e+00 +89 1 8.900000000000000e+01 +89 2 8.800000000000000e+01 +89 3 8.700000000000000e+01 +89 4 8.600000000000000e+01 +89 5 8.500000000000000e+01 +89 6 8.400000000000000e+01 +89 7 8.300000000000000e+01 +89 8 8.200000000000000e+01 +89 9 8.100000000000000e+01 +89 10 8.000000000000000e+01 +89 11 7.900000000000000e+01 +89 12 7.800000000000000e+01 +89 13 7.700000000000000e+01 +89 14 7.600000000000000e+01 +89 15 7.500000000000000e+01 +89 16 7.400000000000000e+01 +89 17 7.300000000000000e+01 +89 18 7.200000000000000e+01 +89 19 7.100000000000000e+01 +89 20 7.000000000000000e+01 +89 21 6.900000000000000e+01 +89 22 6.800000000000000e+01 +89 23 6.700000000000000e+01 +89 24 6.600000000000000e+01 +89 25 6.500000000000000e+01 +89 26 6.400000000000000e+01 +89 27 6.300000000000000e+01 +89 28 6.200000000000000e+01 +89 29 6.100000000000000e+01 +89 30 6.000000000000000e+01 +89 31 5.900000000000000e+01 +89 32 5.800000000000000e+01 +89 33 5.700000000000000e+01 +89 34 5.600000000000000e+01 +89 35 5.500000000000000e+01 +89 36 5.400000000000000e+01 +89 37 5.300000000000000e+01 +89 38 5.200000000000000e+01 +89 39 5.100000000000000e+01 +89 40 5.000000000000000e+01 +89 41 4.900000000000000e+01 +89 42 4.800000000000000e+01 +89 43 4.700000000000000e+01 +89 44 4.600000000000000e+01 +89 45 4.500000000000000e+01 +89 46 4.400000000000000e+01 +89 47 4.300000000000000e+01 +89 48 4.200000000000000e+01 +89 49 4.100000000000000e+01 +89 50 4.000000000000000e+01 +89 51 3.900000000000000e+01 +89 52 3.800000000000000e+01 +89 53 3.700000000000000e+01 +89 54 3.600000000000000e+01 +89 55 3.500000000000000e+01 +89 56 3.400000000000000e+01 +89 57 3.300000000000000e+01 +89 58 3.200000000000000e+01 +89 59 3.100000000000000e+01 +89 60 3.000000000000000e+01 +89 61 2.900000000000000e+01 +89 62 2.800000000000000e+01 +89 63 2.700000000000000e+01 +89 64 2.600000000000000e+01 +89 65 2.500000000000000e+01 +89 66 2.400000000000000e+01 +89 67 2.300000000000000e+01 +89 68 2.200000000000000e+01 +89 69 2.100000000000000e+01 +89 70 2.000000000000000e+01 +89 71 1.900000000000000e+01 +89 72 1.800000000000000e+01 +89 73 1.700000000000000e+01 +89 74 1.600000000000000e+01 +89 75 1.500000000000000e+01 +89 76 1.400000000000000e+01 +89 77 1.300000000000000e+01 +89 78 1.200000000000000e+01 +89 79 1.100000000000000e+01 +89 80 1.000000000000000e+01 +89 81 9.000000000000000e+00 +89 82 8.000000000000000e+00 +89 83 7.000000000000000e+00 +89 84 6.000000000000000e+00 +89 85 5.000000000000000e+00 +89 86 4.000000000000000e+00 +89 87 3.000000000000000e+00 +89 88 2.000000000000000e+00 +89 89 1.000000000000000e+00 +90 1 9.000000000000000e+01 +90 2 8.900000000000000e+01 +90 3 8.800000000000000e+01 +90 4 8.700000000000000e+01 +90 5 8.600000000000000e+01 +90 6 8.500000000000000e+01 +90 7 8.400000000000000e+01 +90 8 8.300000000000000e+01 +90 9 8.200000000000000e+01 +90 10 8.100000000000000e+01 +90 11 8.000000000000000e+01 +90 12 7.900000000000000e+01 +90 13 7.800000000000000e+01 +90 14 7.700000000000000e+01 +90 15 7.600000000000000e+01 +90 16 7.500000000000000e+01 +90 17 7.400000000000000e+01 +90 18 7.300000000000000e+01 +90 19 7.200000000000000e+01 +90 20 7.100000000000000e+01 +90 21 7.000000000000000e+01 +90 22 6.900000000000000e+01 +90 23 6.800000000000000e+01 +90 24 6.700000000000000e+01 +90 25 6.600000000000000e+01 +90 26 6.500000000000000e+01 +90 27 6.400000000000000e+01 +90 28 6.300000000000000e+01 +90 29 6.200000000000000e+01 +90 30 6.100000000000000e+01 +90 31 6.000000000000000e+01 +90 32 5.900000000000000e+01 +90 33 5.800000000000000e+01 +90 34 5.700000000000000e+01 +90 35 5.600000000000000e+01 +90 36 5.500000000000000e+01 +90 37 5.400000000000000e+01 +90 38 5.300000000000000e+01 +90 39 5.200000000000000e+01 +90 40 5.100000000000000e+01 +90 41 5.000000000000000e+01 +90 42 4.900000000000000e+01 +90 43 4.800000000000000e+01 +90 44 4.700000000000000e+01 +90 45 4.600000000000000e+01 +90 46 4.500000000000000e+01 +90 47 4.400000000000000e+01 +90 48 4.300000000000000e+01 +90 49 4.200000000000000e+01 +90 50 4.100000000000000e+01 +90 51 4.000000000000000e+01 +90 52 3.900000000000000e+01 +90 53 3.800000000000000e+01 +90 54 3.700000000000000e+01 +90 55 3.600000000000000e+01 +90 56 3.500000000000000e+01 +90 57 3.400000000000000e+01 +90 58 3.300000000000000e+01 +90 59 3.200000000000000e+01 +90 60 3.100000000000000e+01 +90 61 3.000000000000000e+01 +90 62 2.900000000000000e+01 +90 63 2.800000000000000e+01 +90 64 2.700000000000000e+01 +90 65 2.600000000000000e+01 +90 66 2.500000000000000e+01 +90 67 2.400000000000000e+01 +90 68 2.300000000000000e+01 +90 69 2.200000000000000e+01 +90 70 2.100000000000000e+01 +90 71 2.000000000000000e+01 +90 72 1.900000000000000e+01 +90 73 1.800000000000000e+01 +90 74 1.700000000000000e+01 +90 75 1.600000000000000e+01 +90 76 1.500000000000000e+01 +90 77 1.400000000000000e+01 +90 78 1.300000000000000e+01 +90 79 1.200000000000000e+01 +90 80 1.100000000000000e+01 +90 81 1.000000000000000e+01 +90 82 9.000000000000000e+00 +90 83 8.000000000000000e+00 +90 84 7.000000000000000e+00 +90 85 6.000000000000000e+00 +90 86 5.000000000000000e+00 +90 87 4.000000000000000e+00 +90 88 3.000000000000000e+00 +90 89 2.000000000000000e+00 +90 90 1.000000000000000e+00 +91 1 9.100000000000000e+01 +91 2 9.000000000000000e+01 +91 3 8.900000000000000e+01 +91 4 8.800000000000000e+01 +91 5 8.700000000000000e+01 +91 6 8.600000000000000e+01 +91 7 8.500000000000000e+01 +91 8 8.400000000000000e+01 +91 9 8.300000000000000e+01 +91 10 8.200000000000000e+01 +91 11 8.100000000000000e+01 +91 12 8.000000000000000e+01 +91 13 7.900000000000000e+01 +91 14 7.800000000000000e+01 +91 15 7.700000000000000e+01 +91 16 7.600000000000000e+01 +91 17 7.500000000000000e+01 +91 18 7.400000000000000e+01 +91 19 7.300000000000000e+01 +91 20 7.200000000000000e+01 +91 21 7.100000000000000e+01 +91 22 7.000000000000000e+01 +91 23 6.900000000000000e+01 +91 24 6.800000000000000e+01 +91 25 6.700000000000000e+01 +91 26 6.600000000000000e+01 +91 27 6.500000000000000e+01 +91 28 6.400000000000000e+01 +91 29 6.300000000000000e+01 +91 30 6.200000000000000e+01 +91 31 6.100000000000000e+01 +91 32 6.000000000000000e+01 +91 33 5.900000000000000e+01 +91 34 5.800000000000000e+01 +91 35 5.700000000000000e+01 +91 36 5.600000000000000e+01 +91 37 5.500000000000000e+01 +91 38 5.400000000000000e+01 +91 39 5.300000000000000e+01 +91 40 5.200000000000000e+01 +91 41 5.100000000000000e+01 +91 42 5.000000000000000e+01 +91 43 4.900000000000000e+01 +91 44 4.800000000000000e+01 +91 45 4.700000000000000e+01 +91 46 4.600000000000000e+01 +91 47 4.500000000000000e+01 +91 48 4.400000000000000e+01 +91 49 4.300000000000000e+01 +91 50 4.200000000000000e+01 +91 51 4.100000000000000e+01 +91 52 4.000000000000000e+01 +91 53 3.900000000000000e+01 +91 54 3.800000000000000e+01 +91 55 3.700000000000000e+01 +91 56 3.600000000000000e+01 +91 57 3.500000000000000e+01 +91 58 3.400000000000000e+01 +91 59 3.300000000000000e+01 +91 60 3.200000000000000e+01 +91 61 3.100000000000000e+01 +91 62 3.000000000000000e+01 +91 63 2.900000000000000e+01 +91 64 2.800000000000000e+01 +91 65 2.700000000000000e+01 +91 66 2.600000000000000e+01 +91 67 2.500000000000000e+01 +91 68 2.400000000000000e+01 +91 69 2.300000000000000e+01 +91 70 2.200000000000000e+01 +91 71 2.100000000000000e+01 +91 72 2.000000000000000e+01 +91 73 1.900000000000000e+01 +91 74 1.800000000000000e+01 +91 75 1.700000000000000e+01 +91 76 1.600000000000000e+01 +91 77 1.500000000000000e+01 +91 78 1.400000000000000e+01 +91 79 1.300000000000000e+01 +91 80 1.200000000000000e+01 +91 81 1.100000000000000e+01 +91 82 1.000000000000000e+01 +91 83 9.000000000000000e+00 +91 84 8.000000000000000e+00 +91 85 7.000000000000000e+00 +91 86 6.000000000000000e+00 +91 87 5.000000000000000e+00 +91 88 4.000000000000000e+00 +91 89 3.000000000000000e+00 +91 90 2.000000000000000e+00 +91 91 1.000000000000000e+00 +92 1 9.200000000000000e+01 +92 2 9.100000000000000e+01 +92 3 9.000000000000000e+01 +92 4 8.900000000000000e+01 +92 5 8.800000000000000e+01 +92 6 8.700000000000000e+01 +92 7 8.600000000000000e+01 +92 8 8.500000000000000e+01 +92 9 8.400000000000000e+01 +92 10 8.300000000000000e+01 +92 11 8.200000000000000e+01 +92 12 8.100000000000000e+01 +92 13 8.000000000000000e+01 +92 14 7.900000000000000e+01 +92 15 7.800000000000000e+01 +92 16 7.700000000000000e+01 +92 17 7.600000000000000e+01 +92 18 7.500000000000000e+01 +92 19 7.400000000000000e+01 +92 20 7.300000000000000e+01 +92 21 7.200000000000000e+01 +92 22 7.100000000000000e+01 +92 23 7.000000000000000e+01 +92 24 6.900000000000000e+01 +92 25 6.800000000000000e+01 +92 26 6.700000000000000e+01 +92 27 6.600000000000000e+01 +92 28 6.500000000000000e+01 +92 29 6.400000000000000e+01 +92 30 6.300000000000000e+01 +92 31 6.200000000000000e+01 +92 32 6.100000000000000e+01 +92 33 6.000000000000000e+01 +92 34 5.900000000000000e+01 +92 35 5.800000000000000e+01 +92 36 5.700000000000000e+01 +92 37 5.600000000000000e+01 +92 38 5.500000000000000e+01 +92 39 5.400000000000000e+01 +92 40 5.300000000000000e+01 +92 41 5.200000000000000e+01 +92 42 5.100000000000000e+01 +92 43 5.000000000000000e+01 +92 44 4.900000000000000e+01 +92 45 4.800000000000000e+01 +92 46 4.700000000000000e+01 +92 47 4.600000000000000e+01 +92 48 4.500000000000000e+01 +92 49 4.400000000000000e+01 +92 50 4.300000000000000e+01 +92 51 4.200000000000000e+01 +92 52 4.100000000000000e+01 +92 53 4.000000000000000e+01 +92 54 3.900000000000000e+01 +92 55 3.800000000000000e+01 +92 56 3.700000000000000e+01 +92 57 3.600000000000000e+01 +92 58 3.500000000000000e+01 +92 59 3.400000000000000e+01 +92 60 3.300000000000000e+01 +92 61 3.200000000000000e+01 +92 62 3.100000000000000e+01 +92 63 3.000000000000000e+01 +92 64 2.900000000000000e+01 +92 65 2.800000000000000e+01 +92 66 2.700000000000000e+01 +92 67 2.600000000000000e+01 +92 68 2.500000000000000e+01 +92 69 2.400000000000000e+01 +92 70 2.300000000000000e+01 +92 71 2.200000000000000e+01 +92 72 2.100000000000000e+01 +92 73 2.000000000000000e+01 +92 74 1.900000000000000e+01 +92 75 1.800000000000000e+01 +92 76 1.700000000000000e+01 +92 77 1.600000000000000e+01 +92 78 1.500000000000000e+01 +92 79 1.400000000000000e+01 +92 80 1.300000000000000e+01 +92 81 1.200000000000000e+01 +92 82 1.100000000000000e+01 +92 83 1.000000000000000e+01 +92 84 9.000000000000000e+00 +92 85 8.000000000000000e+00 +92 86 7.000000000000000e+00 +92 87 6.000000000000000e+00 +92 88 5.000000000000000e+00 +92 89 4.000000000000000e+00 +92 90 3.000000000000000e+00 +92 91 2.000000000000000e+00 +92 92 1.000000000000000e+00 +93 1 9.300000000000000e+01 +93 2 9.200000000000000e+01 +93 3 9.100000000000000e+01 +93 4 9.000000000000000e+01 +93 5 8.900000000000000e+01 +93 6 8.800000000000000e+01 +93 7 8.700000000000000e+01 +93 8 8.600000000000000e+01 +93 9 8.500000000000000e+01 +93 10 8.400000000000000e+01 +93 11 8.300000000000000e+01 +93 12 8.200000000000000e+01 +93 13 8.100000000000000e+01 +93 14 8.000000000000000e+01 +93 15 7.900000000000000e+01 +93 16 7.800000000000000e+01 +93 17 7.700000000000000e+01 +93 18 7.600000000000000e+01 +93 19 7.500000000000000e+01 +93 20 7.400000000000000e+01 +93 21 7.300000000000000e+01 +93 22 7.200000000000000e+01 +93 23 7.100000000000000e+01 +93 24 7.000000000000000e+01 +93 25 6.900000000000000e+01 +93 26 6.800000000000000e+01 +93 27 6.700000000000000e+01 +93 28 6.600000000000000e+01 +93 29 6.500000000000000e+01 +93 30 6.400000000000000e+01 +93 31 6.300000000000000e+01 +93 32 6.200000000000000e+01 +93 33 6.100000000000000e+01 +93 34 6.000000000000000e+01 +93 35 5.900000000000000e+01 +93 36 5.800000000000000e+01 +93 37 5.700000000000000e+01 +93 38 5.600000000000000e+01 +93 39 5.500000000000000e+01 +93 40 5.400000000000000e+01 +93 41 5.300000000000000e+01 +93 42 5.200000000000000e+01 +93 43 5.100000000000000e+01 +93 44 5.000000000000000e+01 +93 45 4.900000000000000e+01 +93 46 4.800000000000000e+01 +93 47 4.700000000000000e+01 +93 48 4.600000000000000e+01 +93 49 4.500000000000000e+01 +93 50 4.400000000000000e+01 +93 51 4.300000000000000e+01 +93 52 4.200000000000000e+01 +93 53 4.100000000000000e+01 +93 54 4.000000000000000e+01 +93 55 3.900000000000000e+01 +93 56 3.800000000000000e+01 +93 57 3.700000000000000e+01 +93 58 3.600000000000000e+01 +93 59 3.500000000000000e+01 +93 60 3.400000000000000e+01 +93 61 3.300000000000000e+01 +93 62 3.200000000000000e+01 +93 63 3.100000000000000e+01 +93 64 3.000000000000000e+01 +93 65 2.900000000000000e+01 +93 66 2.800000000000000e+01 +93 67 2.700000000000000e+01 +93 68 2.600000000000000e+01 +93 69 2.500000000000000e+01 +93 70 2.400000000000000e+01 +93 71 2.300000000000000e+01 +93 72 2.200000000000000e+01 +93 73 2.100000000000000e+01 +93 74 2.000000000000000e+01 +93 75 1.900000000000000e+01 +93 76 1.800000000000000e+01 +93 77 1.700000000000000e+01 +93 78 1.600000000000000e+01 +93 79 1.500000000000000e+01 +93 80 1.400000000000000e+01 +93 81 1.300000000000000e+01 +93 82 1.200000000000000e+01 +93 83 1.100000000000000e+01 +93 84 1.000000000000000e+01 +93 85 9.000000000000000e+00 +93 86 8.000000000000000e+00 +93 87 7.000000000000000e+00 +93 88 6.000000000000000e+00 +93 89 5.000000000000000e+00 +93 90 4.000000000000000e+00 +93 91 3.000000000000000e+00 +93 92 2.000000000000000e+00 +93 93 1.000000000000000e+00 +94 1 9.400000000000000e+01 +94 2 9.300000000000000e+01 +94 3 9.200000000000000e+01 +94 4 9.100000000000000e+01 +94 5 9.000000000000000e+01 +94 6 8.900000000000000e+01 +94 7 8.800000000000000e+01 +94 8 8.700000000000000e+01 +94 9 8.600000000000000e+01 +94 10 8.500000000000000e+01 +94 11 8.400000000000000e+01 +94 12 8.300000000000000e+01 +94 13 8.200000000000000e+01 +94 14 8.100000000000000e+01 +94 15 8.000000000000000e+01 +94 16 7.900000000000000e+01 +94 17 7.800000000000000e+01 +94 18 7.700000000000000e+01 +94 19 7.600000000000000e+01 +94 20 7.500000000000000e+01 +94 21 7.400000000000000e+01 +94 22 7.300000000000000e+01 +94 23 7.200000000000000e+01 +94 24 7.100000000000000e+01 +94 25 7.000000000000000e+01 +94 26 6.900000000000000e+01 +94 27 6.800000000000000e+01 +94 28 6.700000000000000e+01 +94 29 6.600000000000000e+01 +94 30 6.500000000000000e+01 +94 31 6.400000000000000e+01 +94 32 6.300000000000000e+01 +94 33 6.200000000000000e+01 +94 34 6.100000000000000e+01 +94 35 6.000000000000000e+01 +94 36 5.900000000000000e+01 +94 37 5.800000000000000e+01 +94 38 5.700000000000000e+01 +94 39 5.600000000000000e+01 +94 40 5.500000000000000e+01 +94 41 5.400000000000000e+01 +94 42 5.300000000000000e+01 +94 43 5.200000000000000e+01 +94 44 5.100000000000000e+01 +94 45 5.000000000000000e+01 +94 46 4.900000000000000e+01 +94 47 4.800000000000000e+01 +94 48 4.700000000000000e+01 +94 49 4.600000000000000e+01 +94 50 4.500000000000000e+01 +94 51 4.400000000000000e+01 +94 52 4.300000000000000e+01 +94 53 4.200000000000000e+01 +94 54 4.100000000000000e+01 +94 55 4.000000000000000e+01 +94 56 3.900000000000000e+01 +94 57 3.800000000000000e+01 +94 58 3.700000000000000e+01 +94 59 3.600000000000000e+01 +94 60 3.500000000000000e+01 +94 61 3.400000000000000e+01 +94 62 3.300000000000000e+01 +94 63 3.200000000000000e+01 +94 64 3.100000000000000e+01 +94 65 3.000000000000000e+01 +94 66 2.900000000000000e+01 +94 67 2.800000000000000e+01 +94 68 2.700000000000000e+01 +94 69 2.600000000000000e+01 +94 70 2.500000000000000e+01 +94 71 2.400000000000000e+01 +94 72 2.300000000000000e+01 +94 73 2.200000000000000e+01 +94 74 2.100000000000000e+01 +94 75 2.000000000000000e+01 +94 76 1.900000000000000e+01 +94 77 1.800000000000000e+01 +94 78 1.700000000000000e+01 +94 79 1.600000000000000e+01 +94 80 1.500000000000000e+01 +94 81 1.400000000000000e+01 +94 82 1.300000000000000e+01 +94 83 1.200000000000000e+01 +94 84 1.100000000000000e+01 +94 85 1.000000000000000e+01 +94 86 9.000000000000000e+00 +94 87 8.000000000000000e+00 +94 88 7.000000000000000e+00 +94 89 6.000000000000000e+00 +94 90 5.000000000000000e+00 +94 91 4.000000000000000e+00 +94 92 3.000000000000000e+00 +94 93 2.000000000000000e+00 +94 94 1.000000000000000e+00 +95 1 9.500000000000000e+01 +95 2 9.400000000000000e+01 +95 3 9.300000000000000e+01 +95 4 9.200000000000000e+01 +95 5 9.100000000000000e+01 +95 6 9.000000000000000e+01 +95 7 8.900000000000000e+01 +95 8 8.800000000000000e+01 +95 9 8.700000000000000e+01 +95 10 8.600000000000000e+01 +95 11 8.500000000000000e+01 +95 12 8.400000000000000e+01 +95 13 8.300000000000000e+01 +95 14 8.200000000000000e+01 +95 15 8.100000000000000e+01 +95 16 8.000000000000000e+01 +95 17 7.900000000000000e+01 +95 18 7.800000000000000e+01 +95 19 7.700000000000000e+01 +95 20 7.600000000000000e+01 +95 21 7.500000000000000e+01 +95 22 7.400000000000000e+01 +95 23 7.300000000000000e+01 +95 24 7.200000000000000e+01 +95 25 7.100000000000000e+01 +95 26 7.000000000000000e+01 +95 27 6.900000000000000e+01 +95 28 6.800000000000000e+01 +95 29 6.700000000000000e+01 +95 30 6.600000000000000e+01 +95 31 6.500000000000000e+01 +95 32 6.400000000000000e+01 +95 33 6.300000000000000e+01 +95 34 6.200000000000000e+01 +95 35 6.100000000000000e+01 +95 36 6.000000000000000e+01 +95 37 5.900000000000000e+01 +95 38 5.800000000000000e+01 +95 39 5.700000000000000e+01 +95 40 5.600000000000000e+01 +95 41 5.500000000000000e+01 +95 42 5.400000000000000e+01 +95 43 5.300000000000000e+01 +95 44 5.200000000000000e+01 +95 45 5.100000000000000e+01 +95 46 5.000000000000000e+01 +95 47 4.900000000000000e+01 +95 48 4.800000000000000e+01 +95 49 4.700000000000000e+01 +95 50 4.600000000000000e+01 +95 51 4.500000000000000e+01 +95 52 4.400000000000000e+01 +95 53 4.300000000000000e+01 +95 54 4.200000000000000e+01 +95 55 4.100000000000000e+01 +95 56 4.000000000000000e+01 +95 57 3.900000000000000e+01 +95 58 3.800000000000000e+01 +95 59 3.700000000000000e+01 +95 60 3.600000000000000e+01 +95 61 3.500000000000000e+01 +95 62 3.400000000000000e+01 +95 63 3.300000000000000e+01 +95 64 3.200000000000000e+01 +95 65 3.100000000000000e+01 +95 66 3.000000000000000e+01 +95 67 2.900000000000000e+01 +95 68 2.800000000000000e+01 +95 69 2.700000000000000e+01 +95 70 2.600000000000000e+01 +95 71 2.500000000000000e+01 +95 72 2.400000000000000e+01 +95 73 2.300000000000000e+01 +95 74 2.200000000000000e+01 +95 75 2.100000000000000e+01 +95 76 2.000000000000000e+01 +95 77 1.900000000000000e+01 +95 78 1.800000000000000e+01 +95 79 1.700000000000000e+01 +95 80 1.600000000000000e+01 +95 81 1.500000000000000e+01 +95 82 1.400000000000000e+01 +95 83 1.300000000000000e+01 +95 84 1.200000000000000e+01 +95 85 1.100000000000000e+01 +95 86 1.000000000000000e+01 +95 87 9.000000000000000e+00 +95 88 8.000000000000000e+00 +95 89 7.000000000000000e+00 +95 90 6.000000000000000e+00 +95 91 5.000000000000000e+00 +95 92 4.000000000000000e+00 +95 93 3.000000000000000e+00 +95 94 2.000000000000000e+00 +95 95 1.000000000000000e+00 +96 1 9.600000000000000e+01 +96 2 9.500000000000000e+01 +96 3 9.400000000000000e+01 +96 4 9.300000000000000e+01 +96 5 9.200000000000000e+01 +96 6 9.100000000000000e+01 +96 7 9.000000000000000e+01 +96 8 8.900000000000000e+01 +96 9 8.800000000000000e+01 +96 10 8.700000000000000e+01 +96 11 8.600000000000000e+01 +96 12 8.500000000000000e+01 +96 13 8.400000000000000e+01 +96 14 8.300000000000000e+01 +96 15 8.200000000000000e+01 +96 16 8.100000000000000e+01 +96 17 8.000000000000000e+01 +96 18 7.900000000000000e+01 +96 19 7.800000000000000e+01 +96 20 7.700000000000000e+01 +96 21 7.600000000000000e+01 +96 22 7.500000000000000e+01 +96 23 7.400000000000000e+01 +96 24 7.300000000000000e+01 +96 25 7.200000000000000e+01 +96 26 7.100000000000000e+01 +96 27 7.000000000000000e+01 +96 28 6.900000000000000e+01 +96 29 6.800000000000000e+01 +96 30 6.700000000000000e+01 +96 31 6.600000000000000e+01 +96 32 6.500000000000000e+01 +96 33 6.400000000000000e+01 +96 34 6.300000000000000e+01 +96 35 6.200000000000000e+01 +96 36 6.100000000000000e+01 +96 37 6.000000000000000e+01 +96 38 5.900000000000000e+01 +96 39 5.800000000000000e+01 +96 40 5.700000000000000e+01 +96 41 5.600000000000000e+01 +96 42 5.500000000000000e+01 +96 43 5.400000000000000e+01 +96 44 5.300000000000000e+01 +96 45 5.200000000000000e+01 +96 46 5.100000000000000e+01 +96 47 5.000000000000000e+01 +96 48 4.900000000000000e+01 +96 49 4.800000000000000e+01 +96 50 4.700000000000000e+01 +96 51 4.600000000000000e+01 +96 52 4.500000000000000e+01 +96 53 4.400000000000000e+01 +96 54 4.300000000000000e+01 +96 55 4.200000000000000e+01 +96 56 4.100000000000000e+01 +96 57 4.000000000000000e+01 +96 58 3.900000000000000e+01 +96 59 3.800000000000000e+01 +96 60 3.700000000000000e+01 +96 61 3.600000000000000e+01 +96 62 3.500000000000000e+01 +96 63 3.400000000000000e+01 +96 64 3.300000000000000e+01 +96 65 3.200000000000000e+01 +96 66 3.100000000000000e+01 +96 67 3.000000000000000e+01 +96 68 2.900000000000000e+01 +96 69 2.800000000000000e+01 +96 70 2.700000000000000e+01 +96 71 2.600000000000000e+01 +96 72 2.500000000000000e+01 +96 73 2.400000000000000e+01 +96 74 2.300000000000000e+01 +96 75 2.200000000000000e+01 +96 76 2.100000000000000e+01 +96 77 2.000000000000000e+01 +96 78 1.900000000000000e+01 +96 79 1.800000000000000e+01 +96 80 1.700000000000000e+01 +96 81 1.600000000000000e+01 +96 82 1.500000000000000e+01 +96 83 1.400000000000000e+01 +96 84 1.300000000000000e+01 +96 85 1.200000000000000e+01 +96 86 1.100000000000000e+01 +96 87 1.000000000000000e+01 +96 88 9.000000000000000e+00 +96 89 8.000000000000000e+00 +96 90 7.000000000000000e+00 +96 91 6.000000000000000e+00 +96 92 5.000000000000000e+00 +96 93 4.000000000000000e+00 +96 94 3.000000000000000e+00 +96 95 2.000000000000000e+00 +96 96 1.000000000000000e+00 +97 1 9.700000000000000e+01 +97 2 9.600000000000000e+01 +97 3 9.500000000000000e+01 +97 4 9.400000000000000e+01 +97 5 9.300000000000000e+01 +97 6 9.200000000000000e+01 +97 7 9.100000000000000e+01 +97 8 9.000000000000000e+01 +97 9 8.900000000000000e+01 +97 10 8.800000000000000e+01 +97 11 8.700000000000000e+01 +97 12 8.600000000000000e+01 +97 13 8.500000000000000e+01 +97 14 8.400000000000000e+01 +97 15 8.300000000000000e+01 +97 16 8.200000000000000e+01 +97 17 8.100000000000000e+01 +97 18 8.000000000000000e+01 +97 19 7.900000000000000e+01 +97 20 7.800000000000000e+01 +97 21 7.700000000000000e+01 +97 22 7.600000000000000e+01 +97 23 7.500000000000000e+01 +97 24 7.400000000000000e+01 +97 25 7.300000000000000e+01 +97 26 7.200000000000000e+01 +97 27 7.100000000000000e+01 +97 28 7.000000000000000e+01 +97 29 6.900000000000000e+01 +97 30 6.800000000000000e+01 +97 31 6.700000000000000e+01 +97 32 6.600000000000000e+01 +97 33 6.500000000000000e+01 +97 34 6.400000000000000e+01 +97 35 6.300000000000000e+01 +97 36 6.200000000000000e+01 +97 37 6.100000000000000e+01 +97 38 6.000000000000000e+01 +97 39 5.900000000000000e+01 +97 40 5.800000000000000e+01 +97 41 5.700000000000000e+01 +97 42 5.600000000000000e+01 +97 43 5.500000000000000e+01 +97 44 5.400000000000000e+01 +97 45 5.300000000000000e+01 +97 46 5.200000000000000e+01 +97 47 5.100000000000000e+01 +97 48 5.000000000000000e+01 +97 49 4.900000000000000e+01 +97 50 4.800000000000000e+01 +97 51 4.700000000000000e+01 +97 52 4.600000000000000e+01 +97 53 4.500000000000000e+01 +97 54 4.400000000000000e+01 +97 55 4.300000000000000e+01 +97 56 4.200000000000000e+01 +97 57 4.100000000000000e+01 +97 58 4.000000000000000e+01 +97 59 3.900000000000000e+01 +97 60 3.800000000000000e+01 +97 61 3.700000000000000e+01 +97 62 3.600000000000000e+01 +97 63 3.500000000000000e+01 +97 64 3.400000000000000e+01 +97 65 3.300000000000000e+01 +97 66 3.200000000000000e+01 +97 67 3.100000000000000e+01 +97 68 3.000000000000000e+01 +97 69 2.900000000000000e+01 +97 70 2.800000000000000e+01 +97 71 2.700000000000000e+01 +97 72 2.600000000000000e+01 +97 73 2.500000000000000e+01 +97 74 2.400000000000000e+01 +97 75 2.300000000000000e+01 +97 76 2.200000000000000e+01 +97 77 2.100000000000000e+01 +97 78 2.000000000000000e+01 +97 79 1.900000000000000e+01 +97 80 1.800000000000000e+01 +97 81 1.700000000000000e+01 +97 82 1.600000000000000e+01 +97 83 1.500000000000000e+01 +97 84 1.400000000000000e+01 +97 85 1.300000000000000e+01 +97 86 1.200000000000000e+01 +97 87 1.100000000000000e+01 +97 88 1.000000000000000e+01 +97 89 9.000000000000000e+00 +97 90 8.000000000000000e+00 +97 91 7.000000000000000e+00 +97 92 6.000000000000000e+00 +97 93 5.000000000000000e+00 +97 94 4.000000000000000e+00 +97 95 3.000000000000000e+00 +97 96 2.000000000000000e+00 +97 97 1.000000000000000e+00 +98 1 9.800000000000000e+01 +98 2 9.700000000000000e+01 +98 3 9.600000000000000e+01 +98 4 9.500000000000000e+01 +98 5 9.400000000000000e+01 +98 6 9.300000000000000e+01 +98 7 9.200000000000000e+01 +98 8 9.100000000000000e+01 +98 9 9.000000000000000e+01 +98 10 8.900000000000000e+01 +98 11 8.800000000000000e+01 +98 12 8.700000000000000e+01 +98 13 8.600000000000000e+01 +98 14 8.500000000000000e+01 +98 15 8.400000000000000e+01 +98 16 8.300000000000000e+01 +98 17 8.200000000000000e+01 +98 18 8.100000000000000e+01 +98 19 8.000000000000000e+01 +98 20 7.900000000000000e+01 +98 21 7.800000000000000e+01 +98 22 7.700000000000000e+01 +98 23 7.600000000000000e+01 +98 24 7.500000000000000e+01 +98 25 7.400000000000000e+01 +98 26 7.300000000000000e+01 +98 27 7.200000000000000e+01 +98 28 7.100000000000000e+01 +98 29 7.000000000000000e+01 +98 30 6.900000000000000e+01 +98 31 6.800000000000000e+01 +98 32 6.700000000000000e+01 +98 33 6.600000000000000e+01 +98 34 6.500000000000000e+01 +98 35 6.400000000000000e+01 +98 36 6.300000000000000e+01 +98 37 6.200000000000000e+01 +98 38 6.100000000000000e+01 +98 39 6.000000000000000e+01 +98 40 5.900000000000000e+01 +98 41 5.800000000000000e+01 +98 42 5.700000000000000e+01 +98 43 5.600000000000000e+01 +98 44 5.500000000000000e+01 +98 45 5.400000000000000e+01 +98 46 5.300000000000000e+01 +98 47 5.200000000000000e+01 +98 48 5.100000000000000e+01 +98 49 5.000000000000000e+01 +98 50 4.900000000000000e+01 +98 51 4.800000000000000e+01 +98 52 4.700000000000000e+01 +98 53 4.600000000000000e+01 +98 54 4.500000000000000e+01 +98 55 4.400000000000000e+01 +98 56 4.300000000000000e+01 +98 57 4.200000000000000e+01 +98 58 4.100000000000000e+01 +98 59 4.000000000000000e+01 +98 60 3.900000000000000e+01 +98 61 3.800000000000000e+01 +98 62 3.700000000000000e+01 +98 63 3.600000000000000e+01 +98 64 3.500000000000000e+01 +98 65 3.400000000000000e+01 +98 66 3.300000000000000e+01 +98 67 3.200000000000000e+01 +98 68 3.100000000000000e+01 +98 69 3.000000000000000e+01 +98 70 2.900000000000000e+01 +98 71 2.800000000000000e+01 +98 72 2.700000000000000e+01 +98 73 2.600000000000000e+01 +98 74 2.500000000000000e+01 +98 75 2.400000000000000e+01 +98 76 2.300000000000000e+01 +98 77 2.200000000000000e+01 +98 78 2.100000000000000e+01 +98 79 2.000000000000000e+01 +98 80 1.900000000000000e+01 +98 81 1.800000000000000e+01 +98 82 1.700000000000000e+01 +98 83 1.600000000000000e+01 +98 84 1.500000000000000e+01 +98 85 1.400000000000000e+01 +98 86 1.300000000000000e+01 +98 87 1.200000000000000e+01 +98 88 1.100000000000000e+01 +98 89 1.000000000000000e+01 +98 90 9.000000000000000e+00 +98 91 8.000000000000000e+00 +98 92 7.000000000000000e+00 +98 93 6.000000000000000e+00 +98 94 5.000000000000000e+00 +98 95 4.000000000000000e+00 +98 96 3.000000000000000e+00 +98 97 2.000000000000000e+00 +98 98 1.000000000000000e+00 +99 1 9.900000000000000e+01 +99 2 9.800000000000000e+01 +99 3 9.700000000000000e+01 +99 4 9.600000000000000e+01 +99 5 9.500000000000000e+01 +99 6 9.400000000000000e+01 +99 7 9.300000000000000e+01 +99 8 9.200000000000000e+01 +99 9 9.100000000000000e+01 +99 10 9.000000000000000e+01 +99 11 8.900000000000000e+01 +99 12 8.800000000000000e+01 +99 13 8.700000000000000e+01 +99 14 8.600000000000000e+01 +99 15 8.500000000000000e+01 +99 16 8.400000000000000e+01 +99 17 8.300000000000000e+01 +99 18 8.200000000000000e+01 +99 19 8.100000000000000e+01 +99 20 8.000000000000000e+01 +99 21 7.900000000000000e+01 +99 22 7.800000000000000e+01 +99 23 7.700000000000000e+01 +99 24 7.600000000000000e+01 +99 25 7.500000000000000e+01 +99 26 7.400000000000000e+01 +99 27 7.300000000000000e+01 +99 28 7.200000000000000e+01 +99 29 7.100000000000000e+01 +99 30 7.000000000000000e+01 +99 31 6.900000000000000e+01 +99 32 6.800000000000000e+01 +99 33 6.700000000000000e+01 +99 34 6.600000000000000e+01 +99 35 6.500000000000000e+01 +99 36 6.400000000000000e+01 +99 37 6.300000000000000e+01 +99 38 6.200000000000000e+01 +99 39 6.100000000000000e+01 +99 40 6.000000000000000e+01 +99 41 5.900000000000000e+01 +99 42 5.800000000000000e+01 +99 43 5.700000000000000e+01 +99 44 5.600000000000000e+01 +99 45 5.500000000000000e+01 +99 46 5.400000000000000e+01 +99 47 5.300000000000000e+01 +99 48 5.200000000000000e+01 +99 49 5.100000000000000e+01 +99 50 5.000000000000000e+01 +99 51 4.900000000000000e+01 +99 52 4.800000000000000e+01 +99 53 4.700000000000000e+01 +99 54 4.600000000000000e+01 +99 55 4.500000000000000e+01 +99 56 4.400000000000000e+01 +99 57 4.300000000000000e+01 +99 58 4.200000000000000e+01 +99 59 4.100000000000000e+01 +99 60 4.000000000000000e+01 +99 61 3.900000000000000e+01 +99 62 3.800000000000000e+01 +99 63 3.700000000000000e+01 +99 64 3.600000000000000e+01 +99 65 3.500000000000000e+01 +99 66 3.400000000000000e+01 +99 67 3.300000000000000e+01 +99 68 3.200000000000000e+01 +99 69 3.100000000000000e+01 +99 70 3.000000000000000e+01 +99 71 2.900000000000000e+01 +99 72 2.800000000000000e+01 +99 73 2.700000000000000e+01 +99 74 2.600000000000000e+01 +99 75 2.500000000000000e+01 +99 76 2.400000000000000e+01 +99 77 2.300000000000000e+01 +99 78 2.200000000000000e+01 +99 79 2.100000000000000e+01 +99 80 2.000000000000000e+01 +99 81 1.900000000000000e+01 +99 82 1.800000000000000e+01 +99 83 1.700000000000000e+01 +99 84 1.600000000000000e+01 +99 85 1.500000000000000e+01 +99 86 1.400000000000000e+01 +99 87 1.300000000000000e+01 +99 88 1.200000000000000e+01 +99 89 1.100000000000000e+01 +99 90 1.000000000000000e+01 +99 91 9.000000000000000e+00 +99 92 8.000000000000000e+00 +99 93 7.000000000000000e+00 +99 94 6.000000000000000e+00 +99 95 5.000000000000000e+00 +99 96 4.000000000000000e+00 +99 97 3.000000000000000e+00 +99 98 2.000000000000000e+00 +99 99 1.000000000000000e+00 +100 1 1.000000000000000e+02 +100 2 9.900000000000000e+01 +100 3 9.800000000000000e+01 +100 4 9.700000000000000e+01 +100 5 9.600000000000000e+01 +100 6 9.500000000000000e+01 +100 7 9.400000000000000e+01 +100 8 9.300000000000000e+01 +100 9 9.200000000000000e+01 +100 10 9.100000000000000e+01 +100 11 9.000000000000000e+01 +100 12 8.900000000000000e+01 +100 13 8.800000000000000e+01 +100 14 8.700000000000000e+01 +100 15 8.600000000000000e+01 +100 16 8.500000000000000e+01 +100 17 8.400000000000000e+01 +100 18 8.300000000000000e+01 +100 19 8.200000000000000e+01 +100 20 8.100000000000000e+01 +100 21 8.000000000000000e+01 +100 22 7.900000000000000e+01 +100 23 7.800000000000000e+01 +100 24 7.700000000000000e+01 +100 25 7.600000000000000e+01 +100 26 7.500000000000000e+01 +100 27 7.400000000000000e+01 +100 28 7.300000000000000e+01 +100 29 7.200000000000000e+01 +100 30 7.100000000000000e+01 +100 31 7.000000000000000e+01 +100 32 6.900000000000000e+01 +100 33 6.800000000000000e+01 +100 34 6.700000000000000e+01 +100 35 6.600000000000000e+01 +100 36 6.500000000000000e+01 +100 37 6.400000000000000e+01 +100 38 6.300000000000000e+01 +100 39 6.200000000000000e+01 +100 40 6.100000000000000e+01 +100 41 6.000000000000000e+01 +100 42 5.900000000000000e+01 +100 43 5.800000000000000e+01 +100 44 5.700000000000000e+01 +100 45 5.600000000000000e+01 +100 46 5.500000000000000e+01 +100 47 5.400000000000000e+01 +100 48 5.300000000000000e+01 +100 49 5.200000000000000e+01 +100 50 5.100000000000000e+01 +100 51 5.000000000000000e+01 +100 52 4.900000000000000e+01 +100 53 4.800000000000000e+01 +100 54 4.700000000000000e+01 +100 55 4.600000000000000e+01 +100 56 4.500000000000000e+01 +100 57 4.400000000000000e+01 +100 58 4.300000000000000e+01 +100 59 4.200000000000000e+01 +100 60 4.100000000000000e+01 +100 61 4.000000000000000e+01 +100 62 3.900000000000000e+01 +100 63 3.800000000000000e+01 +100 64 3.700000000000000e+01 +100 65 3.600000000000000e+01 +100 66 3.500000000000000e+01 +100 67 3.400000000000000e+01 +100 68 3.300000000000000e+01 +100 69 3.200000000000000e+01 +100 70 3.100000000000000e+01 +100 71 3.000000000000000e+01 +100 72 2.900000000000000e+01 +100 73 2.800000000000000e+01 +100 74 2.700000000000000e+01 +100 75 2.600000000000000e+01 +100 76 2.500000000000000e+01 +100 77 2.400000000000000e+01 +100 78 2.300000000000000e+01 +100 79 2.200000000000000e+01 +100 80 2.100000000000000e+01 +100 81 2.000000000000000e+01 +100 82 1.900000000000000e+01 +100 83 1.800000000000000e+01 +100 84 1.700000000000000e+01 +100 85 1.600000000000000e+01 +100 86 1.500000000000000e+01 +100 87 1.400000000000000e+01 +100 88 1.300000000000000e+01 +100 89 1.200000000000000e+01 +100 90 1.100000000000000e+01 +100 91 1.000000000000000e+01 +100 92 9.000000000000000e+00 +100 93 8.000000000000000e+00 +100 94 7.000000000000000e+00 +100 95 6.000000000000000e+00 +100 96 5.000000000000000e+00 +100 97 4.000000000000000e+00 +100 98 3.000000000000000e+00 +100 99 2.000000000000000e+00 +100 100 1.000000000000000e+00 diff --git a/packages/tpetra/core/test/CrsGraphTransposer/LT.mtx b/packages/tpetra/core/test/CrsGraphTransposer/LT.mtx new file mode 100644 index 000000000000..ed0be4a8fffd --- /dev/null +++ b/packages/tpetra/core/test/CrsGraphTransposer/LT.mtx @@ -0,0 +1,5053 @@ +%%MatrixMarket matrix coordinate real general +% +100 100 5050 +1 1 1.000000000000000e+00 +1 2 2.000000000000000e+00 +1 3 3.000000000000000e+00 +1 4 4.000000000000000e+00 +1 5 5.000000000000000e+00 +1 6 6.000000000000000e+00 +1 7 7.000000000000000e+00 +1 8 8.000000000000000e+00 +1 9 9.000000000000000e+00 +1 10 1.000000000000000e+01 +1 11 1.100000000000000e+01 +1 12 1.200000000000000e+01 +1 13 1.300000000000000e+01 +1 14 1.400000000000000e+01 +1 15 1.500000000000000e+01 +1 16 1.600000000000000e+01 +1 17 1.700000000000000e+01 +1 18 1.800000000000000e+01 +1 19 1.900000000000000e+01 +1 20 2.000000000000000e+01 +1 21 2.100000000000000e+01 +1 22 2.200000000000000e+01 +1 23 2.300000000000000e+01 +1 24 2.400000000000000e+01 +1 25 2.500000000000000e+01 +1 26 2.600000000000000e+01 +1 27 2.700000000000000e+01 +1 28 2.800000000000000e+01 +1 29 2.900000000000000e+01 +1 30 3.000000000000000e+01 +1 31 3.100000000000000e+01 +1 32 3.200000000000000e+01 +1 33 3.300000000000000e+01 +1 34 3.400000000000000e+01 +1 35 3.500000000000000e+01 +1 36 3.600000000000000e+01 +1 37 3.700000000000000e+01 +1 38 3.800000000000000e+01 +1 39 3.900000000000000e+01 +1 40 4.000000000000000e+01 +1 41 4.100000000000000e+01 +1 42 4.200000000000000e+01 +1 43 4.300000000000000e+01 +1 44 4.400000000000000e+01 +1 45 4.500000000000000e+01 +1 46 4.600000000000000e+01 +1 47 4.700000000000000e+01 +1 48 4.800000000000000e+01 +1 49 4.900000000000000e+01 +1 50 5.000000000000000e+01 +1 51 5.100000000000000e+01 +1 52 5.200000000000000e+01 +1 53 5.300000000000000e+01 +1 54 5.400000000000000e+01 +1 55 5.500000000000000e+01 +1 56 5.600000000000000e+01 +1 57 5.700000000000000e+01 +1 58 5.800000000000000e+01 +1 59 5.900000000000000e+01 +1 60 6.000000000000000e+01 +1 61 6.100000000000000e+01 +1 62 6.200000000000000e+01 +1 63 6.300000000000000e+01 +1 64 6.400000000000000e+01 +1 65 6.500000000000000e+01 +1 66 6.600000000000000e+01 +1 67 6.700000000000000e+01 +1 68 6.800000000000000e+01 +1 69 6.900000000000000e+01 +1 70 7.000000000000000e+01 +1 71 7.100000000000000e+01 +1 72 7.200000000000000e+01 +1 73 7.300000000000000e+01 +1 74 7.400000000000000e+01 +1 75 7.500000000000000e+01 +1 76 7.600000000000000e+01 +1 77 7.700000000000000e+01 +1 78 7.800000000000000e+01 +1 79 7.900000000000000e+01 +1 80 8.000000000000000e+01 +1 81 8.100000000000000e+01 +1 82 8.200000000000000e+01 +1 83 8.300000000000000e+01 +1 84 8.400000000000000e+01 +1 85 8.500000000000000e+01 +1 86 8.600000000000000e+01 +1 87 8.700000000000000e+01 +1 88 8.800000000000000e+01 +1 89 8.900000000000000e+01 +1 90 9.000000000000000e+01 +1 91 9.100000000000000e+01 +1 92 9.200000000000000e+01 +1 93 9.300000000000000e+01 +1 94 9.400000000000000e+01 +1 95 9.500000000000000e+01 +1 96 9.600000000000000e+01 +1 97 9.700000000000000e+01 +1 98 9.800000000000000e+01 +1 99 9.900000000000000e+01 +1 100 1.000000000000000e+02 +2 2 1.000000000000000e+00 +2 3 2.000000000000000e+00 +2 4 3.000000000000000e+00 +2 5 4.000000000000000e+00 +2 6 5.000000000000000e+00 +2 7 6.000000000000000e+00 +2 8 7.000000000000000e+00 +2 9 8.000000000000000e+00 +2 10 9.000000000000000e+00 +2 11 1.000000000000000e+01 +2 12 1.100000000000000e+01 +2 13 1.200000000000000e+01 +2 14 1.300000000000000e+01 +2 15 1.400000000000000e+01 +2 16 1.500000000000000e+01 +2 17 1.600000000000000e+01 +2 18 1.700000000000000e+01 +2 19 1.800000000000000e+01 +2 20 1.900000000000000e+01 +2 21 2.000000000000000e+01 +2 22 2.100000000000000e+01 +2 23 2.200000000000000e+01 +2 24 2.300000000000000e+01 +2 25 2.400000000000000e+01 +2 26 2.500000000000000e+01 +2 27 2.600000000000000e+01 +2 28 2.700000000000000e+01 +2 29 2.800000000000000e+01 +2 30 2.900000000000000e+01 +2 31 3.000000000000000e+01 +2 32 3.100000000000000e+01 +2 33 3.200000000000000e+01 +2 34 3.300000000000000e+01 +2 35 3.400000000000000e+01 +2 36 3.500000000000000e+01 +2 37 3.600000000000000e+01 +2 38 3.700000000000000e+01 +2 39 3.800000000000000e+01 +2 40 3.900000000000000e+01 +2 41 4.000000000000000e+01 +2 42 4.100000000000000e+01 +2 43 4.200000000000000e+01 +2 44 4.300000000000000e+01 +2 45 4.400000000000000e+01 +2 46 4.500000000000000e+01 +2 47 4.600000000000000e+01 +2 48 4.700000000000000e+01 +2 49 4.800000000000000e+01 +2 50 4.900000000000000e+01 +2 51 5.000000000000000e+01 +2 52 5.100000000000000e+01 +2 53 5.200000000000000e+01 +2 54 5.300000000000000e+01 +2 55 5.400000000000000e+01 +2 56 5.500000000000000e+01 +2 57 5.600000000000000e+01 +2 58 5.700000000000000e+01 +2 59 5.800000000000000e+01 +2 60 5.900000000000000e+01 +2 61 6.000000000000000e+01 +2 62 6.100000000000000e+01 +2 63 6.200000000000000e+01 +2 64 6.300000000000000e+01 +2 65 6.400000000000000e+01 +2 66 6.500000000000000e+01 +2 67 6.600000000000000e+01 +2 68 6.700000000000000e+01 +2 69 6.800000000000000e+01 +2 70 6.900000000000000e+01 +2 71 7.000000000000000e+01 +2 72 7.100000000000000e+01 +2 73 7.200000000000000e+01 +2 74 7.300000000000000e+01 +2 75 7.400000000000000e+01 +2 76 7.500000000000000e+01 +2 77 7.600000000000000e+01 +2 78 7.700000000000000e+01 +2 79 7.800000000000000e+01 +2 80 7.900000000000000e+01 +2 81 8.000000000000000e+01 +2 82 8.100000000000000e+01 +2 83 8.200000000000000e+01 +2 84 8.300000000000000e+01 +2 85 8.400000000000000e+01 +2 86 8.500000000000000e+01 +2 87 8.600000000000000e+01 +2 88 8.700000000000000e+01 +2 89 8.800000000000000e+01 +2 90 8.900000000000000e+01 +2 91 9.000000000000000e+01 +2 92 9.100000000000000e+01 +2 93 9.200000000000000e+01 +2 94 9.300000000000000e+01 +2 95 9.400000000000000e+01 +2 96 9.500000000000000e+01 +2 97 9.600000000000000e+01 +2 98 9.700000000000000e+01 +2 99 9.800000000000000e+01 +2 100 9.900000000000000e+01 +3 3 1.000000000000000e+00 +3 4 2.000000000000000e+00 +3 5 3.000000000000000e+00 +3 6 4.000000000000000e+00 +3 7 5.000000000000000e+00 +3 8 6.000000000000000e+00 +3 9 7.000000000000000e+00 +3 10 8.000000000000000e+00 +3 11 9.000000000000000e+00 +3 12 1.000000000000000e+01 +3 13 1.100000000000000e+01 +3 14 1.200000000000000e+01 +3 15 1.300000000000000e+01 +3 16 1.400000000000000e+01 +3 17 1.500000000000000e+01 +3 18 1.600000000000000e+01 +3 19 1.700000000000000e+01 +3 20 1.800000000000000e+01 +3 21 1.900000000000000e+01 +3 22 2.000000000000000e+01 +3 23 2.100000000000000e+01 +3 24 2.200000000000000e+01 +3 25 2.300000000000000e+01 +3 26 2.400000000000000e+01 +3 27 2.500000000000000e+01 +3 28 2.600000000000000e+01 +3 29 2.700000000000000e+01 +3 30 2.800000000000000e+01 +3 31 2.900000000000000e+01 +3 32 3.000000000000000e+01 +3 33 3.100000000000000e+01 +3 34 3.200000000000000e+01 +3 35 3.300000000000000e+01 +3 36 3.400000000000000e+01 +3 37 3.500000000000000e+01 +3 38 3.600000000000000e+01 +3 39 3.700000000000000e+01 +3 40 3.800000000000000e+01 +3 41 3.900000000000000e+01 +3 42 4.000000000000000e+01 +3 43 4.100000000000000e+01 +3 44 4.200000000000000e+01 +3 45 4.300000000000000e+01 +3 46 4.400000000000000e+01 +3 47 4.500000000000000e+01 +3 48 4.600000000000000e+01 +3 49 4.700000000000000e+01 +3 50 4.800000000000000e+01 +3 51 4.900000000000000e+01 +3 52 5.000000000000000e+01 +3 53 5.100000000000000e+01 +3 54 5.200000000000000e+01 +3 55 5.300000000000000e+01 +3 56 5.400000000000000e+01 +3 57 5.500000000000000e+01 +3 58 5.600000000000000e+01 +3 59 5.700000000000000e+01 +3 60 5.800000000000000e+01 +3 61 5.900000000000000e+01 +3 62 6.000000000000000e+01 +3 63 6.100000000000000e+01 +3 64 6.200000000000000e+01 +3 65 6.300000000000000e+01 +3 66 6.400000000000000e+01 +3 67 6.500000000000000e+01 +3 68 6.600000000000000e+01 +3 69 6.700000000000000e+01 +3 70 6.800000000000000e+01 +3 71 6.900000000000000e+01 +3 72 7.000000000000000e+01 +3 73 7.100000000000000e+01 +3 74 7.200000000000000e+01 +3 75 7.300000000000000e+01 +3 76 7.400000000000000e+01 +3 77 7.500000000000000e+01 +3 78 7.600000000000000e+01 +3 79 7.700000000000000e+01 +3 80 7.800000000000000e+01 +3 81 7.900000000000000e+01 +3 82 8.000000000000000e+01 +3 83 8.100000000000000e+01 +3 84 8.200000000000000e+01 +3 85 8.300000000000000e+01 +3 86 8.400000000000000e+01 +3 87 8.500000000000000e+01 +3 88 8.600000000000000e+01 +3 89 8.700000000000000e+01 +3 90 8.800000000000000e+01 +3 91 8.900000000000000e+01 +3 92 9.000000000000000e+01 +3 93 9.100000000000000e+01 +3 94 9.200000000000000e+01 +3 95 9.300000000000000e+01 +3 96 9.400000000000000e+01 +3 97 9.500000000000000e+01 +3 98 9.600000000000000e+01 +3 99 9.700000000000000e+01 +3 100 9.800000000000000e+01 +4 4 1.000000000000000e+00 +4 5 2.000000000000000e+00 +4 6 3.000000000000000e+00 +4 7 4.000000000000000e+00 +4 8 5.000000000000000e+00 +4 9 6.000000000000000e+00 +4 10 7.000000000000000e+00 +4 11 8.000000000000000e+00 +4 12 9.000000000000000e+00 +4 13 1.000000000000000e+01 +4 14 1.100000000000000e+01 +4 15 1.200000000000000e+01 +4 16 1.300000000000000e+01 +4 17 1.400000000000000e+01 +4 18 1.500000000000000e+01 +4 19 1.600000000000000e+01 +4 20 1.700000000000000e+01 +4 21 1.800000000000000e+01 +4 22 1.900000000000000e+01 +4 23 2.000000000000000e+01 +4 24 2.100000000000000e+01 +4 25 2.200000000000000e+01 +4 26 2.300000000000000e+01 +4 27 2.400000000000000e+01 +4 28 2.500000000000000e+01 +4 29 2.600000000000000e+01 +4 30 2.700000000000000e+01 +4 31 2.800000000000000e+01 +4 32 2.900000000000000e+01 +4 33 3.000000000000000e+01 +4 34 3.100000000000000e+01 +4 35 3.200000000000000e+01 +4 36 3.300000000000000e+01 +4 37 3.400000000000000e+01 +4 38 3.500000000000000e+01 +4 39 3.600000000000000e+01 +4 40 3.700000000000000e+01 +4 41 3.800000000000000e+01 +4 42 3.900000000000000e+01 +4 43 4.000000000000000e+01 +4 44 4.100000000000000e+01 +4 45 4.200000000000000e+01 +4 46 4.300000000000000e+01 +4 47 4.400000000000000e+01 +4 48 4.500000000000000e+01 +4 49 4.600000000000000e+01 +4 50 4.700000000000000e+01 +4 51 4.800000000000000e+01 +4 52 4.900000000000000e+01 +4 53 5.000000000000000e+01 +4 54 5.100000000000000e+01 +4 55 5.200000000000000e+01 +4 56 5.300000000000000e+01 +4 57 5.400000000000000e+01 +4 58 5.500000000000000e+01 +4 59 5.600000000000000e+01 +4 60 5.700000000000000e+01 +4 61 5.800000000000000e+01 +4 62 5.900000000000000e+01 +4 63 6.000000000000000e+01 +4 64 6.100000000000000e+01 +4 65 6.200000000000000e+01 +4 66 6.300000000000000e+01 +4 67 6.400000000000000e+01 +4 68 6.500000000000000e+01 +4 69 6.600000000000000e+01 +4 70 6.700000000000000e+01 +4 71 6.800000000000000e+01 +4 72 6.900000000000000e+01 +4 73 7.000000000000000e+01 +4 74 7.100000000000000e+01 +4 75 7.200000000000000e+01 +4 76 7.300000000000000e+01 +4 77 7.400000000000000e+01 +4 78 7.500000000000000e+01 +4 79 7.600000000000000e+01 +4 80 7.700000000000000e+01 +4 81 7.800000000000000e+01 +4 82 7.900000000000000e+01 +4 83 8.000000000000000e+01 +4 84 8.100000000000000e+01 +4 85 8.200000000000000e+01 +4 86 8.300000000000000e+01 +4 87 8.400000000000000e+01 +4 88 8.500000000000000e+01 +4 89 8.600000000000000e+01 +4 90 8.700000000000000e+01 +4 91 8.800000000000000e+01 +4 92 8.900000000000000e+01 +4 93 9.000000000000000e+01 +4 94 9.100000000000000e+01 +4 95 9.200000000000000e+01 +4 96 9.300000000000000e+01 +4 97 9.400000000000000e+01 +4 98 9.500000000000000e+01 +4 99 9.600000000000000e+01 +4 100 9.700000000000000e+01 +5 5 1.000000000000000e+00 +5 6 2.000000000000000e+00 +5 7 3.000000000000000e+00 +5 8 4.000000000000000e+00 +5 9 5.000000000000000e+00 +5 10 6.000000000000000e+00 +5 11 7.000000000000000e+00 +5 12 8.000000000000000e+00 +5 13 9.000000000000000e+00 +5 14 1.000000000000000e+01 +5 15 1.100000000000000e+01 +5 16 1.200000000000000e+01 +5 17 1.300000000000000e+01 +5 18 1.400000000000000e+01 +5 19 1.500000000000000e+01 +5 20 1.600000000000000e+01 +5 21 1.700000000000000e+01 +5 22 1.800000000000000e+01 +5 23 1.900000000000000e+01 +5 24 2.000000000000000e+01 +5 25 2.100000000000000e+01 +5 26 2.200000000000000e+01 +5 27 2.300000000000000e+01 +5 28 2.400000000000000e+01 +5 29 2.500000000000000e+01 +5 30 2.600000000000000e+01 +5 31 2.700000000000000e+01 +5 32 2.800000000000000e+01 +5 33 2.900000000000000e+01 +5 34 3.000000000000000e+01 +5 35 3.100000000000000e+01 +5 36 3.200000000000000e+01 +5 37 3.300000000000000e+01 +5 38 3.400000000000000e+01 +5 39 3.500000000000000e+01 +5 40 3.600000000000000e+01 +5 41 3.700000000000000e+01 +5 42 3.800000000000000e+01 +5 43 3.900000000000000e+01 +5 44 4.000000000000000e+01 +5 45 4.100000000000000e+01 +5 46 4.200000000000000e+01 +5 47 4.300000000000000e+01 +5 48 4.400000000000000e+01 +5 49 4.500000000000000e+01 +5 50 4.600000000000000e+01 +5 51 4.700000000000000e+01 +5 52 4.800000000000000e+01 +5 53 4.900000000000000e+01 +5 54 5.000000000000000e+01 +5 55 5.100000000000000e+01 +5 56 5.200000000000000e+01 +5 57 5.300000000000000e+01 +5 58 5.400000000000000e+01 +5 59 5.500000000000000e+01 +5 60 5.600000000000000e+01 +5 61 5.700000000000000e+01 +5 62 5.800000000000000e+01 +5 63 5.900000000000000e+01 +5 64 6.000000000000000e+01 +5 65 6.100000000000000e+01 +5 66 6.200000000000000e+01 +5 67 6.300000000000000e+01 +5 68 6.400000000000000e+01 +5 69 6.500000000000000e+01 +5 70 6.600000000000000e+01 +5 71 6.700000000000000e+01 +5 72 6.800000000000000e+01 +5 73 6.900000000000000e+01 +5 74 7.000000000000000e+01 +5 75 7.100000000000000e+01 +5 76 7.200000000000000e+01 +5 77 7.300000000000000e+01 +5 78 7.400000000000000e+01 +5 79 7.500000000000000e+01 +5 80 7.600000000000000e+01 +5 81 7.700000000000000e+01 +5 82 7.800000000000000e+01 +5 83 7.900000000000000e+01 +5 84 8.000000000000000e+01 +5 85 8.100000000000000e+01 +5 86 8.200000000000000e+01 +5 87 8.300000000000000e+01 +5 88 8.400000000000000e+01 +5 89 8.500000000000000e+01 +5 90 8.600000000000000e+01 +5 91 8.700000000000000e+01 +5 92 8.800000000000000e+01 +5 93 8.900000000000000e+01 +5 94 9.000000000000000e+01 +5 95 9.100000000000000e+01 +5 96 9.200000000000000e+01 +5 97 9.300000000000000e+01 +5 98 9.400000000000000e+01 +5 99 9.500000000000000e+01 +5 100 9.600000000000000e+01 +6 6 1.000000000000000e+00 +6 7 2.000000000000000e+00 +6 8 3.000000000000000e+00 +6 9 4.000000000000000e+00 +6 10 5.000000000000000e+00 +6 11 6.000000000000000e+00 +6 12 7.000000000000000e+00 +6 13 8.000000000000000e+00 +6 14 9.000000000000000e+00 +6 15 1.000000000000000e+01 +6 16 1.100000000000000e+01 +6 17 1.200000000000000e+01 +6 18 1.300000000000000e+01 +6 19 1.400000000000000e+01 +6 20 1.500000000000000e+01 +6 21 1.600000000000000e+01 +6 22 1.700000000000000e+01 +6 23 1.800000000000000e+01 +6 24 1.900000000000000e+01 +6 25 2.000000000000000e+01 +6 26 2.100000000000000e+01 +6 27 2.200000000000000e+01 +6 28 2.300000000000000e+01 +6 29 2.400000000000000e+01 +6 30 2.500000000000000e+01 +6 31 2.600000000000000e+01 +6 32 2.700000000000000e+01 +6 33 2.800000000000000e+01 +6 34 2.900000000000000e+01 +6 35 3.000000000000000e+01 +6 36 3.100000000000000e+01 +6 37 3.200000000000000e+01 +6 38 3.300000000000000e+01 +6 39 3.400000000000000e+01 +6 40 3.500000000000000e+01 +6 41 3.600000000000000e+01 +6 42 3.700000000000000e+01 +6 43 3.800000000000000e+01 +6 44 3.900000000000000e+01 +6 45 4.000000000000000e+01 +6 46 4.100000000000000e+01 +6 47 4.200000000000000e+01 +6 48 4.300000000000000e+01 +6 49 4.400000000000000e+01 +6 50 4.500000000000000e+01 +6 51 4.600000000000000e+01 +6 52 4.700000000000000e+01 +6 53 4.800000000000000e+01 +6 54 4.900000000000000e+01 +6 55 5.000000000000000e+01 +6 56 5.100000000000000e+01 +6 57 5.200000000000000e+01 +6 58 5.300000000000000e+01 +6 59 5.400000000000000e+01 +6 60 5.500000000000000e+01 +6 61 5.600000000000000e+01 +6 62 5.700000000000000e+01 +6 63 5.800000000000000e+01 +6 64 5.900000000000000e+01 +6 65 6.000000000000000e+01 +6 66 6.100000000000000e+01 +6 67 6.200000000000000e+01 +6 68 6.300000000000000e+01 +6 69 6.400000000000000e+01 +6 70 6.500000000000000e+01 +6 71 6.600000000000000e+01 +6 72 6.700000000000000e+01 +6 73 6.800000000000000e+01 +6 74 6.900000000000000e+01 +6 75 7.000000000000000e+01 +6 76 7.100000000000000e+01 +6 77 7.200000000000000e+01 +6 78 7.300000000000000e+01 +6 79 7.400000000000000e+01 +6 80 7.500000000000000e+01 +6 81 7.600000000000000e+01 +6 82 7.700000000000000e+01 +6 83 7.800000000000000e+01 +6 84 7.900000000000000e+01 +6 85 8.000000000000000e+01 +6 86 8.100000000000000e+01 +6 87 8.200000000000000e+01 +6 88 8.300000000000000e+01 +6 89 8.400000000000000e+01 +6 90 8.500000000000000e+01 +6 91 8.600000000000000e+01 +6 92 8.700000000000000e+01 +6 93 8.800000000000000e+01 +6 94 8.900000000000000e+01 +6 95 9.000000000000000e+01 +6 96 9.100000000000000e+01 +6 97 9.200000000000000e+01 +6 98 9.300000000000000e+01 +6 99 9.400000000000000e+01 +6 100 9.500000000000000e+01 +7 7 1.000000000000000e+00 +7 8 2.000000000000000e+00 +7 9 3.000000000000000e+00 +7 10 4.000000000000000e+00 +7 11 5.000000000000000e+00 +7 12 6.000000000000000e+00 +7 13 7.000000000000000e+00 +7 14 8.000000000000000e+00 +7 15 9.000000000000000e+00 +7 16 1.000000000000000e+01 +7 17 1.100000000000000e+01 +7 18 1.200000000000000e+01 +7 19 1.300000000000000e+01 +7 20 1.400000000000000e+01 +7 21 1.500000000000000e+01 +7 22 1.600000000000000e+01 +7 23 1.700000000000000e+01 +7 24 1.800000000000000e+01 +7 25 1.900000000000000e+01 +7 26 2.000000000000000e+01 +7 27 2.100000000000000e+01 +7 28 2.200000000000000e+01 +7 29 2.300000000000000e+01 +7 30 2.400000000000000e+01 +7 31 2.500000000000000e+01 +7 32 2.600000000000000e+01 +7 33 2.700000000000000e+01 +7 34 2.800000000000000e+01 +7 35 2.900000000000000e+01 +7 36 3.000000000000000e+01 +7 37 3.100000000000000e+01 +7 38 3.200000000000000e+01 +7 39 3.300000000000000e+01 +7 40 3.400000000000000e+01 +7 41 3.500000000000000e+01 +7 42 3.600000000000000e+01 +7 43 3.700000000000000e+01 +7 44 3.800000000000000e+01 +7 45 3.900000000000000e+01 +7 46 4.000000000000000e+01 +7 47 4.100000000000000e+01 +7 48 4.200000000000000e+01 +7 49 4.300000000000000e+01 +7 50 4.400000000000000e+01 +7 51 4.500000000000000e+01 +7 52 4.600000000000000e+01 +7 53 4.700000000000000e+01 +7 54 4.800000000000000e+01 +7 55 4.900000000000000e+01 +7 56 5.000000000000000e+01 +7 57 5.100000000000000e+01 +7 58 5.200000000000000e+01 +7 59 5.300000000000000e+01 +7 60 5.400000000000000e+01 +7 61 5.500000000000000e+01 +7 62 5.600000000000000e+01 +7 63 5.700000000000000e+01 +7 64 5.800000000000000e+01 +7 65 5.900000000000000e+01 +7 66 6.000000000000000e+01 +7 67 6.100000000000000e+01 +7 68 6.200000000000000e+01 +7 69 6.300000000000000e+01 +7 70 6.400000000000000e+01 +7 71 6.500000000000000e+01 +7 72 6.600000000000000e+01 +7 73 6.700000000000000e+01 +7 74 6.800000000000000e+01 +7 75 6.900000000000000e+01 +7 76 7.000000000000000e+01 +7 77 7.100000000000000e+01 +7 78 7.200000000000000e+01 +7 79 7.300000000000000e+01 +7 80 7.400000000000000e+01 +7 81 7.500000000000000e+01 +7 82 7.600000000000000e+01 +7 83 7.700000000000000e+01 +7 84 7.800000000000000e+01 +7 85 7.900000000000000e+01 +7 86 8.000000000000000e+01 +7 87 8.100000000000000e+01 +7 88 8.200000000000000e+01 +7 89 8.300000000000000e+01 +7 90 8.400000000000000e+01 +7 91 8.500000000000000e+01 +7 92 8.600000000000000e+01 +7 93 8.700000000000000e+01 +7 94 8.800000000000000e+01 +7 95 8.900000000000000e+01 +7 96 9.000000000000000e+01 +7 97 9.100000000000000e+01 +7 98 9.200000000000000e+01 +7 99 9.300000000000000e+01 +7 100 9.400000000000000e+01 +8 8 1.000000000000000e+00 +8 9 2.000000000000000e+00 +8 10 3.000000000000000e+00 +8 11 4.000000000000000e+00 +8 12 5.000000000000000e+00 +8 13 6.000000000000000e+00 +8 14 7.000000000000000e+00 +8 15 8.000000000000000e+00 +8 16 9.000000000000000e+00 +8 17 1.000000000000000e+01 +8 18 1.100000000000000e+01 +8 19 1.200000000000000e+01 +8 20 1.300000000000000e+01 +8 21 1.400000000000000e+01 +8 22 1.500000000000000e+01 +8 23 1.600000000000000e+01 +8 24 1.700000000000000e+01 +8 25 1.800000000000000e+01 +8 26 1.900000000000000e+01 +8 27 2.000000000000000e+01 +8 28 2.100000000000000e+01 +8 29 2.200000000000000e+01 +8 30 2.300000000000000e+01 +8 31 2.400000000000000e+01 +8 32 2.500000000000000e+01 +8 33 2.600000000000000e+01 +8 34 2.700000000000000e+01 +8 35 2.800000000000000e+01 +8 36 2.900000000000000e+01 +8 37 3.000000000000000e+01 +8 38 3.100000000000000e+01 +8 39 3.200000000000000e+01 +8 40 3.300000000000000e+01 +8 41 3.400000000000000e+01 +8 42 3.500000000000000e+01 +8 43 3.600000000000000e+01 +8 44 3.700000000000000e+01 +8 45 3.800000000000000e+01 +8 46 3.900000000000000e+01 +8 47 4.000000000000000e+01 +8 48 4.100000000000000e+01 +8 49 4.200000000000000e+01 +8 50 4.300000000000000e+01 +8 51 4.400000000000000e+01 +8 52 4.500000000000000e+01 +8 53 4.600000000000000e+01 +8 54 4.700000000000000e+01 +8 55 4.800000000000000e+01 +8 56 4.900000000000000e+01 +8 57 5.000000000000000e+01 +8 58 5.100000000000000e+01 +8 59 5.200000000000000e+01 +8 60 5.300000000000000e+01 +8 61 5.400000000000000e+01 +8 62 5.500000000000000e+01 +8 63 5.600000000000000e+01 +8 64 5.700000000000000e+01 +8 65 5.800000000000000e+01 +8 66 5.900000000000000e+01 +8 67 6.000000000000000e+01 +8 68 6.100000000000000e+01 +8 69 6.200000000000000e+01 +8 70 6.300000000000000e+01 +8 71 6.400000000000000e+01 +8 72 6.500000000000000e+01 +8 73 6.600000000000000e+01 +8 74 6.700000000000000e+01 +8 75 6.800000000000000e+01 +8 76 6.900000000000000e+01 +8 77 7.000000000000000e+01 +8 78 7.100000000000000e+01 +8 79 7.200000000000000e+01 +8 80 7.300000000000000e+01 +8 81 7.400000000000000e+01 +8 82 7.500000000000000e+01 +8 83 7.600000000000000e+01 +8 84 7.700000000000000e+01 +8 85 7.800000000000000e+01 +8 86 7.900000000000000e+01 +8 87 8.000000000000000e+01 +8 88 8.100000000000000e+01 +8 89 8.200000000000000e+01 +8 90 8.300000000000000e+01 +8 91 8.400000000000000e+01 +8 92 8.500000000000000e+01 +8 93 8.600000000000000e+01 +8 94 8.700000000000000e+01 +8 95 8.800000000000000e+01 +8 96 8.900000000000000e+01 +8 97 9.000000000000000e+01 +8 98 9.100000000000000e+01 +8 99 9.200000000000000e+01 +8 100 9.300000000000000e+01 +9 9 1.000000000000000e+00 +9 10 2.000000000000000e+00 +9 11 3.000000000000000e+00 +9 12 4.000000000000000e+00 +9 13 5.000000000000000e+00 +9 14 6.000000000000000e+00 +9 15 7.000000000000000e+00 +9 16 8.000000000000000e+00 +9 17 9.000000000000000e+00 +9 18 1.000000000000000e+01 +9 19 1.100000000000000e+01 +9 20 1.200000000000000e+01 +9 21 1.300000000000000e+01 +9 22 1.400000000000000e+01 +9 23 1.500000000000000e+01 +9 24 1.600000000000000e+01 +9 25 1.700000000000000e+01 +9 26 1.800000000000000e+01 +9 27 1.900000000000000e+01 +9 28 2.000000000000000e+01 +9 29 2.100000000000000e+01 +9 30 2.200000000000000e+01 +9 31 2.300000000000000e+01 +9 32 2.400000000000000e+01 +9 33 2.500000000000000e+01 +9 34 2.600000000000000e+01 +9 35 2.700000000000000e+01 +9 36 2.800000000000000e+01 +9 37 2.900000000000000e+01 +9 38 3.000000000000000e+01 +9 39 3.100000000000000e+01 +9 40 3.200000000000000e+01 +9 41 3.300000000000000e+01 +9 42 3.400000000000000e+01 +9 43 3.500000000000000e+01 +9 44 3.600000000000000e+01 +9 45 3.700000000000000e+01 +9 46 3.800000000000000e+01 +9 47 3.900000000000000e+01 +9 48 4.000000000000000e+01 +9 49 4.100000000000000e+01 +9 50 4.200000000000000e+01 +9 51 4.300000000000000e+01 +9 52 4.400000000000000e+01 +9 53 4.500000000000000e+01 +9 54 4.600000000000000e+01 +9 55 4.700000000000000e+01 +9 56 4.800000000000000e+01 +9 57 4.900000000000000e+01 +9 58 5.000000000000000e+01 +9 59 5.100000000000000e+01 +9 60 5.200000000000000e+01 +9 61 5.300000000000000e+01 +9 62 5.400000000000000e+01 +9 63 5.500000000000000e+01 +9 64 5.600000000000000e+01 +9 65 5.700000000000000e+01 +9 66 5.800000000000000e+01 +9 67 5.900000000000000e+01 +9 68 6.000000000000000e+01 +9 69 6.100000000000000e+01 +9 70 6.200000000000000e+01 +9 71 6.300000000000000e+01 +9 72 6.400000000000000e+01 +9 73 6.500000000000000e+01 +9 74 6.600000000000000e+01 +9 75 6.700000000000000e+01 +9 76 6.800000000000000e+01 +9 77 6.900000000000000e+01 +9 78 7.000000000000000e+01 +9 79 7.100000000000000e+01 +9 80 7.200000000000000e+01 +9 81 7.300000000000000e+01 +9 82 7.400000000000000e+01 +9 83 7.500000000000000e+01 +9 84 7.600000000000000e+01 +9 85 7.700000000000000e+01 +9 86 7.800000000000000e+01 +9 87 7.900000000000000e+01 +9 88 8.000000000000000e+01 +9 89 8.100000000000000e+01 +9 90 8.200000000000000e+01 +9 91 8.300000000000000e+01 +9 92 8.400000000000000e+01 +9 93 8.500000000000000e+01 +9 94 8.600000000000000e+01 +9 95 8.700000000000000e+01 +9 96 8.800000000000000e+01 +9 97 8.900000000000000e+01 +9 98 9.000000000000000e+01 +9 99 9.100000000000000e+01 +9 100 9.200000000000000e+01 +10 10 1.000000000000000e+00 +10 11 2.000000000000000e+00 +10 12 3.000000000000000e+00 +10 13 4.000000000000000e+00 +10 14 5.000000000000000e+00 +10 15 6.000000000000000e+00 +10 16 7.000000000000000e+00 +10 17 8.000000000000000e+00 +10 18 9.000000000000000e+00 +10 19 1.000000000000000e+01 +10 20 1.100000000000000e+01 +10 21 1.200000000000000e+01 +10 22 1.300000000000000e+01 +10 23 1.400000000000000e+01 +10 24 1.500000000000000e+01 +10 25 1.600000000000000e+01 +10 26 1.700000000000000e+01 +10 27 1.800000000000000e+01 +10 28 1.900000000000000e+01 +10 29 2.000000000000000e+01 +10 30 2.100000000000000e+01 +10 31 2.200000000000000e+01 +10 32 2.300000000000000e+01 +10 33 2.400000000000000e+01 +10 34 2.500000000000000e+01 +10 35 2.600000000000000e+01 +10 36 2.700000000000000e+01 +10 37 2.800000000000000e+01 +10 38 2.900000000000000e+01 +10 39 3.000000000000000e+01 +10 40 3.100000000000000e+01 +10 41 3.200000000000000e+01 +10 42 3.300000000000000e+01 +10 43 3.400000000000000e+01 +10 44 3.500000000000000e+01 +10 45 3.600000000000000e+01 +10 46 3.700000000000000e+01 +10 47 3.800000000000000e+01 +10 48 3.900000000000000e+01 +10 49 4.000000000000000e+01 +10 50 4.100000000000000e+01 +10 51 4.200000000000000e+01 +10 52 4.300000000000000e+01 +10 53 4.400000000000000e+01 +10 54 4.500000000000000e+01 +10 55 4.600000000000000e+01 +10 56 4.700000000000000e+01 +10 57 4.800000000000000e+01 +10 58 4.900000000000000e+01 +10 59 5.000000000000000e+01 +10 60 5.100000000000000e+01 +10 61 5.200000000000000e+01 +10 62 5.300000000000000e+01 +10 63 5.400000000000000e+01 +10 64 5.500000000000000e+01 +10 65 5.600000000000000e+01 +10 66 5.700000000000000e+01 +10 67 5.800000000000000e+01 +10 68 5.900000000000000e+01 +10 69 6.000000000000000e+01 +10 70 6.100000000000000e+01 +10 71 6.200000000000000e+01 +10 72 6.300000000000000e+01 +10 73 6.400000000000000e+01 +10 74 6.500000000000000e+01 +10 75 6.600000000000000e+01 +10 76 6.700000000000000e+01 +10 77 6.800000000000000e+01 +10 78 6.900000000000000e+01 +10 79 7.000000000000000e+01 +10 80 7.100000000000000e+01 +10 81 7.200000000000000e+01 +10 82 7.300000000000000e+01 +10 83 7.400000000000000e+01 +10 84 7.500000000000000e+01 +10 85 7.600000000000000e+01 +10 86 7.700000000000000e+01 +10 87 7.800000000000000e+01 +10 88 7.900000000000000e+01 +10 89 8.000000000000000e+01 +10 90 8.100000000000000e+01 +10 91 8.200000000000000e+01 +10 92 8.300000000000000e+01 +10 93 8.400000000000000e+01 +10 94 8.500000000000000e+01 +10 95 8.600000000000000e+01 +10 96 8.700000000000000e+01 +10 97 8.800000000000000e+01 +10 98 8.900000000000000e+01 +10 99 9.000000000000000e+01 +10 100 9.100000000000000e+01 +11 11 1.000000000000000e+00 +11 12 2.000000000000000e+00 +11 13 3.000000000000000e+00 +11 14 4.000000000000000e+00 +11 15 5.000000000000000e+00 +11 16 6.000000000000000e+00 +11 17 7.000000000000000e+00 +11 18 8.000000000000000e+00 +11 19 9.000000000000000e+00 +11 20 1.000000000000000e+01 +11 21 1.100000000000000e+01 +11 22 1.200000000000000e+01 +11 23 1.300000000000000e+01 +11 24 1.400000000000000e+01 +11 25 1.500000000000000e+01 +11 26 1.600000000000000e+01 +11 27 1.700000000000000e+01 +11 28 1.800000000000000e+01 +11 29 1.900000000000000e+01 +11 30 2.000000000000000e+01 +11 31 2.100000000000000e+01 +11 32 2.200000000000000e+01 +11 33 2.300000000000000e+01 +11 34 2.400000000000000e+01 +11 35 2.500000000000000e+01 +11 36 2.600000000000000e+01 +11 37 2.700000000000000e+01 +11 38 2.800000000000000e+01 +11 39 2.900000000000000e+01 +11 40 3.000000000000000e+01 +11 41 3.100000000000000e+01 +11 42 3.200000000000000e+01 +11 43 3.300000000000000e+01 +11 44 3.400000000000000e+01 +11 45 3.500000000000000e+01 +11 46 3.600000000000000e+01 +11 47 3.700000000000000e+01 +11 48 3.800000000000000e+01 +11 49 3.900000000000000e+01 +11 50 4.000000000000000e+01 +11 51 4.100000000000000e+01 +11 52 4.200000000000000e+01 +11 53 4.300000000000000e+01 +11 54 4.400000000000000e+01 +11 55 4.500000000000000e+01 +11 56 4.600000000000000e+01 +11 57 4.700000000000000e+01 +11 58 4.800000000000000e+01 +11 59 4.900000000000000e+01 +11 60 5.000000000000000e+01 +11 61 5.100000000000000e+01 +11 62 5.200000000000000e+01 +11 63 5.300000000000000e+01 +11 64 5.400000000000000e+01 +11 65 5.500000000000000e+01 +11 66 5.600000000000000e+01 +11 67 5.700000000000000e+01 +11 68 5.800000000000000e+01 +11 69 5.900000000000000e+01 +11 70 6.000000000000000e+01 +11 71 6.100000000000000e+01 +11 72 6.200000000000000e+01 +11 73 6.300000000000000e+01 +11 74 6.400000000000000e+01 +11 75 6.500000000000000e+01 +11 76 6.600000000000000e+01 +11 77 6.700000000000000e+01 +11 78 6.800000000000000e+01 +11 79 6.900000000000000e+01 +11 80 7.000000000000000e+01 +11 81 7.100000000000000e+01 +11 82 7.200000000000000e+01 +11 83 7.300000000000000e+01 +11 84 7.400000000000000e+01 +11 85 7.500000000000000e+01 +11 86 7.600000000000000e+01 +11 87 7.700000000000000e+01 +11 88 7.800000000000000e+01 +11 89 7.900000000000000e+01 +11 90 8.000000000000000e+01 +11 91 8.100000000000000e+01 +11 92 8.200000000000000e+01 +11 93 8.300000000000000e+01 +11 94 8.400000000000000e+01 +11 95 8.500000000000000e+01 +11 96 8.600000000000000e+01 +11 97 8.700000000000000e+01 +11 98 8.800000000000000e+01 +11 99 8.900000000000000e+01 +11 100 9.000000000000000e+01 +12 12 1.000000000000000e+00 +12 13 2.000000000000000e+00 +12 14 3.000000000000000e+00 +12 15 4.000000000000000e+00 +12 16 5.000000000000000e+00 +12 17 6.000000000000000e+00 +12 18 7.000000000000000e+00 +12 19 8.000000000000000e+00 +12 20 9.000000000000000e+00 +12 21 1.000000000000000e+01 +12 22 1.100000000000000e+01 +12 23 1.200000000000000e+01 +12 24 1.300000000000000e+01 +12 25 1.400000000000000e+01 +12 26 1.500000000000000e+01 +12 27 1.600000000000000e+01 +12 28 1.700000000000000e+01 +12 29 1.800000000000000e+01 +12 30 1.900000000000000e+01 +12 31 2.000000000000000e+01 +12 32 2.100000000000000e+01 +12 33 2.200000000000000e+01 +12 34 2.300000000000000e+01 +12 35 2.400000000000000e+01 +12 36 2.500000000000000e+01 +12 37 2.600000000000000e+01 +12 38 2.700000000000000e+01 +12 39 2.800000000000000e+01 +12 40 2.900000000000000e+01 +12 41 3.000000000000000e+01 +12 42 3.100000000000000e+01 +12 43 3.200000000000000e+01 +12 44 3.300000000000000e+01 +12 45 3.400000000000000e+01 +12 46 3.500000000000000e+01 +12 47 3.600000000000000e+01 +12 48 3.700000000000000e+01 +12 49 3.800000000000000e+01 +12 50 3.900000000000000e+01 +12 51 4.000000000000000e+01 +12 52 4.100000000000000e+01 +12 53 4.200000000000000e+01 +12 54 4.300000000000000e+01 +12 55 4.400000000000000e+01 +12 56 4.500000000000000e+01 +12 57 4.600000000000000e+01 +12 58 4.700000000000000e+01 +12 59 4.800000000000000e+01 +12 60 4.900000000000000e+01 +12 61 5.000000000000000e+01 +12 62 5.100000000000000e+01 +12 63 5.200000000000000e+01 +12 64 5.300000000000000e+01 +12 65 5.400000000000000e+01 +12 66 5.500000000000000e+01 +12 67 5.600000000000000e+01 +12 68 5.700000000000000e+01 +12 69 5.800000000000000e+01 +12 70 5.900000000000000e+01 +12 71 6.000000000000000e+01 +12 72 6.100000000000000e+01 +12 73 6.200000000000000e+01 +12 74 6.300000000000000e+01 +12 75 6.400000000000000e+01 +12 76 6.500000000000000e+01 +12 77 6.600000000000000e+01 +12 78 6.700000000000000e+01 +12 79 6.800000000000000e+01 +12 80 6.900000000000000e+01 +12 81 7.000000000000000e+01 +12 82 7.100000000000000e+01 +12 83 7.200000000000000e+01 +12 84 7.300000000000000e+01 +12 85 7.400000000000000e+01 +12 86 7.500000000000000e+01 +12 87 7.600000000000000e+01 +12 88 7.700000000000000e+01 +12 89 7.800000000000000e+01 +12 90 7.900000000000000e+01 +12 91 8.000000000000000e+01 +12 92 8.100000000000000e+01 +12 93 8.200000000000000e+01 +12 94 8.300000000000000e+01 +12 95 8.400000000000000e+01 +12 96 8.500000000000000e+01 +12 97 8.600000000000000e+01 +12 98 8.700000000000000e+01 +12 99 8.800000000000000e+01 +12 100 8.900000000000000e+01 +13 13 1.000000000000000e+00 +13 14 2.000000000000000e+00 +13 15 3.000000000000000e+00 +13 16 4.000000000000000e+00 +13 17 5.000000000000000e+00 +13 18 6.000000000000000e+00 +13 19 7.000000000000000e+00 +13 20 8.000000000000000e+00 +13 21 9.000000000000000e+00 +13 22 1.000000000000000e+01 +13 23 1.100000000000000e+01 +13 24 1.200000000000000e+01 +13 25 1.300000000000000e+01 +13 26 1.400000000000000e+01 +13 27 1.500000000000000e+01 +13 28 1.600000000000000e+01 +13 29 1.700000000000000e+01 +13 30 1.800000000000000e+01 +13 31 1.900000000000000e+01 +13 32 2.000000000000000e+01 +13 33 2.100000000000000e+01 +13 34 2.200000000000000e+01 +13 35 2.300000000000000e+01 +13 36 2.400000000000000e+01 +13 37 2.500000000000000e+01 +13 38 2.600000000000000e+01 +13 39 2.700000000000000e+01 +13 40 2.800000000000000e+01 +13 41 2.900000000000000e+01 +13 42 3.000000000000000e+01 +13 43 3.100000000000000e+01 +13 44 3.200000000000000e+01 +13 45 3.300000000000000e+01 +13 46 3.400000000000000e+01 +13 47 3.500000000000000e+01 +13 48 3.600000000000000e+01 +13 49 3.700000000000000e+01 +13 50 3.800000000000000e+01 +13 51 3.900000000000000e+01 +13 52 4.000000000000000e+01 +13 53 4.100000000000000e+01 +13 54 4.200000000000000e+01 +13 55 4.300000000000000e+01 +13 56 4.400000000000000e+01 +13 57 4.500000000000000e+01 +13 58 4.600000000000000e+01 +13 59 4.700000000000000e+01 +13 60 4.800000000000000e+01 +13 61 4.900000000000000e+01 +13 62 5.000000000000000e+01 +13 63 5.100000000000000e+01 +13 64 5.200000000000000e+01 +13 65 5.300000000000000e+01 +13 66 5.400000000000000e+01 +13 67 5.500000000000000e+01 +13 68 5.600000000000000e+01 +13 69 5.700000000000000e+01 +13 70 5.800000000000000e+01 +13 71 5.900000000000000e+01 +13 72 6.000000000000000e+01 +13 73 6.100000000000000e+01 +13 74 6.200000000000000e+01 +13 75 6.300000000000000e+01 +13 76 6.400000000000000e+01 +13 77 6.500000000000000e+01 +13 78 6.600000000000000e+01 +13 79 6.700000000000000e+01 +13 80 6.800000000000000e+01 +13 81 6.900000000000000e+01 +13 82 7.000000000000000e+01 +13 83 7.100000000000000e+01 +13 84 7.200000000000000e+01 +13 85 7.300000000000000e+01 +13 86 7.400000000000000e+01 +13 87 7.500000000000000e+01 +13 88 7.600000000000000e+01 +13 89 7.700000000000000e+01 +13 90 7.800000000000000e+01 +13 91 7.900000000000000e+01 +13 92 8.000000000000000e+01 +13 93 8.100000000000000e+01 +13 94 8.200000000000000e+01 +13 95 8.300000000000000e+01 +13 96 8.400000000000000e+01 +13 97 8.500000000000000e+01 +13 98 8.600000000000000e+01 +13 99 8.700000000000000e+01 +13 100 8.800000000000000e+01 +14 14 1.000000000000000e+00 +14 15 2.000000000000000e+00 +14 16 3.000000000000000e+00 +14 17 4.000000000000000e+00 +14 18 5.000000000000000e+00 +14 19 6.000000000000000e+00 +14 20 7.000000000000000e+00 +14 21 8.000000000000000e+00 +14 22 9.000000000000000e+00 +14 23 1.000000000000000e+01 +14 24 1.100000000000000e+01 +14 25 1.200000000000000e+01 +14 26 1.300000000000000e+01 +14 27 1.400000000000000e+01 +14 28 1.500000000000000e+01 +14 29 1.600000000000000e+01 +14 30 1.700000000000000e+01 +14 31 1.800000000000000e+01 +14 32 1.900000000000000e+01 +14 33 2.000000000000000e+01 +14 34 2.100000000000000e+01 +14 35 2.200000000000000e+01 +14 36 2.300000000000000e+01 +14 37 2.400000000000000e+01 +14 38 2.500000000000000e+01 +14 39 2.600000000000000e+01 +14 40 2.700000000000000e+01 +14 41 2.800000000000000e+01 +14 42 2.900000000000000e+01 +14 43 3.000000000000000e+01 +14 44 3.100000000000000e+01 +14 45 3.200000000000000e+01 +14 46 3.300000000000000e+01 +14 47 3.400000000000000e+01 +14 48 3.500000000000000e+01 +14 49 3.600000000000000e+01 +14 50 3.700000000000000e+01 +14 51 3.800000000000000e+01 +14 52 3.900000000000000e+01 +14 53 4.000000000000000e+01 +14 54 4.100000000000000e+01 +14 55 4.200000000000000e+01 +14 56 4.300000000000000e+01 +14 57 4.400000000000000e+01 +14 58 4.500000000000000e+01 +14 59 4.600000000000000e+01 +14 60 4.700000000000000e+01 +14 61 4.800000000000000e+01 +14 62 4.900000000000000e+01 +14 63 5.000000000000000e+01 +14 64 5.100000000000000e+01 +14 65 5.200000000000000e+01 +14 66 5.300000000000000e+01 +14 67 5.400000000000000e+01 +14 68 5.500000000000000e+01 +14 69 5.600000000000000e+01 +14 70 5.700000000000000e+01 +14 71 5.800000000000000e+01 +14 72 5.900000000000000e+01 +14 73 6.000000000000000e+01 +14 74 6.100000000000000e+01 +14 75 6.200000000000000e+01 +14 76 6.300000000000000e+01 +14 77 6.400000000000000e+01 +14 78 6.500000000000000e+01 +14 79 6.600000000000000e+01 +14 80 6.700000000000000e+01 +14 81 6.800000000000000e+01 +14 82 6.900000000000000e+01 +14 83 7.000000000000000e+01 +14 84 7.100000000000000e+01 +14 85 7.200000000000000e+01 +14 86 7.300000000000000e+01 +14 87 7.400000000000000e+01 +14 88 7.500000000000000e+01 +14 89 7.600000000000000e+01 +14 90 7.700000000000000e+01 +14 91 7.800000000000000e+01 +14 92 7.900000000000000e+01 +14 93 8.000000000000000e+01 +14 94 8.100000000000000e+01 +14 95 8.200000000000000e+01 +14 96 8.300000000000000e+01 +14 97 8.400000000000000e+01 +14 98 8.500000000000000e+01 +14 99 8.600000000000000e+01 +14 100 8.700000000000000e+01 +15 15 1.000000000000000e+00 +15 16 2.000000000000000e+00 +15 17 3.000000000000000e+00 +15 18 4.000000000000000e+00 +15 19 5.000000000000000e+00 +15 20 6.000000000000000e+00 +15 21 7.000000000000000e+00 +15 22 8.000000000000000e+00 +15 23 9.000000000000000e+00 +15 24 1.000000000000000e+01 +15 25 1.100000000000000e+01 +15 26 1.200000000000000e+01 +15 27 1.300000000000000e+01 +15 28 1.400000000000000e+01 +15 29 1.500000000000000e+01 +15 30 1.600000000000000e+01 +15 31 1.700000000000000e+01 +15 32 1.800000000000000e+01 +15 33 1.900000000000000e+01 +15 34 2.000000000000000e+01 +15 35 2.100000000000000e+01 +15 36 2.200000000000000e+01 +15 37 2.300000000000000e+01 +15 38 2.400000000000000e+01 +15 39 2.500000000000000e+01 +15 40 2.600000000000000e+01 +15 41 2.700000000000000e+01 +15 42 2.800000000000000e+01 +15 43 2.900000000000000e+01 +15 44 3.000000000000000e+01 +15 45 3.100000000000000e+01 +15 46 3.200000000000000e+01 +15 47 3.300000000000000e+01 +15 48 3.400000000000000e+01 +15 49 3.500000000000000e+01 +15 50 3.600000000000000e+01 +15 51 3.700000000000000e+01 +15 52 3.800000000000000e+01 +15 53 3.900000000000000e+01 +15 54 4.000000000000000e+01 +15 55 4.100000000000000e+01 +15 56 4.200000000000000e+01 +15 57 4.300000000000000e+01 +15 58 4.400000000000000e+01 +15 59 4.500000000000000e+01 +15 60 4.600000000000000e+01 +15 61 4.700000000000000e+01 +15 62 4.800000000000000e+01 +15 63 4.900000000000000e+01 +15 64 5.000000000000000e+01 +15 65 5.100000000000000e+01 +15 66 5.200000000000000e+01 +15 67 5.300000000000000e+01 +15 68 5.400000000000000e+01 +15 69 5.500000000000000e+01 +15 70 5.600000000000000e+01 +15 71 5.700000000000000e+01 +15 72 5.800000000000000e+01 +15 73 5.900000000000000e+01 +15 74 6.000000000000000e+01 +15 75 6.100000000000000e+01 +15 76 6.200000000000000e+01 +15 77 6.300000000000000e+01 +15 78 6.400000000000000e+01 +15 79 6.500000000000000e+01 +15 80 6.600000000000000e+01 +15 81 6.700000000000000e+01 +15 82 6.800000000000000e+01 +15 83 6.900000000000000e+01 +15 84 7.000000000000000e+01 +15 85 7.100000000000000e+01 +15 86 7.200000000000000e+01 +15 87 7.300000000000000e+01 +15 88 7.400000000000000e+01 +15 89 7.500000000000000e+01 +15 90 7.600000000000000e+01 +15 91 7.700000000000000e+01 +15 92 7.800000000000000e+01 +15 93 7.900000000000000e+01 +15 94 8.000000000000000e+01 +15 95 8.100000000000000e+01 +15 96 8.200000000000000e+01 +15 97 8.300000000000000e+01 +15 98 8.400000000000000e+01 +15 99 8.500000000000000e+01 +15 100 8.600000000000000e+01 +16 16 1.000000000000000e+00 +16 17 2.000000000000000e+00 +16 18 3.000000000000000e+00 +16 19 4.000000000000000e+00 +16 20 5.000000000000000e+00 +16 21 6.000000000000000e+00 +16 22 7.000000000000000e+00 +16 23 8.000000000000000e+00 +16 24 9.000000000000000e+00 +16 25 1.000000000000000e+01 +16 26 1.100000000000000e+01 +16 27 1.200000000000000e+01 +16 28 1.300000000000000e+01 +16 29 1.400000000000000e+01 +16 30 1.500000000000000e+01 +16 31 1.600000000000000e+01 +16 32 1.700000000000000e+01 +16 33 1.800000000000000e+01 +16 34 1.900000000000000e+01 +16 35 2.000000000000000e+01 +16 36 2.100000000000000e+01 +16 37 2.200000000000000e+01 +16 38 2.300000000000000e+01 +16 39 2.400000000000000e+01 +16 40 2.500000000000000e+01 +16 41 2.600000000000000e+01 +16 42 2.700000000000000e+01 +16 43 2.800000000000000e+01 +16 44 2.900000000000000e+01 +16 45 3.000000000000000e+01 +16 46 3.100000000000000e+01 +16 47 3.200000000000000e+01 +16 48 3.300000000000000e+01 +16 49 3.400000000000000e+01 +16 50 3.500000000000000e+01 +16 51 3.600000000000000e+01 +16 52 3.700000000000000e+01 +16 53 3.800000000000000e+01 +16 54 3.900000000000000e+01 +16 55 4.000000000000000e+01 +16 56 4.100000000000000e+01 +16 57 4.200000000000000e+01 +16 58 4.300000000000000e+01 +16 59 4.400000000000000e+01 +16 60 4.500000000000000e+01 +16 61 4.600000000000000e+01 +16 62 4.700000000000000e+01 +16 63 4.800000000000000e+01 +16 64 4.900000000000000e+01 +16 65 5.000000000000000e+01 +16 66 5.100000000000000e+01 +16 67 5.200000000000000e+01 +16 68 5.300000000000000e+01 +16 69 5.400000000000000e+01 +16 70 5.500000000000000e+01 +16 71 5.600000000000000e+01 +16 72 5.700000000000000e+01 +16 73 5.800000000000000e+01 +16 74 5.900000000000000e+01 +16 75 6.000000000000000e+01 +16 76 6.100000000000000e+01 +16 77 6.200000000000000e+01 +16 78 6.300000000000000e+01 +16 79 6.400000000000000e+01 +16 80 6.500000000000000e+01 +16 81 6.600000000000000e+01 +16 82 6.700000000000000e+01 +16 83 6.800000000000000e+01 +16 84 6.900000000000000e+01 +16 85 7.000000000000000e+01 +16 86 7.100000000000000e+01 +16 87 7.200000000000000e+01 +16 88 7.300000000000000e+01 +16 89 7.400000000000000e+01 +16 90 7.500000000000000e+01 +16 91 7.600000000000000e+01 +16 92 7.700000000000000e+01 +16 93 7.800000000000000e+01 +16 94 7.900000000000000e+01 +16 95 8.000000000000000e+01 +16 96 8.100000000000000e+01 +16 97 8.200000000000000e+01 +16 98 8.300000000000000e+01 +16 99 8.400000000000000e+01 +16 100 8.500000000000000e+01 +17 17 1.000000000000000e+00 +17 18 2.000000000000000e+00 +17 19 3.000000000000000e+00 +17 20 4.000000000000000e+00 +17 21 5.000000000000000e+00 +17 22 6.000000000000000e+00 +17 23 7.000000000000000e+00 +17 24 8.000000000000000e+00 +17 25 9.000000000000000e+00 +17 26 1.000000000000000e+01 +17 27 1.100000000000000e+01 +17 28 1.200000000000000e+01 +17 29 1.300000000000000e+01 +17 30 1.400000000000000e+01 +17 31 1.500000000000000e+01 +17 32 1.600000000000000e+01 +17 33 1.700000000000000e+01 +17 34 1.800000000000000e+01 +17 35 1.900000000000000e+01 +17 36 2.000000000000000e+01 +17 37 2.100000000000000e+01 +17 38 2.200000000000000e+01 +17 39 2.300000000000000e+01 +17 40 2.400000000000000e+01 +17 41 2.500000000000000e+01 +17 42 2.600000000000000e+01 +17 43 2.700000000000000e+01 +17 44 2.800000000000000e+01 +17 45 2.900000000000000e+01 +17 46 3.000000000000000e+01 +17 47 3.100000000000000e+01 +17 48 3.200000000000000e+01 +17 49 3.300000000000000e+01 +17 50 3.400000000000000e+01 +17 51 3.500000000000000e+01 +17 52 3.600000000000000e+01 +17 53 3.700000000000000e+01 +17 54 3.800000000000000e+01 +17 55 3.900000000000000e+01 +17 56 4.000000000000000e+01 +17 57 4.100000000000000e+01 +17 58 4.200000000000000e+01 +17 59 4.300000000000000e+01 +17 60 4.400000000000000e+01 +17 61 4.500000000000000e+01 +17 62 4.600000000000000e+01 +17 63 4.700000000000000e+01 +17 64 4.800000000000000e+01 +17 65 4.900000000000000e+01 +17 66 5.000000000000000e+01 +17 67 5.100000000000000e+01 +17 68 5.200000000000000e+01 +17 69 5.300000000000000e+01 +17 70 5.400000000000000e+01 +17 71 5.500000000000000e+01 +17 72 5.600000000000000e+01 +17 73 5.700000000000000e+01 +17 74 5.800000000000000e+01 +17 75 5.900000000000000e+01 +17 76 6.000000000000000e+01 +17 77 6.100000000000000e+01 +17 78 6.200000000000000e+01 +17 79 6.300000000000000e+01 +17 80 6.400000000000000e+01 +17 81 6.500000000000000e+01 +17 82 6.600000000000000e+01 +17 83 6.700000000000000e+01 +17 84 6.800000000000000e+01 +17 85 6.900000000000000e+01 +17 86 7.000000000000000e+01 +17 87 7.100000000000000e+01 +17 88 7.200000000000000e+01 +17 89 7.300000000000000e+01 +17 90 7.400000000000000e+01 +17 91 7.500000000000000e+01 +17 92 7.600000000000000e+01 +17 93 7.700000000000000e+01 +17 94 7.800000000000000e+01 +17 95 7.900000000000000e+01 +17 96 8.000000000000000e+01 +17 97 8.100000000000000e+01 +17 98 8.200000000000000e+01 +17 99 8.300000000000000e+01 +17 100 8.400000000000000e+01 +18 18 1.000000000000000e+00 +18 19 2.000000000000000e+00 +18 20 3.000000000000000e+00 +18 21 4.000000000000000e+00 +18 22 5.000000000000000e+00 +18 23 6.000000000000000e+00 +18 24 7.000000000000000e+00 +18 25 8.000000000000000e+00 +18 26 9.000000000000000e+00 +18 27 1.000000000000000e+01 +18 28 1.100000000000000e+01 +18 29 1.200000000000000e+01 +18 30 1.300000000000000e+01 +18 31 1.400000000000000e+01 +18 32 1.500000000000000e+01 +18 33 1.600000000000000e+01 +18 34 1.700000000000000e+01 +18 35 1.800000000000000e+01 +18 36 1.900000000000000e+01 +18 37 2.000000000000000e+01 +18 38 2.100000000000000e+01 +18 39 2.200000000000000e+01 +18 40 2.300000000000000e+01 +18 41 2.400000000000000e+01 +18 42 2.500000000000000e+01 +18 43 2.600000000000000e+01 +18 44 2.700000000000000e+01 +18 45 2.800000000000000e+01 +18 46 2.900000000000000e+01 +18 47 3.000000000000000e+01 +18 48 3.100000000000000e+01 +18 49 3.200000000000000e+01 +18 50 3.300000000000000e+01 +18 51 3.400000000000000e+01 +18 52 3.500000000000000e+01 +18 53 3.600000000000000e+01 +18 54 3.700000000000000e+01 +18 55 3.800000000000000e+01 +18 56 3.900000000000000e+01 +18 57 4.000000000000000e+01 +18 58 4.100000000000000e+01 +18 59 4.200000000000000e+01 +18 60 4.300000000000000e+01 +18 61 4.400000000000000e+01 +18 62 4.500000000000000e+01 +18 63 4.600000000000000e+01 +18 64 4.700000000000000e+01 +18 65 4.800000000000000e+01 +18 66 4.900000000000000e+01 +18 67 5.000000000000000e+01 +18 68 5.100000000000000e+01 +18 69 5.200000000000000e+01 +18 70 5.300000000000000e+01 +18 71 5.400000000000000e+01 +18 72 5.500000000000000e+01 +18 73 5.600000000000000e+01 +18 74 5.700000000000000e+01 +18 75 5.800000000000000e+01 +18 76 5.900000000000000e+01 +18 77 6.000000000000000e+01 +18 78 6.100000000000000e+01 +18 79 6.200000000000000e+01 +18 80 6.300000000000000e+01 +18 81 6.400000000000000e+01 +18 82 6.500000000000000e+01 +18 83 6.600000000000000e+01 +18 84 6.700000000000000e+01 +18 85 6.800000000000000e+01 +18 86 6.900000000000000e+01 +18 87 7.000000000000000e+01 +18 88 7.100000000000000e+01 +18 89 7.200000000000000e+01 +18 90 7.300000000000000e+01 +18 91 7.400000000000000e+01 +18 92 7.500000000000000e+01 +18 93 7.600000000000000e+01 +18 94 7.700000000000000e+01 +18 95 7.800000000000000e+01 +18 96 7.900000000000000e+01 +18 97 8.000000000000000e+01 +18 98 8.100000000000000e+01 +18 99 8.200000000000000e+01 +18 100 8.300000000000000e+01 +19 19 1.000000000000000e+00 +19 20 2.000000000000000e+00 +19 21 3.000000000000000e+00 +19 22 4.000000000000000e+00 +19 23 5.000000000000000e+00 +19 24 6.000000000000000e+00 +19 25 7.000000000000000e+00 +19 26 8.000000000000000e+00 +19 27 9.000000000000000e+00 +19 28 1.000000000000000e+01 +19 29 1.100000000000000e+01 +19 30 1.200000000000000e+01 +19 31 1.300000000000000e+01 +19 32 1.400000000000000e+01 +19 33 1.500000000000000e+01 +19 34 1.600000000000000e+01 +19 35 1.700000000000000e+01 +19 36 1.800000000000000e+01 +19 37 1.900000000000000e+01 +19 38 2.000000000000000e+01 +19 39 2.100000000000000e+01 +19 40 2.200000000000000e+01 +19 41 2.300000000000000e+01 +19 42 2.400000000000000e+01 +19 43 2.500000000000000e+01 +19 44 2.600000000000000e+01 +19 45 2.700000000000000e+01 +19 46 2.800000000000000e+01 +19 47 2.900000000000000e+01 +19 48 3.000000000000000e+01 +19 49 3.100000000000000e+01 +19 50 3.200000000000000e+01 +19 51 3.300000000000000e+01 +19 52 3.400000000000000e+01 +19 53 3.500000000000000e+01 +19 54 3.600000000000000e+01 +19 55 3.700000000000000e+01 +19 56 3.800000000000000e+01 +19 57 3.900000000000000e+01 +19 58 4.000000000000000e+01 +19 59 4.100000000000000e+01 +19 60 4.200000000000000e+01 +19 61 4.300000000000000e+01 +19 62 4.400000000000000e+01 +19 63 4.500000000000000e+01 +19 64 4.600000000000000e+01 +19 65 4.700000000000000e+01 +19 66 4.800000000000000e+01 +19 67 4.900000000000000e+01 +19 68 5.000000000000000e+01 +19 69 5.100000000000000e+01 +19 70 5.200000000000000e+01 +19 71 5.300000000000000e+01 +19 72 5.400000000000000e+01 +19 73 5.500000000000000e+01 +19 74 5.600000000000000e+01 +19 75 5.700000000000000e+01 +19 76 5.800000000000000e+01 +19 77 5.900000000000000e+01 +19 78 6.000000000000000e+01 +19 79 6.100000000000000e+01 +19 80 6.200000000000000e+01 +19 81 6.300000000000000e+01 +19 82 6.400000000000000e+01 +19 83 6.500000000000000e+01 +19 84 6.600000000000000e+01 +19 85 6.700000000000000e+01 +19 86 6.800000000000000e+01 +19 87 6.900000000000000e+01 +19 88 7.000000000000000e+01 +19 89 7.100000000000000e+01 +19 90 7.200000000000000e+01 +19 91 7.300000000000000e+01 +19 92 7.400000000000000e+01 +19 93 7.500000000000000e+01 +19 94 7.600000000000000e+01 +19 95 7.700000000000000e+01 +19 96 7.800000000000000e+01 +19 97 7.900000000000000e+01 +19 98 8.000000000000000e+01 +19 99 8.100000000000000e+01 +19 100 8.200000000000000e+01 +20 20 1.000000000000000e+00 +20 21 2.000000000000000e+00 +20 22 3.000000000000000e+00 +20 23 4.000000000000000e+00 +20 24 5.000000000000000e+00 +20 25 6.000000000000000e+00 +20 26 7.000000000000000e+00 +20 27 8.000000000000000e+00 +20 28 9.000000000000000e+00 +20 29 1.000000000000000e+01 +20 30 1.100000000000000e+01 +20 31 1.200000000000000e+01 +20 32 1.300000000000000e+01 +20 33 1.400000000000000e+01 +20 34 1.500000000000000e+01 +20 35 1.600000000000000e+01 +20 36 1.700000000000000e+01 +20 37 1.800000000000000e+01 +20 38 1.900000000000000e+01 +20 39 2.000000000000000e+01 +20 40 2.100000000000000e+01 +20 41 2.200000000000000e+01 +20 42 2.300000000000000e+01 +20 43 2.400000000000000e+01 +20 44 2.500000000000000e+01 +20 45 2.600000000000000e+01 +20 46 2.700000000000000e+01 +20 47 2.800000000000000e+01 +20 48 2.900000000000000e+01 +20 49 3.000000000000000e+01 +20 50 3.100000000000000e+01 +20 51 3.200000000000000e+01 +20 52 3.300000000000000e+01 +20 53 3.400000000000000e+01 +20 54 3.500000000000000e+01 +20 55 3.600000000000000e+01 +20 56 3.700000000000000e+01 +20 57 3.800000000000000e+01 +20 58 3.900000000000000e+01 +20 59 4.000000000000000e+01 +20 60 4.100000000000000e+01 +20 61 4.200000000000000e+01 +20 62 4.300000000000000e+01 +20 63 4.400000000000000e+01 +20 64 4.500000000000000e+01 +20 65 4.600000000000000e+01 +20 66 4.700000000000000e+01 +20 67 4.800000000000000e+01 +20 68 4.900000000000000e+01 +20 69 5.000000000000000e+01 +20 70 5.100000000000000e+01 +20 71 5.200000000000000e+01 +20 72 5.300000000000000e+01 +20 73 5.400000000000000e+01 +20 74 5.500000000000000e+01 +20 75 5.600000000000000e+01 +20 76 5.700000000000000e+01 +20 77 5.800000000000000e+01 +20 78 5.900000000000000e+01 +20 79 6.000000000000000e+01 +20 80 6.100000000000000e+01 +20 81 6.200000000000000e+01 +20 82 6.300000000000000e+01 +20 83 6.400000000000000e+01 +20 84 6.500000000000000e+01 +20 85 6.600000000000000e+01 +20 86 6.700000000000000e+01 +20 87 6.800000000000000e+01 +20 88 6.900000000000000e+01 +20 89 7.000000000000000e+01 +20 90 7.100000000000000e+01 +20 91 7.200000000000000e+01 +20 92 7.300000000000000e+01 +20 93 7.400000000000000e+01 +20 94 7.500000000000000e+01 +20 95 7.600000000000000e+01 +20 96 7.700000000000000e+01 +20 97 7.800000000000000e+01 +20 98 7.900000000000000e+01 +20 99 8.000000000000000e+01 +20 100 8.100000000000000e+01 +21 21 1.000000000000000e+00 +21 22 2.000000000000000e+00 +21 23 3.000000000000000e+00 +21 24 4.000000000000000e+00 +21 25 5.000000000000000e+00 +21 26 6.000000000000000e+00 +21 27 7.000000000000000e+00 +21 28 8.000000000000000e+00 +21 29 9.000000000000000e+00 +21 30 1.000000000000000e+01 +21 31 1.100000000000000e+01 +21 32 1.200000000000000e+01 +21 33 1.300000000000000e+01 +21 34 1.400000000000000e+01 +21 35 1.500000000000000e+01 +21 36 1.600000000000000e+01 +21 37 1.700000000000000e+01 +21 38 1.800000000000000e+01 +21 39 1.900000000000000e+01 +21 40 2.000000000000000e+01 +21 41 2.100000000000000e+01 +21 42 2.200000000000000e+01 +21 43 2.300000000000000e+01 +21 44 2.400000000000000e+01 +21 45 2.500000000000000e+01 +21 46 2.600000000000000e+01 +21 47 2.700000000000000e+01 +21 48 2.800000000000000e+01 +21 49 2.900000000000000e+01 +21 50 3.000000000000000e+01 +21 51 3.100000000000000e+01 +21 52 3.200000000000000e+01 +21 53 3.300000000000000e+01 +21 54 3.400000000000000e+01 +21 55 3.500000000000000e+01 +21 56 3.600000000000000e+01 +21 57 3.700000000000000e+01 +21 58 3.800000000000000e+01 +21 59 3.900000000000000e+01 +21 60 4.000000000000000e+01 +21 61 4.100000000000000e+01 +21 62 4.200000000000000e+01 +21 63 4.300000000000000e+01 +21 64 4.400000000000000e+01 +21 65 4.500000000000000e+01 +21 66 4.600000000000000e+01 +21 67 4.700000000000000e+01 +21 68 4.800000000000000e+01 +21 69 4.900000000000000e+01 +21 70 5.000000000000000e+01 +21 71 5.100000000000000e+01 +21 72 5.200000000000000e+01 +21 73 5.300000000000000e+01 +21 74 5.400000000000000e+01 +21 75 5.500000000000000e+01 +21 76 5.600000000000000e+01 +21 77 5.700000000000000e+01 +21 78 5.800000000000000e+01 +21 79 5.900000000000000e+01 +21 80 6.000000000000000e+01 +21 81 6.100000000000000e+01 +21 82 6.200000000000000e+01 +21 83 6.300000000000000e+01 +21 84 6.400000000000000e+01 +21 85 6.500000000000000e+01 +21 86 6.600000000000000e+01 +21 87 6.700000000000000e+01 +21 88 6.800000000000000e+01 +21 89 6.900000000000000e+01 +21 90 7.000000000000000e+01 +21 91 7.100000000000000e+01 +21 92 7.200000000000000e+01 +21 93 7.300000000000000e+01 +21 94 7.400000000000000e+01 +21 95 7.500000000000000e+01 +21 96 7.600000000000000e+01 +21 97 7.700000000000000e+01 +21 98 7.800000000000000e+01 +21 99 7.900000000000000e+01 +21 100 8.000000000000000e+01 +22 22 1.000000000000000e+00 +22 23 2.000000000000000e+00 +22 24 3.000000000000000e+00 +22 25 4.000000000000000e+00 +22 26 5.000000000000000e+00 +22 27 6.000000000000000e+00 +22 28 7.000000000000000e+00 +22 29 8.000000000000000e+00 +22 30 9.000000000000000e+00 +22 31 1.000000000000000e+01 +22 32 1.100000000000000e+01 +22 33 1.200000000000000e+01 +22 34 1.300000000000000e+01 +22 35 1.400000000000000e+01 +22 36 1.500000000000000e+01 +22 37 1.600000000000000e+01 +22 38 1.700000000000000e+01 +22 39 1.800000000000000e+01 +22 40 1.900000000000000e+01 +22 41 2.000000000000000e+01 +22 42 2.100000000000000e+01 +22 43 2.200000000000000e+01 +22 44 2.300000000000000e+01 +22 45 2.400000000000000e+01 +22 46 2.500000000000000e+01 +22 47 2.600000000000000e+01 +22 48 2.700000000000000e+01 +22 49 2.800000000000000e+01 +22 50 2.900000000000000e+01 +22 51 3.000000000000000e+01 +22 52 3.100000000000000e+01 +22 53 3.200000000000000e+01 +22 54 3.300000000000000e+01 +22 55 3.400000000000000e+01 +22 56 3.500000000000000e+01 +22 57 3.600000000000000e+01 +22 58 3.700000000000000e+01 +22 59 3.800000000000000e+01 +22 60 3.900000000000000e+01 +22 61 4.000000000000000e+01 +22 62 4.100000000000000e+01 +22 63 4.200000000000000e+01 +22 64 4.300000000000000e+01 +22 65 4.400000000000000e+01 +22 66 4.500000000000000e+01 +22 67 4.600000000000000e+01 +22 68 4.700000000000000e+01 +22 69 4.800000000000000e+01 +22 70 4.900000000000000e+01 +22 71 5.000000000000000e+01 +22 72 5.100000000000000e+01 +22 73 5.200000000000000e+01 +22 74 5.300000000000000e+01 +22 75 5.400000000000000e+01 +22 76 5.500000000000000e+01 +22 77 5.600000000000000e+01 +22 78 5.700000000000000e+01 +22 79 5.800000000000000e+01 +22 80 5.900000000000000e+01 +22 81 6.000000000000000e+01 +22 82 6.100000000000000e+01 +22 83 6.200000000000000e+01 +22 84 6.300000000000000e+01 +22 85 6.400000000000000e+01 +22 86 6.500000000000000e+01 +22 87 6.600000000000000e+01 +22 88 6.700000000000000e+01 +22 89 6.800000000000000e+01 +22 90 6.900000000000000e+01 +22 91 7.000000000000000e+01 +22 92 7.100000000000000e+01 +22 93 7.200000000000000e+01 +22 94 7.300000000000000e+01 +22 95 7.400000000000000e+01 +22 96 7.500000000000000e+01 +22 97 7.600000000000000e+01 +22 98 7.700000000000000e+01 +22 99 7.800000000000000e+01 +22 100 7.900000000000000e+01 +23 23 1.000000000000000e+00 +23 24 2.000000000000000e+00 +23 25 3.000000000000000e+00 +23 26 4.000000000000000e+00 +23 27 5.000000000000000e+00 +23 28 6.000000000000000e+00 +23 29 7.000000000000000e+00 +23 30 8.000000000000000e+00 +23 31 9.000000000000000e+00 +23 32 1.000000000000000e+01 +23 33 1.100000000000000e+01 +23 34 1.200000000000000e+01 +23 35 1.300000000000000e+01 +23 36 1.400000000000000e+01 +23 37 1.500000000000000e+01 +23 38 1.600000000000000e+01 +23 39 1.700000000000000e+01 +23 40 1.800000000000000e+01 +23 41 1.900000000000000e+01 +23 42 2.000000000000000e+01 +23 43 2.100000000000000e+01 +23 44 2.200000000000000e+01 +23 45 2.300000000000000e+01 +23 46 2.400000000000000e+01 +23 47 2.500000000000000e+01 +23 48 2.600000000000000e+01 +23 49 2.700000000000000e+01 +23 50 2.800000000000000e+01 +23 51 2.900000000000000e+01 +23 52 3.000000000000000e+01 +23 53 3.100000000000000e+01 +23 54 3.200000000000000e+01 +23 55 3.300000000000000e+01 +23 56 3.400000000000000e+01 +23 57 3.500000000000000e+01 +23 58 3.600000000000000e+01 +23 59 3.700000000000000e+01 +23 60 3.800000000000000e+01 +23 61 3.900000000000000e+01 +23 62 4.000000000000000e+01 +23 63 4.100000000000000e+01 +23 64 4.200000000000000e+01 +23 65 4.300000000000000e+01 +23 66 4.400000000000000e+01 +23 67 4.500000000000000e+01 +23 68 4.600000000000000e+01 +23 69 4.700000000000000e+01 +23 70 4.800000000000000e+01 +23 71 4.900000000000000e+01 +23 72 5.000000000000000e+01 +23 73 5.100000000000000e+01 +23 74 5.200000000000000e+01 +23 75 5.300000000000000e+01 +23 76 5.400000000000000e+01 +23 77 5.500000000000000e+01 +23 78 5.600000000000000e+01 +23 79 5.700000000000000e+01 +23 80 5.800000000000000e+01 +23 81 5.900000000000000e+01 +23 82 6.000000000000000e+01 +23 83 6.100000000000000e+01 +23 84 6.200000000000000e+01 +23 85 6.300000000000000e+01 +23 86 6.400000000000000e+01 +23 87 6.500000000000000e+01 +23 88 6.600000000000000e+01 +23 89 6.700000000000000e+01 +23 90 6.800000000000000e+01 +23 91 6.900000000000000e+01 +23 92 7.000000000000000e+01 +23 93 7.100000000000000e+01 +23 94 7.200000000000000e+01 +23 95 7.300000000000000e+01 +23 96 7.400000000000000e+01 +23 97 7.500000000000000e+01 +23 98 7.600000000000000e+01 +23 99 7.700000000000000e+01 +23 100 7.800000000000000e+01 +24 24 1.000000000000000e+00 +24 25 2.000000000000000e+00 +24 26 3.000000000000000e+00 +24 27 4.000000000000000e+00 +24 28 5.000000000000000e+00 +24 29 6.000000000000000e+00 +24 30 7.000000000000000e+00 +24 31 8.000000000000000e+00 +24 32 9.000000000000000e+00 +24 33 1.000000000000000e+01 +24 34 1.100000000000000e+01 +24 35 1.200000000000000e+01 +24 36 1.300000000000000e+01 +24 37 1.400000000000000e+01 +24 38 1.500000000000000e+01 +24 39 1.600000000000000e+01 +24 40 1.700000000000000e+01 +24 41 1.800000000000000e+01 +24 42 1.900000000000000e+01 +24 43 2.000000000000000e+01 +24 44 2.100000000000000e+01 +24 45 2.200000000000000e+01 +24 46 2.300000000000000e+01 +24 47 2.400000000000000e+01 +24 48 2.500000000000000e+01 +24 49 2.600000000000000e+01 +24 50 2.700000000000000e+01 +24 51 2.800000000000000e+01 +24 52 2.900000000000000e+01 +24 53 3.000000000000000e+01 +24 54 3.100000000000000e+01 +24 55 3.200000000000000e+01 +24 56 3.300000000000000e+01 +24 57 3.400000000000000e+01 +24 58 3.500000000000000e+01 +24 59 3.600000000000000e+01 +24 60 3.700000000000000e+01 +24 61 3.800000000000000e+01 +24 62 3.900000000000000e+01 +24 63 4.000000000000000e+01 +24 64 4.100000000000000e+01 +24 65 4.200000000000000e+01 +24 66 4.300000000000000e+01 +24 67 4.400000000000000e+01 +24 68 4.500000000000000e+01 +24 69 4.600000000000000e+01 +24 70 4.700000000000000e+01 +24 71 4.800000000000000e+01 +24 72 4.900000000000000e+01 +24 73 5.000000000000000e+01 +24 74 5.100000000000000e+01 +24 75 5.200000000000000e+01 +24 76 5.300000000000000e+01 +24 77 5.400000000000000e+01 +24 78 5.500000000000000e+01 +24 79 5.600000000000000e+01 +24 80 5.700000000000000e+01 +24 81 5.800000000000000e+01 +24 82 5.900000000000000e+01 +24 83 6.000000000000000e+01 +24 84 6.100000000000000e+01 +24 85 6.200000000000000e+01 +24 86 6.300000000000000e+01 +24 87 6.400000000000000e+01 +24 88 6.500000000000000e+01 +24 89 6.600000000000000e+01 +24 90 6.700000000000000e+01 +24 91 6.800000000000000e+01 +24 92 6.900000000000000e+01 +24 93 7.000000000000000e+01 +24 94 7.100000000000000e+01 +24 95 7.200000000000000e+01 +24 96 7.300000000000000e+01 +24 97 7.400000000000000e+01 +24 98 7.500000000000000e+01 +24 99 7.600000000000000e+01 +24 100 7.700000000000000e+01 +25 25 1.000000000000000e+00 +25 26 2.000000000000000e+00 +25 27 3.000000000000000e+00 +25 28 4.000000000000000e+00 +25 29 5.000000000000000e+00 +25 30 6.000000000000000e+00 +25 31 7.000000000000000e+00 +25 32 8.000000000000000e+00 +25 33 9.000000000000000e+00 +25 34 1.000000000000000e+01 +25 35 1.100000000000000e+01 +25 36 1.200000000000000e+01 +25 37 1.300000000000000e+01 +25 38 1.400000000000000e+01 +25 39 1.500000000000000e+01 +25 40 1.600000000000000e+01 +25 41 1.700000000000000e+01 +25 42 1.800000000000000e+01 +25 43 1.900000000000000e+01 +25 44 2.000000000000000e+01 +25 45 2.100000000000000e+01 +25 46 2.200000000000000e+01 +25 47 2.300000000000000e+01 +25 48 2.400000000000000e+01 +25 49 2.500000000000000e+01 +25 50 2.600000000000000e+01 +25 51 2.700000000000000e+01 +25 52 2.800000000000000e+01 +25 53 2.900000000000000e+01 +25 54 3.000000000000000e+01 +25 55 3.100000000000000e+01 +25 56 3.200000000000000e+01 +25 57 3.300000000000000e+01 +25 58 3.400000000000000e+01 +25 59 3.500000000000000e+01 +25 60 3.600000000000000e+01 +25 61 3.700000000000000e+01 +25 62 3.800000000000000e+01 +25 63 3.900000000000000e+01 +25 64 4.000000000000000e+01 +25 65 4.100000000000000e+01 +25 66 4.200000000000000e+01 +25 67 4.300000000000000e+01 +25 68 4.400000000000000e+01 +25 69 4.500000000000000e+01 +25 70 4.600000000000000e+01 +25 71 4.700000000000000e+01 +25 72 4.800000000000000e+01 +25 73 4.900000000000000e+01 +25 74 5.000000000000000e+01 +25 75 5.100000000000000e+01 +25 76 5.200000000000000e+01 +25 77 5.300000000000000e+01 +25 78 5.400000000000000e+01 +25 79 5.500000000000000e+01 +25 80 5.600000000000000e+01 +25 81 5.700000000000000e+01 +25 82 5.800000000000000e+01 +25 83 5.900000000000000e+01 +25 84 6.000000000000000e+01 +25 85 6.100000000000000e+01 +25 86 6.200000000000000e+01 +25 87 6.300000000000000e+01 +25 88 6.400000000000000e+01 +25 89 6.500000000000000e+01 +25 90 6.600000000000000e+01 +25 91 6.700000000000000e+01 +25 92 6.800000000000000e+01 +25 93 6.900000000000000e+01 +25 94 7.000000000000000e+01 +25 95 7.100000000000000e+01 +25 96 7.200000000000000e+01 +25 97 7.300000000000000e+01 +25 98 7.400000000000000e+01 +25 99 7.500000000000000e+01 +25 100 7.600000000000000e+01 +26 26 1.000000000000000e+00 +26 27 2.000000000000000e+00 +26 28 3.000000000000000e+00 +26 29 4.000000000000000e+00 +26 30 5.000000000000000e+00 +26 31 6.000000000000000e+00 +26 32 7.000000000000000e+00 +26 33 8.000000000000000e+00 +26 34 9.000000000000000e+00 +26 35 1.000000000000000e+01 +26 36 1.100000000000000e+01 +26 37 1.200000000000000e+01 +26 38 1.300000000000000e+01 +26 39 1.400000000000000e+01 +26 40 1.500000000000000e+01 +26 41 1.600000000000000e+01 +26 42 1.700000000000000e+01 +26 43 1.800000000000000e+01 +26 44 1.900000000000000e+01 +26 45 2.000000000000000e+01 +26 46 2.100000000000000e+01 +26 47 2.200000000000000e+01 +26 48 2.300000000000000e+01 +26 49 2.400000000000000e+01 +26 50 2.500000000000000e+01 +26 51 2.600000000000000e+01 +26 52 2.700000000000000e+01 +26 53 2.800000000000000e+01 +26 54 2.900000000000000e+01 +26 55 3.000000000000000e+01 +26 56 3.100000000000000e+01 +26 57 3.200000000000000e+01 +26 58 3.300000000000000e+01 +26 59 3.400000000000000e+01 +26 60 3.500000000000000e+01 +26 61 3.600000000000000e+01 +26 62 3.700000000000000e+01 +26 63 3.800000000000000e+01 +26 64 3.900000000000000e+01 +26 65 4.000000000000000e+01 +26 66 4.100000000000000e+01 +26 67 4.200000000000000e+01 +26 68 4.300000000000000e+01 +26 69 4.400000000000000e+01 +26 70 4.500000000000000e+01 +26 71 4.600000000000000e+01 +26 72 4.700000000000000e+01 +26 73 4.800000000000000e+01 +26 74 4.900000000000000e+01 +26 75 5.000000000000000e+01 +26 76 5.100000000000000e+01 +26 77 5.200000000000000e+01 +26 78 5.300000000000000e+01 +26 79 5.400000000000000e+01 +26 80 5.500000000000000e+01 +26 81 5.600000000000000e+01 +26 82 5.700000000000000e+01 +26 83 5.800000000000000e+01 +26 84 5.900000000000000e+01 +26 85 6.000000000000000e+01 +26 86 6.100000000000000e+01 +26 87 6.200000000000000e+01 +26 88 6.300000000000000e+01 +26 89 6.400000000000000e+01 +26 90 6.500000000000000e+01 +26 91 6.600000000000000e+01 +26 92 6.700000000000000e+01 +26 93 6.800000000000000e+01 +26 94 6.900000000000000e+01 +26 95 7.000000000000000e+01 +26 96 7.100000000000000e+01 +26 97 7.200000000000000e+01 +26 98 7.300000000000000e+01 +26 99 7.400000000000000e+01 +26 100 7.500000000000000e+01 +27 27 1.000000000000000e+00 +27 28 2.000000000000000e+00 +27 29 3.000000000000000e+00 +27 30 4.000000000000000e+00 +27 31 5.000000000000000e+00 +27 32 6.000000000000000e+00 +27 33 7.000000000000000e+00 +27 34 8.000000000000000e+00 +27 35 9.000000000000000e+00 +27 36 1.000000000000000e+01 +27 37 1.100000000000000e+01 +27 38 1.200000000000000e+01 +27 39 1.300000000000000e+01 +27 40 1.400000000000000e+01 +27 41 1.500000000000000e+01 +27 42 1.600000000000000e+01 +27 43 1.700000000000000e+01 +27 44 1.800000000000000e+01 +27 45 1.900000000000000e+01 +27 46 2.000000000000000e+01 +27 47 2.100000000000000e+01 +27 48 2.200000000000000e+01 +27 49 2.300000000000000e+01 +27 50 2.400000000000000e+01 +27 51 2.500000000000000e+01 +27 52 2.600000000000000e+01 +27 53 2.700000000000000e+01 +27 54 2.800000000000000e+01 +27 55 2.900000000000000e+01 +27 56 3.000000000000000e+01 +27 57 3.100000000000000e+01 +27 58 3.200000000000000e+01 +27 59 3.300000000000000e+01 +27 60 3.400000000000000e+01 +27 61 3.500000000000000e+01 +27 62 3.600000000000000e+01 +27 63 3.700000000000000e+01 +27 64 3.800000000000000e+01 +27 65 3.900000000000000e+01 +27 66 4.000000000000000e+01 +27 67 4.100000000000000e+01 +27 68 4.200000000000000e+01 +27 69 4.300000000000000e+01 +27 70 4.400000000000000e+01 +27 71 4.500000000000000e+01 +27 72 4.600000000000000e+01 +27 73 4.700000000000000e+01 +27 74 4.800000000000000e+01 +27 75 4.900000000000000e+01 +27 76 5.000000000000000e+01 +27 77 5.100000000000000e+01 +27 78 5.200000000000000e+01 +27 79 5.300000000000000e+01 +27 80 5.400000000000000e+01 +27 81 5.500000000000000e+01 +27 82 5.600000000000000e+01 +27 83 5.700000000000000e+01 +27 84 5.800000000000000e+01 +27 85 5.900000000000000e+01 +27 86 6.000000000000000e+01 +27 87 6.100000000000000e+01 +27 88 6.200000000000000e+01 +27 89 6.300000000000000e+01 +27 90 6.400000000000000e+01 +27 91 6.500000000000000e+01 +27 92 6.600000000000000e+01 +27 93 6.700000000000000e+01 +27 94 6.800000000000000e+01 +27 95 6.900000000000000e+01 +27 96 7.000000000000000e+01 +27 97 7.100000000000000e+01 +27 98 7.200000000000000e+01 +27 99 7.300000000000000e+01 +27 100 7.400000000000000e+01 +28 28 1.000000000000000e+00 +28 29 2.000000000000000e+00 +28 30 3.000000000000000e+00 +28 31 4.000000000000000e+00 +28 32 5.000000000000000e+00 +28 33 6.000000000000000e+00 +28 34 7.000000000000000e+00 +28 35 8.000000000000000e+00 +28 36 9.000000000000000e+00 +28 37 1.000000000000000e+01 +28 38 1.100000000000000e+01 +28 39 1.200000000000000e+01 +28 40 1.300000000000000e+01 +28 41 1.400000000000000e+01 +28 42 1.500000000000000e+01 +28 43 1.600000000000000e+01 +28 44 1.700000000000000e+01 +28 45 1.800000000000000e+01 +28 46 1.900000000000000e+01 +28 47 2.000000000000000e+01 +28 48 2.100000000000000e+01 +28 49 2.200000000000000e+01 +28 50 2.300000000000000e+01 +28 51 2.400000000000000e+01 +28 52 2.500000000000000e+01 +28 53 2.600000000000000e+01 +28 54 2.700000000000000e+01 +28 55 2.800000000000000e+01 +28 56 2.900000000000000e+01 +28 57 3.000000000000000e+01 +28 58 3.100000000000000e+01 +28 59 3.200000000000000e+01 +28 60 3.300000000000000e+01 +28 61 3.400000000000000e+01 +28 62 3.500000000000000e+01 +28 63 3.600000000000000e+01 +28 64 3.700000000000000e+01 +28 65 3.800000000000000e+01 +28 66 3.900000000000000e+01 +28 67 4.000000000000000e+01 +28 68 4.100000000000000e+01 +28 69 4.200000000000000e+01 +28 70 4.300000000000000e+01 +28 71 4.400000000000000e+01 +28 72 4.500000000000000e+01 +28 73 4.600000000000000e+01 +28 74 4.700000000000000e+01 +28 75 4.800000000000000e+01 +28 76 4.900000000000000e+01 +28 77 5.000000000000000e+01 +28 78 5.100000000000000e+01 +28 79 5.200000000000000e+01 +28 80 5.300000000000000e+01 +28 81 5.400000000000000e+01 +28 82 5.500000000000000e+01 +28 83 5.600000000000000e+01 +28 84 5.700000000000000e+01 +28 85 5.800000000000000e+01 +28 86 5.900000000000000e+01 +28 87 6.000000000000000e+01 +28 88 6.100000000000000e+01 +28 89 6.200000000000000e+01 +28 90 6.300000000000000e+01 +28 91 6.400000000000000e+01 +28 92 6.500000000000000e+01 +28 93 6.600000000000000e+01 +28 94 6.700000000000000e+01 +28 95 6.800000000000000e+01 +28 96 6.900000000000000e+01 +28 97 7.000000000000000e+01 +28 98 7.100000000000000e+01 +28 99 7.200000000000000e+01 +28 100 7.300000000000000e+01 +29 29 1.000000000000000e+00 +29 30 2.000000000000000e+00 +29 31 3.000000000000000e+00 +29 32 4.000000000000000e+00 +29 33 5.000000000000000e+00 +29 34 6.000000000000000e+00 +29 35 7.000000000000000e+00 +29 36 8.000000000000000e+00 +29 37 9.000000000000000e+00 +29 38 1.000000000000000e+01 +29 39 1.100000000000000e+01 +29 40 1.200000000000000e+01 +29 41 1.300000000000000e+01 +29 42 1.400000000000000e+01 +29 43 1.500000000000000e+01 +29 44 1.600000000000000e+01 +29 45 1.700000000000000e+01 +29 46 1.800000000000000e+01 +29 47 1.900000000000000e+01 +29 48 2.000000000000000e+01 +29 49 2.100000000000000e+01 +29 50 2.200000000000000e+01 +29 51 2.300000000000000e+01 +29 52 2.400000000000000e+01 +29 53 2.500000000000000e+01 +29 54 2.600000000000000e+01 +29 55 2.700000000000000e+01 +29 56 2.800000000000000e+01 +29 57 2.900000000000000e+01 +29 58 3.000000000000000e+01 +29 59 3.100000000000000e+01 +29 60 3.200000000000000e+01 +29 61 3.300000000000000e+01 +29 62 3.400000000000000e+01 +29 63 3.500000000000000e+01 +29 64 3.600000000000000e+01 +29 65 3.700000000000000e+01 +29 66 3.800000000000000e+01 +29 67 3.900000000000000e+01 +29 68 4.000000000000000e+01 +29 69 4.100000000000000e+01 +29 70 4.200000000000000e+01 +29 71 4.300000000000000e+01 +29 72 4.400000000000000e+01 +29 73 4.500000000000000e+01 +29 74 4.600000000000000e+01 +29 75 4.700000000000000e+01 +29 76 4.800000000000000e+01 +29 77 4.900000000000000e+01 +29 78 5.000000000000000e+01 +29 79 5.100000000000000e+01 +29 80 5.200000000000000e+01 +29 81 5.300000000000000e+01 +29 82 5.400000000000000e+01 +29 83 5.500000000000000e+01 +29 84 5.600000000000000e+01 +29 85 5.700000000000000e+01 +29 86 5.800000000000000e+01 +29 87 5.900000000000000e+01 +29 88 6.000000000000000e+01 +29 89 6.100000000000000e+01 +29 90 6.200000000000000e+01 +29 91 6.300000000000000e+01 +29 92 6.400000000000000e+01 +29 93 6.500000000000000e+01 +29 94 6.600000000000000e+01 +29 95 6.700000000000000e+01 +29 96 6.800000000000000e+01 +29 97 6.900000000000000e+01 +29 98 7.000000000000000e+01 +29 99 7.100000000000000e+01 +29 100 7.200000000000000e+01 +30 30 1.000000000000000e+00 +30 31 2.000000000000000e+00 +30 32 3.000000000000000e+00 +30 33 4.000000000000000e+00 +30 34 5.000000000000000e+00 +30 35 6.000000000000000e+00 +30 36 7.000000000000000e+00 +30 37 8.000000000000000e+00 +30 38 9.000000000000000e+00 +30 39 1.000000000000000e+01 +30 40 1.100000000000000e+01 +30 41 1.200000000000000e+01 +30 42 1.300000000000000e+01 +30 43 1.400000000000000e+01 +30 44 1.500000000000000e+01 +30 45 1.600000000000000e+01 +30 46 1.700000000000000e+01 +30 47 1.800000000000000e+01 +30 48 1.900000000000000e+01 +30 49 2.000000000000000e+01 +30 50 2.100000000000000e+01 +30 51 2.200000000000000e+01 +30 52 2.300000000000000e+01 +30 53 2.400000000000000e+01 +30 54 2.500000000000000e+01 +30 55 2.600000000000000e+01 +30 56 2.700000000000000e+01 +30 57 2.800000000000000e+01 +30 58 2.900000000000000e+01 +30 59 3.000000000000000e+01 +30 60 3.100000000000000e+01 +30 61 3.200000000000000e+01 +30 62 3.300000000000000e+01 +30 63 3.400000000000000e+01 +30 64 3.500000000000000e+01 +30 65 3.600000000000000e+01 +30 66 3.700000000000000e+01 +30 67 3.800000000000000e+01 +30 68 3.900000000000000e+01 +30 69 4.000000000000000e+01 +30 70 4.100000000000000e+01 +30 71 4.200000000000000e+01 +30 72 4.300000000000000e+01 +30 73 4.400000000000000e+01 +30 74 4.500000000000000e+01 +30 75 4.600000000000000e+01 +30 76 4.700000000000000e+01 +30 77 4.800000000000000e+01 +30 78 4.900000000000000e+01 +30 79 5.000000000000000e+01 +30 80 5.100000000000000e+01 +30 81 5.200000000000000e+01 +30 82 5.300000000000000e+01 +30 83 5.400000000000000e+01 +30 84 5.500000000000000e+01 +30 85 5.600000000000000e+01 +30 86 5.700000000000000e+01 +30 87 5.800000000000000e+01 +30 88 5.900000000000000e+01 +30 89 6.000000000000000e+01 +30 90 6.100000000000000e+01 +30 91 6.200000000000000e+01 +30 92 6.300000000000000e+01 +30 93 6.400000000000000e+01 +30 94 6.500000000000000e+01 +30 95 6.600000000000000e+01 +30 96 6.700000000000000e+01 +30 97 6.800000000000000e+01 +30 98 6.900000000000000e+01 +30 99 7.000000000000000e+01 +30 100 7.100000000000000e+01 +31 31 1.000000000000000e+00 +31 32 2.000000000000000e+00 +31 33 3.000000000000000e+00 +31 34 4.000000000000000e+00 +31 35 5.000000000000000e+00 +31 36 6.000000000000000e+00 +31 37 7.000000000000000e+00 +31 38 8.000000000000000e+00 +31 39 9.000000000000000e+00 +31 40 1.000000000000000e+01 +31 41 1.100000000000000e+01 +31 42 1.200000000000000e+01 +31 43 1.300000000000000e+01 +31 44 1.400000000000000e+01 +31 45 1.500000000000000e+01 +31 46 1.600000000000000e+01 +31 47 1.700000000000000e+01 +31 48 1.800000000000000e+01 +31 49 1.900000000000000e+01 +31 50 2.000000000000000e+01 +31 51 2.100000000000000e+01 +31 52 2.200000000000000e+01 +31 53 2.300000000000000e+01 +31 54 2.400000000000000e+01 +31 55 2.500000000000000e+01 +31 56 2.600000000000000e+01 +31 57 2.700000000000000e+01 +31 58 2.800000000000000e+01 +31 59 2.900000000000000e+01 +31 60 3.000000000000000e+01 +31 61 3.100000000000000e+01 +31 62 3.200000000000000e+01 +31 63 3.300000000000000e+01 +31 64 3.400000000000000e+01 +31 65 3.500000000000000e+01 +31 66 3.600000000000000e+01 +31 67 3.700000000000000e+01 +31 68 3.800000000000000e+01 +31 69 3.900000000000000e+01 +31 70 4.000000000000000e+01 +31 71 4.100000000000000e+01 +31 72 4.200000000000000e+01 +31 73 4.300000000000000e+01 +31 74 4.400000000000000e+01 +31 75 4.500000000000000e+01 +31 76 4.600000000000000e+01 +31 77 4.700000000000000e+01 +31 78 4.800000000000000e+01 +31 79 4.900000000000000e+01 +31 80 5.000000000000000e+01 +31 81 5.100000000000000e+01 +31 82 5.200000000000000e+01 +31 83 5.300000000000000e+01 +31 84 5.400000000000000e+01 +31 85 5.500000000000000e+01 +31 86 5.600000000000000e+01 +31 87 5.700000000000000e+01 +31 88 5.800000000000000e+01 +31 89 5.900000000000000e+01 +31 90 6.000000000000000e+01 +31 91 6.100000000000000e+01 +31 92 6.200000000000000e+01 +31 93 6.300000000000000e+01 +31 94 6.400000000000000e+01 +31 95 6.500000000000000e+01 +31 96 6.600000000000000e+01 +31 97 6.700000000000000e+01 +31 98 6.800000000000000e+01 +31 99 6.900000000000000e+01 +31 100 7.000000000000000e+01 +32 32 1.000000000000000e+00 +32 33 2.000000000000000e+00 +32 34 3.000000000000000e+00 +32 35 4.000000000000000e+00 +32 36 5.000000000000000e+00 +32 37 6.000000000000000e+00 +32 38 7.000000000000000e+00 +32 39 8.000000000000000e+00 +32 40 9.000000000000000e+00 +32 41 1.000000000000000e+01 +32 42 1.100000000000000e+01 +32 43 1.200000000000000e+01 +32 44 1.300000000000000e+01 +32 45 1.400000000000000e+01 +32 46 1.500000000000000e+01 +32 47 1.600000000000000e+01 +32 48 1.700000000000000e+01 +32 49 1.800000000000000e+01 +32 50 1.900000000000000e+01 +32 51 2.000000000000000e+01 +32 52 2.100000000000000e+01 +32 53 2.200000000000000e+01 +32 54 2.300000000000000e+01 +32 55 2.400000000000000e+01 +32 56 2.500000000000000e+01 +32 57 2.600000000000000e+01 +32 58 2.700000000000000e+01 +32 59 2.800000000000000e+01 +32 60 2.900000000000000e+01 +32 61 3.000000000000000e+01 +32 62 3.100000000000000e+01 +32 63 3.200000000000000e+01 +32 64 3.300000000000000e+01 +32 65 3.400000000000000e+01 +32 66 3.500000000000000e+01 +32 67 3.600000000000000e+01 +32 68 3.700000000000000e+01 +32 69 3.800000000000000e+01 +32 70 3.900000000000000e+01 +32 71 4.000000000000000e+01 +32 72 4.100000000000000e+01 +32 73 4.200000000000000e+01 +32 74 4.300000000000000e+01 +32 75 4.400000000000000e+01 +32 76 4.500000000000000e+01 +32 77 4.600000000000000e+01 +32 78 4.700000000000000e+01 +32 79 4.800000000000000e+01 +32 80 4.900000000000000e+01 +32 81 5.000000000000000e+01 +32 82 5.100000000000000e+01 +32 83 5.200000000000000e+01 +32 84 5.300000000000000e+01 +32 85 5.400000000000000e+01 +32 86 5.500000000000000e+01 +32 87 5.600000000000000e+01 +32 88 5.700000000000000e+01 +32 89 5.800000000000000e+01 +32 90 5.900000000000000e+01 +32 91 6.000000000000000e+01 +32 92 6.100000000000000e+01 +32 93 6.200000000000000e+01 +32 94 6.300000000000000e+01 +32 95 6.400000000000000e+01 +32 96 6.500000000000000e+01 +32 97 6.600000000000000e+01 +32 98 6.700000000000000e+01 +32 99 6.800000000000000e+01 +32 100 6.900000000000000e+01 +33 33 1.000000000000000e+00 +33 34 2.000000000000000e+00 +33 35 3.000000000000000e+00 +33 36 4.000000000000000e+00 +33 37 5.000000000000000e+00 +33 38 6.000000000000000e+00 +33 39 7.000000000000000e+00 +33 40 8.000000000000000e+00 +33 41 9.000000000000000e+00 +33 42 1.000000000000000e+01 +33 43 1.100000000000000e+01 +33 44 1.200000000000000e+01 +33 45 1.300000000000000e+01 +33 46 1.400000000000000e+01 +33 47 1.500000000000000e+01 +33 48 1.600000000000000e+01 +33 49 1.700000000000000e+01 +33 50 1.800000000000000e+01 +33 51 1.900000000000000e+01 +33 52 2.000000000000000e+01 +33 53 2.100000000000000e+01 +33 54 2.200000000000000e+01 +33 55 2.300000000000000e+01 +33 56 2.400000000000000e+01 +33 57 2.500000000000000e+01 +33 58 2.600000000000000e+01 +33 59 2.700000000000000e+01 +33 60 2.800000000000000e+01 +33 61 2.900000000000000e+01 +33 62 3.000000000000000e+01 +33 63 3.100000000000000e+01 +33 64 3.200000000000000e+01 +33 65 3.300000000000000e+01 +33 66 3.400000000000000e+01 +33 67 3.500000000000000e+01 +33 68 3.600000000000000e+01 +33 69 3.700000000000000e+01 +33 70 3.800000000000000e+01 +33 71 3.900000000000000e+01 +33 72 4.000000000000000e+01 +33 73 4.100000000000000e+01 +33 74 4.200000000000000e+01 +33 75 4.300000000000000e+01 +33 76 4.400000000000000e+01 +33 77 4.500000000000000e+01 +33 78 4.600000000000000e+01 +33 79 4.700000000000000e+01 +33 80 4.800000000000000e+01 +33 81 4.900000000000000e+01 +33 82 5.000000000000000e+01 +33 83 5.100000000000000e+01 +33 84 5.200000000000000e+01 +33 85 5.300000000000000e+01 +33 86 5.400000000000000e+01 +33 87 5.500000000000000e+01 +33 88 5.600000000000000e+01 +33 89 5.700000000000000e+01 +33 90 5.800000000000000e+01 +33 91 5.900000000000000e+01 +33 92 6.000000000000000e+01 +33 93 6.100000000000000e+01 +33 94 6.200000000000000e+01 +33 95 6.300000000000000e+01 +33 96 6.400000000000000e+01 +33 97 6.500000000000000e+01 +33 98 6.600000000000000e+01 +33 99 6.700000000000000e+01 +33 100 6.800000000000000e+01 +34 34 1.000000000000000e+00 +34 35 2.000000000000000e+00 +34 36 3.000000000000000e+00 +34 37 4.000000000000000e+00 +34 38 5.000000000000000e+00 +34 39 6.000000000000000e+00 +34 40 7.000000000000000e+00 +34 41 8.000000000000000e+00 +34 42 9.000000000000000e+00 +34 43 1.000000000000000e+01 +34 44 1.100000000000000e+01 +34 45 1.200000000000000e+01 +34 46 1.300000000000000e+01 +34 47 1.400000000000000e+01 +34 48 1.500000000000000e+01 +34 49 1.600000000000000e+01 +34 50 1.700000000000000e+01 +34 51 1.800000000000000e+01 +34 52 1.900000000000000e+01 +34 53 2.000000000000000e+01 +34 54 2.100000000000000e+01 +34 55 2.200000000000000e+01 +34 56 2.300000000000000e+01 +34 57 2.400000000000000e+01 +34 58 2.500000000000000e+01 +34 59 2.600000000000000e+01 +34 60 2.700000000000000e+01 +34 61 2.800000000000000e+01 +34 62 2.900000000000000e+01 +34 63 3.000000000000000e+01 +34 64 3.100000000000000e+01 +34 65 3.200000000000000e+01 +34 66 3.300000000000000e+01 +34 67 3.400000000000000e+01 +34 68 3.500000000000000e+01 +34 69 3.600000000000000e+01 +34 70 3.700000000000000e+01 +34 71 3.800000000000000e+01 +34 72 3.900000000000000e+01 +34 73 4.000000000000000e+01 +34 74 4.100000000000000e+01 +34 75 4.200000000000000e+01 +34 76 4.300000000000000e+01 +34 77 4.400000000000000e+01 +34 78 4.500000000000000e+01 +34 79 4.600000000000000e+01 +34 80 4.700000000000000e+01 +34 81 4.800000000000000e+01 +34 82 4.900000000000000e+01 +34 83 5.000000000000000e+01 +34 84 5.100000000000000e+01 +34 85 5.200000000000000e+01 +34 86 5.300000000000000e+01 +34 87 5.400000000000000e+01 +34 88 5.500000000000000e+01 +34 89 5.600000000000000e+01 +34 90 5.700000000000000e+01 +34 91 5.800000000000000e+01 +34 92 5.900000000000000e+01 +34 93 6.000000000000000e+01 +34 94 6.100000000000000e+01 +34 95 6.200000000000000e+01 +34 96 6.300000000000000e+01 +34 97 6.400000000000000e+01 +34 98 6.500000000000000e+01 +34 99 6.600000000000000e+01 +34 100 6.700000000000000e+01 +35 35 1.000000000000000e+00 +35 36 2.000000000000000e+00 +35 37 3.000000000000000e+00 +35 38 4.000000000000000e+00 +35 39 5.000000000000000e+00 +35 40 6.000000000000000e+00 +35 41 7.000000000000000e+00 +35 42 8.000000000000000e+00 +35 43 9.000000000000000e+00 +35 44 1.000000000000000e+01 +35 45 1.100000000000000e+01 +35 46 1.200000000000000e+01 +35 47 1.300000000000000e+01 +35 48 1.400000000000000e+01 +35 49 1.500000000000000e+01 +35 50 1.600000000000000e+01 +35 51 1.700000000000000e+01 +35 52 1.800000000000000e+01 +35 53 1.900000000000000e+01 +35 54 2.000000000000000e+01 +35 55 2.100000000000000e+01 +35 56 2.200000000000000e+01 +35 57 2.300000000000000e+01 +35 58 2.400000000000000e+01 +35 59 2.500000000000000e+01 +35 60 2.600000000000000e+01 +35 61 2.700000000000000e+01 +35 62 2.800000000000000e+01 +35 63 2.900000000000000e+01 +35 64 3.000000000000000e+01 +35 65 3.100000000000000e+01 +35 66 3.200000000000000e+01 +35 67 3.300000000000000e+01 +35 68 3.400000000000000e+01 +35 69 3.500000000000000e+01 +35 70 3.600000000000000e+01 +35 71 3.700000000000000e+01 +35 72 3.800000000000000e+01 +35 73 3.900000000000000e+01 +35 74 4.000000000000000e+01 +35 75 4.100000000000000e+01 +35 76 4.200000000000000e+01 +35 77 4.300000000000000e+01 +35 78 4.400000000000000e+01 +35 79 4.500000000000000e+01 +35 80 4.600000000000000e+01 +35 81 4.700000000000000e+01 +35 82 4.800000000000000e+01 +35 83 4.900000000000000e+01 +35 84 5.000000000000000e+01 +35 85 5.100000000000000e+01 +35 86 5.200000000000000e+01 +35 87 5.300000000000000e+01 +35 88 5.400000000000000e+01 +35 89 5.500000000000000e+01 +35 90 5.600000000000000e+01 +35 91 5.700000000000000e+01 +35 92 5.800000000000000e+01 +35 93 5.900000000000000e+01 +35 94 6.000000000000000e+01 +35 95 6.100000000000000e+01 +35 96 6.200000000000000e+01 +35 97 6.300000000000000e+01 +35 98 6.400000000000000e+01 +35 99 6.500000000000000e+01 +35 100 6.600000000000000e+01 +36 36 1.000000000000000e+00 +36 37 2.000000000000000e+00 +36 38 3.000000000000000e+00 +36 39 4.000000000000000e+00 +36 40 5.000000000000000e+00 +36 41 6.000000000000000e+00 +36 42 7.000000000000000e+00 +36 43 8.000000000000000e+00 +36 44 9.000000000000000e+00 +36 45 1.000000000000000e+01 +36 46 1.100000000000000e+01 +36 47 1.200000000000000e+01 +36 48 1.300000000000000e+01 +36 49 1.400000000000000e+01 +36 50 1.500000000000000e+01 +36 51 1.600000000000000e+01 +36 52 1.700000000000000e+01 +36 53 1.800000000000000e+01 +36 54 1.900000000000000e+01 +36 55 2.000000000000000e+01 +36 56 2.100000000000000e+01 +36 57 2.200000000000000e+01 +36 58 2.300000000000000e+01 +36 59 2.400000000000000e+01 +36 60 2.500000000000000e+01 +36 61 2.600000000000000e+01 +36 62 2.700000000000000e+01 +36 63 2.800000000000000e+01 +36 64 2.900000000000000e+01 +36 65 3.000000000000000e+01 +36 66 3.100000000000000e+01 +36 67 3.200000000000000e+01 +36 68 3.300000000000000e+01 +36 69 3.400000000000000e+01 +36 70 3.500000000000000e+01 +36 71 3.600000000000000e+01 +36 72 3.700000000000000e+01 +36 73 3.800000000000000e+01 +36 74 3.900000000000000e+01 +36 75 4.000000000000000e+01 +36 76 4.100000000000000e+01 +36 77 4.200000000000000e+01 +36 78 4.300000000000000e+01 +36 79 4.400000000000000e+01 +36 80 4.500000000000000e+01 +36 81 4.600000000000000e+01 +36 82 4.700000000000000e+01 +36 83 4.800000000000000e+01 +36 84 4.900000000000000e+01 +36 85 5.000000000000000e+01 +36 86 5.100000000000000e+01 +36 87 5.200000000000000e+01 +36 88 5.300000000000000e+01 +36 89 5.400000000000000e+01 +36 90 5.500000000000000e+01 +36 91 5.600000000000000e+01 +36 92 5.700000000000000e+01 +36 93 5.800000000000000e+01 +36 94 5.900000000000000e+01 +36 95 6.000000000000000e+01 +36 96 6.100000000000000e+01 +36 97 6.200000000000000e+01 +36 98 6.300000000000000e+01 +36 99 6.400000000000000e+01 +36 100 6.500000000000000e+01 +37 37 1.000000000000000e+00 +37 38 2.000000000000000e+00 +37 39 3.000000000000000e+00 +37 40 4.000000000000000e+00 +37 41 5.000000000000000e+00 +37 42 6.000000000000000e+00 +37 43 7.000000000000000e+00 +37 44 8.000000000000000e+00 +37 45 9.000000000000000e+00 +37 46 1.000000000000000e+01 +37 47 1.100000000000000e+01 +37 48 1.200000000000000e+01 +37 49 1.300000000000000e+01 +37 50 1.400000000000000e+01 +37 51 1.500000000000000e+01 +37 52 1.600000000000000e+01 +37 53 1.700000000000000e+01 +37 54 1.800000000000000e+01 +37 55 1.900000000000000e+01 +37 56 2.000000000000000e+01 +37 57 2.100000000000000e+01 +37 58 2.200000000000000e+01 +37 59 2.300000000000000e+01 +37 60 2.400000000000000e+01 +37 61 2.500000000000000e+01 +37 62 2.600000000000000e+01 +37 63 2.700000000000000e+01 +37 64 2.800000000000000e+01 +37 65 2.900000000000000e+01 +37 66 3.000000000000000e+01 +37 67 3.100000000000000e+01 +37 68 3.200000000000000e+01 +37 69 3.300000000000000e+01 +37 70 3.400000000000000e+01 +37 71 3.500000000000000e+01 +37 72 3.600000000000000e+01 +37 73 3.700000000000000e+01 +37 74 3.800000000000000e+01 +37 75 3.900000000000000e+01 +37 76 4.000000000000000e+01 +37 77 4.100000000000000e+01 +37 78 4.200000000000000e+01 +37 79 4.300000000000000e+01 +37 80 4.400000000000000e+01 +37 81 4.500000000000000e+01 +37 82 4.600000000000000e+01 +37 83 4.700000000000000e+01 +37 84 4.800000000000000e+01 +37 85 4.900000000000000e+01 +37 86 5.000000000000000e+01 +37 87 5.100000000000000e+01 +37 88 5.200000000000000e+01 +37 89 5.300000000000000e+01 +37 90 5.400000000000000e+01 +37 91 5.500000000000000e+01 +37 92 5.600000000000000e+01 +37 93 5.700000000000000e+01 +37 94 5.800000000000000e+01 +37 95 5.900000000000000e+01 +37 96 6.000000000000000e+01 +37 97 6.100000000000000e+01 +37 98 6.200000000000000e+01 +37 99 6.300000000000000e+01 +37 100 6.400000000000000e+01 +38 38 1.000000000000000e+00 +38 39 2.000000000000000e+00 +38 40 3.000000000000000e+00 +38 41 4.000000000000000e+00 +38 42 5.000000000000000e+00 +38 43 6.000000000000000e+00 +38 44 7.000000000000000e+00 +38 45 8.000000000000000e+00 +38 46 9.000000000000000e+00 +38 47 1.000000000000000e+01 +38 48 1.100000000000000e+01 +38 49 1.200000000000000e+01 +38 50 1.300000000000000e+01 +38 51 1.400000000000000e+01 +38 52 1.500000000000000e+01 +38 53 1.600000000000000e+01 +38 54 1.700000000000000e+01 +38 55 1.800000000000000e+01 +38 56 1.900000000000000e+01 +38 57 2.000000000000000e+01 +38 58 2.100000000000000e+01 +38 59 2.200000000000000e+01 +38 60 2.300000000000000e+01 +38 61 2.400000000000000e+01 +38 62 2.500000000000000e+01 +38 63 2.600000000000000e+01 +38 64 2.700000000000000e+01 +38 65 2.800000000000000e+01 +38 66 2.900000000000000e+01 +38 67 3.000000000000000e+01 +38 68 3.100000000000000e+01 +38 69 3.200000000000000e+01 +38 70 3.300000000000000e+01 +38 71 3.400000000000000e+01 +38 72 3.500000000000000e+01 +38 73 3.600000000000000e+01 +38 74 3.700000000000000e+01 +38 75 3.800000000000000e+01 +38 76 3.900000000000000e+01 +38 77 4.000000000000000e+01 +38 78 4.100000000000000e+01 +38 79 4.200000000000000e+01 +38 80 4.300000000000000e+01 +38 81 4.400000000000000e+01 +38 82 4.500000000000000e+01 +38 83 4.600000000000000e+01 +38 84 4.700000000000000e+01 +38 85 4.800000000000000e+01 +38 86 4.900000000000000e+01 +38 87 5.000000000000000e+01 +38 88 5.100000000000000e+01 +38 89 5.200000000000000e+01 +38 90 5.300000000000000e+01 +38 91 5.400000000000000e+01 +38 92 5.500000000000000e+01 +38 93 5.600000000000000e+01 +38 94 5.700000000000000e+01 +38 95 5.800000000000000e+01 +38 96 5.900000000000000e+01 +38 97 6.000000000000000e+01 +38 98 6.100000000000000e+01 +38 99 6.200000000000000e+01 +38 100 6.300000000000000e+01 +39 39 1.000000000000000e+00 +39 40 2.000000000000000e+00 +39 41 3.000000000000000e+00 +39 42 4.000000000000000e+00 +39 43 5.000000000000000e+00 +39 44 6.000000000000000e+00 +39 45 7.000000000000000e+00 +39 46 8.000000000000000e+00 +39 47 9.000000000000000e+00 +39 48 1.000000000000000e+01 +39 49 1.100000000000000e+01 +39 50 1.200000000000000e+01 +39 51 1.300000000000000e+01 +39 52 1.400000000000000e+01 +39 53 1.500000000000000e+01 +39 54 1.600000000000000e+01 +39 55 1.700000000000000e+01 +39 56 1.800000000000000e+01 +39 57 1.900000000000000e+01 +39 58 2.000000000000000e+01 +39 59 2.100000000000000e+01 +39 60 2.200000000000000e+01 +39 61 2.300000000000000e+01 +39 62 2.400000000000000e+01 +39 63 2.500000000000000e+01 +39 64 2.600000000000000e+01 +39 65 2.700000000000000e+01 +39 66 2.800000000000000e+01 +39 67 2.900000000000000e+01 +39 68 3.000000000000000e+01 +39 69 3.100000000000000e+01 +39 70 3.200000000000000e+01 +39 71 3.300000000000000e+01 +39 72 3.400000000000000e+01 +39 73 3.500000000000000e+01 +39 74 3.600000000000000e+01 +39 75 3.700000000000000e+01 +39 76 3.800000000000000e+01 +39 77 3.900000000000000e+01 +39 78 4.000000000000000e+01 +39 79 4.100000000000000e+01 +39 80 4.200000000000000e+01 +39 81 4.300000000000000e+01 +39 82 4.400000000000000e+01 +39 83 4.500000000000000e+01 +39 84 4.600000000000000e+01 +39 85 4.700000000000000e+01 +39 86 4.800000000000000e+01 +39 87 4.900000000000000e+01 +39 88 5.000000000000000e+01 +39 89 5.100000000000000e+01 +39 90 5.200000000000000e+01 +39 91 5.300000000000000e+01 +39 92 5.400000000000000e+01 +39 93 5.500000000000000e+01 +39 94 5.600000000000000e+01 +39 95 5.700000000000000e+01 +39 96 5.800000000000000e+01 +39 97 5.900000000000000e+01 +39 98 6.000000000000000e+01 +39 99 6.100000000000000e+01 +39 100 6.200000000000000e+01 +40 40 1.000000000000000e+00 +40 41 2.000000000000000e+00 +40 42 3.000000000000000e+00 +40 43 4.000000000000000e+00 +40 44 5.000000000000000e+00 +40 45 6.000000000000000e+00 +40 46 7.000000000000000e+00 +40 47 8.000000000000000e+00 +40 48 9.000000000000000e+00 +40 49 1.000000000000000e+01 +40 50 1.100000000000000e+01 +40 51 1.200000000000000e+01 +40 52 1.300000000000000e+01 +40 53 1.400000000000000e+01 +40 54 1.500000000000000e+01 +40 55 1.600000000000000e+01 +40 56 1.700000000000000e+01 +40 57 1.800000000000000e+01 +40 58 1.900000000000000e+01 +40 59 2.000000000000000e+01 +40 60 2.100000000000000e+01 +40 61 2.200000000000000e+01 +40 62 2.300000000000000e+01 +40 63 2.400000000000000e+01 +40 64 2.500000000000000e+01 +40 65 2.600000000000000e+01 +40 66 2.700000000000000e+01 +40 67 2.800000000000000e+01 +40 68 2.900000000000000e+01 +40 69 3.000000000000000e+01 +40 70 3.100000000000000e+01 +40 71 3.200000000000000e+01 +40 72 3.300000000000000e+01 +40 73 3.400000000000000e+01 +40 74 3.500000000000000e+01 +40 75 3.600000000000000e+01 +40 76 3.700000000000000e+01 +40 77 3.800000000000000e+01 +40 78 3.900000000000000e+01 +40 79 4.000000000000000e+01 +40 80 4.100000000000000e+01 +40 81 4.200000000000000e+01 +40 82 4.300000000000000e+01 +40 83 4.400000000000000e+01 +40 84 4.500000000000000e+01 +40 85 4.600000000000000e+01 +40 86 4.700000000000000e+01 +40 87 4.800000000000000e+01 +40 88 4.900000000000000e+01 +40 89 5.000000000000000e+01 +40 90 5.100000000000000e+01 +40 91 5.200000000000000e+01 +40 92 5.300000000000000e+01 +40 93 5.400000000000000e+01 +40 94 5.500000000000000e+01 +40 95 5.600000000000000e+01 +40 96 5.700000000000000e+01 +40 97 5.800000000000000e+01 +40 98 5.900000000000000e+01 +40 99 6.000000000000000e+01 +40 100 6.100000000000000e+01 +41 41 1.000000000000000e+00 +41 42 2.000000000000000e+00 +41 43 3.000000000000000e+00 +41 44 4.000000000000000e+00 +41 45 5.000000000000000e+00 +41 46 6.000000000000000e+00 +41 47 7.000000000000000e+00 +41 48 8.000000000000000e+00 +41 49 9.000000000000000e+00 +41 50 1.000000000000000e+01 +41 51 1.100000000000000e+01 +41 52 1.200000000000000e+01 +41 53 1.300000000000000e+01 +41 54 1.400000000000000e+01 +41 55 1.500000000000000e+01 +41 56 1.600000000000000e+01 +41 57 1.700000000000000e+01 +41 58 1.800000000000000e+01 +41 59 1.900000000000000e+01 +41 60 2.000000000000000e+01 +41 61 2.100000000000000e+01 +41 62 2.200000000000000e+01 +41 63 2.300000000000000e+01 +41 64 2.400000000000000e+01 +41 65 2.500000000000000e+01 +41 66 2.600000000000000e+01 +41 67 2.700000000000000e+01 +41 68 2.800000000000000e+01 +41 69 2.900000000000000e+01 +41 70 3.000000000000000e+01 +41 71 3.100000000000000e+01 +41 72 3.200000000000000e+01 +41 73 3.300000000000000e+01 +41 74 3.400000000000000e+01 +41 75 3.500000000000000e+01 +41 76 3.600000000000000e+01 +41 77 3.700000000000000e+01 +41 78 3.800000000000000e+01 +41 79 3.900000000000000e+01 +41 80 4.000000000000000e+01 +41 81 4.100000000000000e+01 +41 82 4.200000000000000e+01 +41 83 4.300000000000000e+01 +41 84 4.400000000000000e+01 +41 85 4.500000000000000e+01 +41 86 4.600000000000000e+01 +41 87 4.700000000000000e+01 +41 88 4.800000000000000e+01 +41 89 4.900000000000000e+01 +41 90 5.000000000000000e+01 +41 91 5.100000000000000e+01 +41 92 5.200000000000000e+01 +41 93 5.300000000000000e+01 +41 94 5.400000000000000e+01 +41 95 5.500000000000000e+01 +41 96 5.600000000000000e+01 +41 97 5.700000000000000e+01 +41 98 5.800000000000000e+01 +41 99 5.900000000000000e+01 +41 100 6.000000000000000e+01 +42 42 1.000000000000000e+00 +42 43 2.000000000000000e+00 +42 44 3.000000000000000e+00 +42 45 4.000000000000000e+00 +42 46 5.000000000000000e+00 +42 47 6.000000000000000e+00 +42 48 7.000000000000000e+00 +42 49 8.000000000000000e+00 +42 50 9.000000000000000e+00 +42 51 1.000000000000000e+01 +42 52 1.100000000000000e+01 +42 53 1.200000000000000e+01 +42 54 1.300000000000000e+01 +42 55 1.400000000000000e+01 +42 56 1.500000000000000e+01 +42 57 1.600000000000000e+01 +42 58 1.700000000000000e+01 +42 59 1.800000000000000e+01 +42 60 1.900000000000000e+01 +42 61 2.000000000000000e+01 +42 62 2.100000000000000e+01 +42 63 2.200000000000000e+01 +42 64 2.300000000000000e+01 +42 65 2.400000000000000e+01 +42 66 2.500000000000000e+01 +42 67 2.600000000000000e+01 +42 68 2.700000000000000e+01 +42 69 2.800000000000000e+01 +42 70 2.900000000000000e+01 +42 71 3.000000000000000e+01 +42 72 3.100000000000000e+01 +42 73 3.200000000000000e+01 +42 74 3.300000000000000e+01 +42 75 3.400000000000000e+01 +42 76 3.500000000000000e+01 +42 77 3.600000000000000e+01 +42 78 3.700000000000000e+01 +42 79 3.800000000000000e+01 +42 80 3.900000000000000e+01 +42 81 4.000000000000000e+01 +42 82 4.100000000000000e+01 +42 83 4.200000000000000e+01 +42 84 4.300000000000000e+01 +42 85 4.400000000000000e+01 +42 86 4.500000000000000e+01 +42 87 4.600000000000000e+01 +42 88 4.700000000000000e+01 +42 89 4.800000000000000e+01 +42 90 4.900000000000000e+01 +42 91 5.000000000000000e+01 +42 92 5.100000000000000e+01 +42 93 5.200000000000000e+01 +42 94 5.300000000000000e+01 +42 95 5.400000000000000e+01 +42 96 5.500000000000000e+01 +42 97 5.600000000000000e+01 +42 98 5.700000000000000e+01 +42 99 5.800000000000000e+01 +42 100 5.900000000000000e+01 +43 43 1.000000000000000e+00 +43 44 2.000000000000000e+00 +43 45 3.000000000000000e+00 +43 46 4.000000000000000e+00 +43 47 5.000000000000000e+00 +43 48 6.000000000000000e+00 +43 49 7.000000000000000e+00 +43 50 8.000000000000000e+00 +43 51 9.000000000000000e+00 +43 52 1.000000000000000e+01 +43 53 1.100000000000000e+01 +43 54 1.200000000000000e+01 +43 55 1.300000000000000e+01 +43 56 1.400000000000000e+01 +43 57 1.500000000000000e+01 +43 58 1.600000000000000e+01 +43 59 1.700000000000000e+01 +43 60 1.800000000000000e+01 +43 61 1.900000000000000e+01 +43 62 2.000000000000000e+01 +43 63 2.100000000000000e+01 +43 64 2.200000000000000e+01 +43 65 2.300000000000000e+01 +43 66 2.400000000000000e+01 +43 67 2.500000000000000e+01 +43 68 2.600000000000000e+01 +43 69 2.700000000000000e+01 +43 70 2.800000000000000e+01 +43 71 2.900000000000000e+01 +43 72 3.000000000000000e+01 +43 73 3.100000000000000e+01 +43 74 3.200000000000000e+01 +43 75 3.300000000000000e+01 +43 76 3.400000000000000e+01 +43 77 3.500000000000000e+01 +43 78 3.600000000000000e+01 +43 79 3.700000000000000e+01 +43 80 3.800000000000000e+01 +43 81 3.900000000000000e+01 +43 82 4.000000000000000e+01 +43 83 4.100000000000000e+01 +43 84 4.200000000000000e+01 +43 85 4.300000000000000e+01 +43 86 4.400000000000000e+01 +43 87 4.500000000000000e+01 +43 88 4.600000000000000e+01 +43 89 4.700000000000000e+01 +43 90 4.800000000000000e+01 +43 91 4.900000000000000e+01 +43 92 5.000000000000000e+01 +43 93 5.100000000000000e+01 +43 94 5.200000000000000e+01 +43 95 5.300000000000000e+01 +43 96 5.400000000000000e+01 +43 97 5.500000000000000e+01 +43 98 5.600000000000000e+01 +43 99 5.700000000000000e+01 +43 100 5.800000000000000e+01 +44 44 1.000000000000000e+00 +44 45 2.000000000000000e+00 +44 46 3.000000000000000e+00 +44 47 4.000000000000000e+00 +44 48 5.000000000000000e+00 +44 49 6.000000000000000e+00 +44 50 7.000000000000000e+00 +44 51 8.000000000000000e+00 +44 52 9.000000000000000e+00 +44 53 1.000000000000000e+01 +44 54 1.100000000000000e+01 +44 55 1.200000000000000e+01 +44 56 1.300000000000000e+01 +44 57 1.400000000000000e+01 +44 58 1.500000000000000e+01 +44 59 1.600000000000000e+01 +44 60 1.700000000000000e+01 +44 61 1.800000000000000e+01 +44 62 1.900000000000000e+01 +44 63 2.000000000000000e+01 +44 64 2.100000000000000e+01 +44 65 2.200000000000000e+01 +44 66 2.300000000000000e+01 +44 67 2.400000000000000e+01 +44 68 2.500000000000000e+01 +44 69 2.600000000000000e+01 +44 70 2.700000000000000e+01 +44 71 2.800000000000000e+01 +44 72 2.900000000000000e+01 +44 73 3.000000000000000e+01 +44 74 3.100000000000000e+01 +44 75 3.200000000000000e+01 +44 76 3.300000000000000e+01 +44 77 3.400000000000000e+01 +44 78 3.500000000000000e+01 +44 79 3.600000000000000e+01 +44 80 3.700000000000000e+01 +44 81 3.800000000000000e+01 +44 82 3.900000000000000e+01 +44 83 4.000000000000000e+01 +44 84 4.100000000000000e+01 +44 85 4.200000000000000e+01 +44 86 4.300000000000000e+01 +44 87 4.400000000000000e+01 +44 88 4.500000000000000e+01 +44 89 4.600000000000000e+01 +44 90 4.700000000000000e+01 +44 91 4.800000000000000e+01 +44 92 4.900000000000000e+01 +44 93 5.000000000000000e+01 +44 94 5.100000000000000e+01 +44 95 5.200000000000000e+01 +44 96 5.300000000000000e+01 +44 97 5.400000000000000e+01 +44 98 5.500000000000000e+01 +44 99 5.600000000000000e+01 +44 100 5.700000000000000e+01 +45 45 1.000000000000000e+00 +45 46 2.000000000000000e+00 +45 47 3.000000000000000e+00 +45 48 4.000000000000000e+00 +45 49 5.000000000000000e+00 +45 50 6.000000000000000e+00 +45 51 7.000000000000000e+00 +45 52 8.000000000000000e+00 +45 53 9.000000000000000e+00 +45 54 1.000000000000000e+01 +45 55 1.100000000000000e+01 +45 56 1.200000000000000e+01 +45 57 1.300000000000000e+01 +45 58 1.400000000000000e+01 +45 59 1.500000000000000e+01 +45 60 1.600000000000000e+01 +45 61 1.700000000000000e+01 +45 62 1.800000000000000e+01 +45 63 1.900000000000000e+01 +45 64 2.000000000000000e+01 +45 65 2.100000000000000e+01 +45 66 2.200000000000000e+01 +45 67 2.300000000000000e+01 +45 68 2.400000000000000e+01 +45 69 2.500000000000000e+01 +45 70 2.600000000000000e+01 +45 71 2.700000000000000e+01 +45 72 2.800000000000000e+01 +45 73 2.900000000000000e+01 +45 74 3.000000000000000e+01 +45 75 3.100000000000000e+01 +45 76 3.200000000000000e+01 +45 77 3.300000000000000e+01 +45 78 3.400000000000000e+01 +45 79 3.500000000000000e+01 +45 80 3.600000000000000e+01 +45 81 3.700000000000000e+01 +45 82 3.800000000000000e+01 +45 83 3.900000000000000e+01 +45 84 4.000000000000000e+01 +45 85 4.100000000000000e+01 +45 86 4.200000000000000e+01 +45 87 4.300000000000000e+01 +45 88 4.400000000000000e+01 +45 89 4.500000000000000e+01 +45 90 4.600000000000000e+01 +45 91 4.700000000000000e+01 +45 92 4.800000000000000e+01 +45 93 4.900000000000000e+01 +45 94 5.000000000000000e+01 +45 95 5.100000000000000e+01 +45 96 5.200000000000000e+01 +45 97 5.300000000000000e+01 +45 98 5.400000000000000e+01 +45 99 5.500000000000000e+01 +45 100 5.600000000000000e+01 +46 46 1.000000000000000e+00 +46 47 2.000000000000000e+00 +46 48 3.000000000000000e+00 +46 49 4.000000000000000e+00 +46 50 5.000000000000000e+00 +46 51 6.000000000000000e+00 +46 52 7.000000000000000e+00 +46 53 8.000000000000000e+00 +46 54 9.000000000000000e+00 +46 55 1.000000000000000e+01 +46 56 1.100000000000000e+01 +46 57 1.200000000000000e+01 +46 58 1.300000000000000e+01 +46 59 1.400000000000000e+01 +46 60 1.500000000000000e+01 +46 61 1.600000000000000e+01 +46 62 1.700000000000000e+01 +46 63 1.800000000000000e+01 +46 64 1.900000000000000e+01 +46 65 2.000000000000000e+01 +46 66 2.100000000000000e+01 +46 67 2.200000000000000e+01 +46 68 2.300000000000000e+01 +46 69 2.400000000000000e+01 +46 70 2.500000000000000e+01 +46 71 2.600000000000000e+01 +46 72 2.700000000000000e+01 +46 73 2.800000000000000e+01 +46 74 2.900000000000000e+01 +46 75 3.000000000000000e+01 +46 76 3.100000000000000e+01 +46 77 3.200000000000000e+01 +46 78 3.300000000000000e+01 +46 79 3.400000000000000e+01 +46 80 3.500000000000000e+01 +46 81 3.600000000000000e+01 +46 82 3.700000000000000e+01 +46 83 3.800000000000000e+01 +46 84 3.900000000000000e+01 +46 85 4.000000000000000e+01 +46 86 4.100000000000000e+01 +46 87 4.200000000000000e+01 +46 88 4.300000000000000e+01 +46 89 4.400000000000000e+01 +46 90 4.500000000000000e+01 +46 91 4.600000000000000e+01 +46 92 4.700000000000000e+01 +46 93 4.800000000000000e+01 +46 94 4.900000000000000e+01 +46 95 5.000000000000000e+01 +46 96 5.100000000000000e+01 +46 97 5.200000000000000e+01 +46 98 5.300000000000000e+01 +46 99 5.400000000000000e+01 +46 100 5.500000000000000e+01 +47 47 1.000000000000000e+00 +47 48 2.000000000000000e+00 +47 49 3.000000000000000e+00 +47 50 4.000000000000000e+00 +47 51 5.000000000000000e+00 +47 52 6.000000000000000e+00 +47 53 7.000000000000000e+00 +47 54 8.000000000000000e+00 +47 55 9.000000000000000e+00 +47 56 1.000000000000000e+01 +47 57 1.100000000000000e+01 +47 58 1.200000000000000e+01 +47 59 1.300000000000000e+01 +47 60 1.400000000000000e+01 +47 61 1.500000000000000e+01 +47 62 1.600000000000000e+01 +47 63 1.700000000000000e+01 +47 64 1.800000000000000e+01 +47 65 1.900000000000000e+01 +47 66 2.000000000000000e+01 +47 67 2.100000000000000e+01 +47 68 2.200000000000000e+01 +47 69 2.300000000000000e+01 +47 70 2.400000000000000e+01 +47 71 2.500000000000000e+01 +47 72 2.600000000000000e+01 +47 73 2.700000000000000e+01 +47 74 2.800000000000000e+01 +47 75 2.900000000000000e+01 +47 76 3.000000000000000e+01 +47 77 3.100000000000000e+01 +47 78 3.200000000000000e+01 +47 79 3.300000000000000e+01 +47 80 3.400000000000000e+01 +47 81 3.500000000000000e+01 +47 82 3.600000000000000e+01 +47 83 3.700000000000000e+01 +47 84 3.800000000000000e+01 +47 85 3.900000000000000e+01 +47 86 4.000000000000000e+01 +47 87 4.100000000000000e+01 +47 88 4.200000000000000e+01 +47 89 4.300000000000000e+01 +47 90 4.400000000000000e+01 +47 91 4.500000000000000e+01 +47 92 4.600000000000000e+01 +47 93 4.700000000000000e+01 +47 94 4.800000000000000e+01 +47 95 4.900000000000000e+01 +47 96 5.000000000000000e+01 +47 97 5.100000000000000e+01 +47 98 5.200000000000000e+01 +47 99 5.300000000000000e+01 +47 100 5.400000000000000e+01 +48 48 1.000000000000000e+00 +48 49 2.000000000000000e+00 +48 50 3.000000000000000e+00 +48 51 4.000000000000000e+00 +48 52 5.000000000000000e+00 +48 53 6.000000000000000e+00 +48 54 7.000000000000000e+00 +48 55 8.000000000000000e+00 +48 56 9.000000000000000e+00 +48 57 1.000000000000000e+01 +48 58 1.100000000000000e+01 +48 59 1.200000000000000e+01 +48 60 1.300000000000000e+01 +48 61 1.400000000000000e+01 +48 62 1.500000000000000e+01 +48 63 1.600000000000000e+01 +48 64 1.700000000000000e+01 +48 65 1.800000000000000e+01 +48 66 1.900000000000000e+01 +48 67 2.000000000000000e+01 +48 68 2.100000000000000e+01 +48 69 2.200000000000000e+01 +48 70 2.300000000000000e+01 +48 71 2.400000000000000e+01 +48 72 2.500000000000000e+01 +48 73 2.600000000000000e+01 +48 74 2.700000000000000e+01 +48 75 2.800000000000000e+01 +48 76 2.900000000000000e+01 +48 77 3.000000000000000e+01 +48 78 3.100000000000000e+01 +48 79 3.200000000000000e+01 +48 80 3.300000000000000e+01 +48 81 3.400000000000000e+01 +48 82 3.500000000000000e+01 +48 83 3.600000000000000e+01 +48 84 3.700000000000000e+01 +48 85 3.800000000000000e+01 +48 86 3.900000000000000e+01 +48 87 4.000000000000000e+01 +48 88 4.100000000000000e+01 +48 89 4.200000000000000e+01 +48 90 4.300000000000000e+01 +48 91 4.400000000000000e+01 +48 92 4.500000000000000e+01 +48 93 4.600000000000000e+01 +48 94 4.700000000000000e+01 +48 95 4.800000000000000e+01 +48 96 4.900000000000000e+01 +48 97 5.000000000000000e+01 +48 98 5.100000000000000e+01 +48 99 5.200000000000000e+01 +48 100 5.300000000000000e+01 +49 49 1.000000000000000e+00 +49 50 2.000000000000000e+00 +49 51 3.000000000000000e+00 +49 52 4.000000000000000e+00 +49 53 5.000000000000000e+00 +49 54 6.000000000000000e+00 +49 55 7.000000000000000e+00 +49 56 8.000000000000000e+00 +49 57 9.000000000000000e+00 +49 58 1.000000000000000e+01 +49 59 1.100000000000000e+01 +49 60 1.200000000000000e+01 +49 61 1.300000000000000e+01 +49 62 1.400000000000000e+01 +49 63 1.500000000000000e+01 +49 64 1.600000000000000e+01 +49 65 1.700000000000000e+01 +49 66 1.800000000000000e+01 +49 67 1.900000000000000e+01 +49 68 2.000000000000000e+01 +49 69 2.100000000000000e+01 +49 70 2.200000000000000e+01 +49 71 2.300000000000000e+01 +49 72 2.400000000000000e+01 +49 73 2.500000000000000e+01 +49 74 2.600000000000000e+01 +49 75 2.700000000000000e+01 +49 76 2.800000000000000e+01 +49 77 2.900000000000000e+01 +49 78 3.000000000000000e+01 +49 79 3.100000000000000e+01 +49 80 3.200000000000000e+01 +49 81 3.300000000000000e+01 +49 82 3.400000000000000e+01 +49 83 3.500000000000000e+01 +49 84 3.600000000000000e+01 +49 85 3.700000000000000e+01 +49 86 3.800000000000000e+01 +49 87 3.900000000000000e+01 +49 88 4.000000000000000e+01 +49 89 4.100000000000000e+01 +49 90 4.200000000000000e+01 +49 91 4.300000000000000e+01 +49 92 4.400000000000000e+01 +49 93 4.500000000000000e+01 +49 94 4.600000000000000e+01 +49 95 4.700000000000000e+01 +49 96 4.800000000000000e+01 +49 97 4.900000000000000e+01 +49 98 5.000000000000000e+01 +49 99 5.100000000000000e+01 +49 100 5.200000000000000e+01 +50 50 1.000000000000000e+00 +50 51 2.000000000000000e+00 +50 52 3.000000000000000e+00 +50 53 4.000000000000000e+00 +50 54 5.000000000000000e+00 +50 55 6.000000000000000e+00 +50 56 7.000000000000000e+00 +50 57 8.000000000000000e+00 +50 58 9.000000000000000e+00 +50 59 1.000000000000000e+01 +50 60 1.100000000000000e+01 +50 61 1.200000000000000e+01 +50 62 1.300000000000000e+01 +50 63 1.400000000000000e+01 +50 64 1.500000000000000e+01 +50 65 1.600000000000000e+01 +50 66 1.700000000000000e+01 +50 67 1.800000000000000e+01 +50 68 1.900000000000000e+01 +50 69 2.000000000000000e+01 +50 70 2.100000000000000e+01 +50 71 2.200000000000000e+01 +50 72 2.300000000000000e+01 +50 73 2.400000000000000e+01 +50 74 2.500000000000000e+01 +50 75 2.600000000000000e+01 +50 76 2.700000000000000e+01 +50 77 2.800000000000000e+01 +50 78 2.900000000000000e+01 +50 79 3.000000000000000e+01 +50 80 3.100000000000000e+01 +50 81 3.200000000000000e+01 +50 82 3.300000000000000e+01 +50 83 3.400000000000000e+01 +50 84 3.500000000000000e+01 +50 85 3.600000000000000e+01 +50 86 3.700000000000000e+01 +50 87 3.800000000000000e+01 +50 88 3.900000000000000e+01 +50 89 4.000000000000000e+01 +50 90 4.100000000000000e+01 +50 91 4.200000000000000e+01 +50 92 4.300000000000000e+01 +50 93 4.400000000000000e+01 +50 94 4.500000000000000e+01 +50 95 4.600000000000000e+01 +50 96 4.700000000000000e+01 +50 97 4.800000000000000e+01 +50 98 4.900000000000000e+01 +50 99 5.000000000000000e+01 +50 100 5.100000000000000e+01 +51 51 1.000000000000000e+00 +51 52 2.000000000000000e+00 +51 53 3.000000000000000e+00 +51 54 4.000000000000000e+00 +51 55 5.000000000000000e+00 +51 56 6.000000000000000e+00 +51 57 7.000000000000000e+00 +51 58 8.000000000000000e+00 +51 59 9.000000000000000e+00 +51 60 1.000000000000000e+01 +51 61 1.100000000000000e+01 +51 62 1.200000000000000e+01 +51 63 1.300000000000000e+01 +51 64 1.400000000000000e+01 +51 65 1.500000000000000e+01 +51 66 1.600000000000000e+01 +51 67 1.700000000000000e+01 +51 68 1.800000000000000e+01 +51 69 1.900000000000000e+01 +51 70 2.000000000000000e+01 +51 71 2.100000000000000e+01 +51 72 2.200000000000000e+01 +51 73 2.300000000000000e+01 +51 74 2.400000000000000e+01 +51 75 2.500000000000000e+01 +51 76 2.600000000000000e+01 +51 77 2.700000000000000e+01 +51 78 2.800000000000000e+01 +51 79 2.900000000000000e+01 +51 80 3.000000000000000e+01 +51 81 3.100000000000000e+01 +51 82 3.200000000000000e+01 +51 83 3.300000000000000e+01 +51 84 3.400000000000000e+01 +51 85 3.500000000000000e+01 +51 86 3.600000000000000e+01 +51 87 3.700000000000000e+01 +51 88 3.800000000000000e+01 +51 89 3.900000000000000e+01 +51 90 4.000000000000000e+01 +51 91 4.100000000000000e+01 +51 92 4.200000000000000e+01 +51 93 4.300000000000000e+01 +51 94 4.400000000000000e+01 +51 95 4.500000000000000e+01 +51 96 4.600000000000000e+01 +51 97 4.700000000000000e+01 +51 98 4.800000000000000e+01 +51 99 4.900000000000000e+01 +51 100 5.000000000000000e+01 +52 52 1.000000000000000e+00 +52 53 2.000000000000000e+00 +52 54 3.000000000000000e+00 +52 55 4.000000000000000e+00 +52 56 5.000000000000000e+00 +52 57 6.000000000000000e+00 +52 58 7.000000000000000e+00 +52 59 8.000000000000000e+00 +52 60 9.000000000000000e+00 +52 61 1.000000000000000e+01 +52 62 1.100000000000000e+01 +52 63 1.200000000000000e+01 +52 64 1.300000000000000e+01 +52 65 1.400000000000000e+01 +52 66 1.500000000000000e+01 +52 67 1.600000000000000e+01 +52 68 1.700000000000000e+01 +52 69 1.800000000000000e+01 +52 70 1.900000000000000e+01 +52 71 2.000000000000000e+01 +52 72 2.100000000000000e+01 +52 73 2.200000000000000e+01 +52 74 2.300000000000000e+01 +52 75 2.400000000000000e+01 +52 76 2.500000000000000e+01 +52 77 2.600000000000000e+01 +52 78 2.700000000000000e+01 +52 79 2.800000000000000e+01 +52 80 2.900000000000000e+01 +52 81 3.000000000000000e+01 +52 82 3.100000000000000e+01 +52 83 3.200000000000000e+01 +52 84 3.300000000000000e+01 +52 85 3.400000000000000e+01 +52 86 3.500000000000000e+01 +52 87 3.600000000000000e+01 +52 88 3.700000000000000e+01 +52 89 3.800000000000000e+01 +52 90 3.900000000000000e+01 +52 91 4.000000000000000e+01 +52 92 4.100000000000000e+01 +52 93 4.200000000000000e+01 +52 94 4.300000000000000e+01 +52 95 4.400000000000000e+01 +52 96 4.500000000000000e+01 +52 97 4.600000000000000e+01 +52 98 4.700000000000000e+01 +52 99 4.800000000000000e+01 +52 100 4.900000000000000e+01 +53 53 1.000000000000000e+00 +53 54 2.000000000000000e+00 +53 55 3.000000000000000e+00 +53 56 4.000000000000000e+00 +53 57 5.000000000000000e+00 +53 58 6.000000000000000e+00 +53 59 7.000000000000000e+00 +53 60 8.000000000000000e+00 +53 61 9.000000000000000e+00 +53 62 1.000000000000000e+01 +53 63 1.100000000000000e+01 +53 64 1.200000000000000e+01 +53 65 1.300000000000000e+01 +53 66 1.400000000000000e+01 +53 67 1.500000000000000e+01 +53 68 1.600000000000000e+01 +53 69 1.700000000000000e+01 +53 70 1.800000000000000e+01 +53 71 1.900000000000000e+01 +53 72 2.000000000000000e+01 +53 73 2.100000000000000e+01 +53 74 2.200000000000000e+01 +53 75 2.300000000000000e+01 +53 76 2.400000000000000e+01 +53 77 2.500000000000000e+01 +53 78 2.600000000000000e+01 +53 79 2.700000000000000e+01 +53 80 2.800000000000000e+01 +53 81 2.900000000000000e+01 +53 82 3.000000000000000e+01 +53 83 3.100000000000000e+01 +53 84 3.200000000000000e+01 +53 85 3.300000000000000e+01 +53 86 3.400000000000000e+01 +53 87 3.500000000000000e+01 +53 88 3.600000000000000e+01 +53 89 3.700000000000000e+01 +53 90 3.800000000000000e+01 +53 91 3.900000000000000e+01 +53 92 4.000000000000000e+01 +53 93 4.100000000000000e+01 +53 94 4.200000000000000e+01 +53 95 4.300000000000000e+01 +53 96 4.400000000000000e+01 +53 97 4.500000000000000e+01 +53 98 4.600000000000000e+01 +53 99 4.700000000000000e+01 +53 100 4.800000000000000e+01 +54 54 1.000000000000000e+00 +54 55 2.000000000000000e+00 +54 56 3.000000000000000e+00 +54 57 4.000000000000000e+00 +54 58 5.000000000000000e+00 +54 59 6.000000000000000e+00 +54 60 7.000000000000000e+00 +54 61 8.000000000000000e+00 +54 62 9.000000000000000e+00 +54 63 1.000000000000000e+01 +54 64 1.100000000000000e+01 +54 65 1.200000000000000e+01 +54 66 1.300000000000000e+01 +54 67 1.400000000000000e+01 +54 68 1.500000000000000e+01 +54 69 1.600000000000000e+01 +54 70 1.700000000000000e+01 +54 71 1.800000000000000e+01 +54 72 1.900000000000000e+01 +54 73 2.000000000000000e+01 +54 74 2.100000000000000e+01 +54 75 2.200000000000000e+01 +54 76 2.300000000000000e+01 +54 77 2.400000000000000e+01 +54 78 2.500000000000000e+01 +54 79 2.600000000000000e+01 +54 80 2.700000000000000e+01 +54 81 2.800000000000000e+01 +54 82 2.900000000000000e+01 +54 83 3.000000000000000e+01 +54 84 3.100000000000000e+01 +54 85 3.200000000000000e+01 +54 86 3.300000000000000e+01 +54 87 3.400000000000000e+01 +54 88 3.500000000000000e+01 +54 89 3.600000000000000e+01 +54 90 3.700000000000000e+01 +54 91 3.800000000000000e+01 +54 92 3.900000000000000e+01 +54 93 4.000000000000000e+01 +54 94 4.100000000000000e+01 +54 95 4.200000000000000e+01 +54 96 4.300000000000000e+01 +54 97 4.400000000000000e+01 +54 98 4.500000000000000e+01 +54 99 4.600000000000000e+01 +54 100 4.700000000000000e+01 +55 55 1.000000000000000e+00 +55 56 2.000000000000000e+00 +55 57 3.000000000000000e+00 +55 58 4.000000000000000e+00 +55 59 5.000000000000000e+00 +55 60 6.000000000000000e+00 +55 61 7.000000000000000e+00 +55 62 8.000000000000000e+00 +55 63 9.000000000000000e+00 +55 64 1.000000000000000e+01 +55 65 1.100000000000000e+01 +55 66 1.200000000000000e+01 +55 67 1.300000000000000e+01 +55 68 1.400000000000000e+01 +55 69 1.500000000000000e+01 +55 70 1.600000000000000e+01 +55 71 1.700000000000000e+01 +55 72 1.800000000000000e+01 +55 73 1.900000000000000e+01 +55 74 2.000000000000000e+01 +55 75 2.100000000000000e+01 +55 76 2.200000000000000e+01 +55 77 2.300000000000000e+01 +55 78 2.400000000000000e+01 +55 79 2.500000000000000e+01 +55 80 2.600000000000000e+01 +55 81 2.700000000000000e+01 +55 82 2.800000000000000e+01 +55 83 2.900000000000000e+01 +55 84 3.000000000000000e+01 +55 85 3.100000000000000e+01 +55 86 3.200000000000000e+01 +55 87 3.300000000000000e+01 +55 88 3.400000000000000e+01 +55 89 3.500000000000000e+01 +55 90 3.600000000000000e+01 +55 91 3.700000000000000e+01 +55 92 3.800000000000000e+01 +55 93 3.900000000000000e+01 +55 94 4.000000000000000e+01 +55 95 4.100000000000000e+01 +55 96 4.200000000000000e+01 +55 97 4.300000000000000e+01 +55 98 4.400000000000000e+01 +55 99 4.500000000000000e+01 +55 100 4.600000000000000e+01 +56 56 1.000000000000000e+00 +56 57 2.000000000000000e+00 +56 58 3.000000000000000e+00 +56 59 4.000000000000000e+00 +56 60 5.000000000000000e+00 +56 61 6.000000000000000e+00 +56 62 7.000000000000000e+00 +56 63 8.000000000000000e+00 +56 64 9.000000000000000e+00 +56 65 1.000000000000000e+01 +56 66 1.100000000000000e+01 +56 67 1.200000000000000e+01 +56 68 1.300000000000000e+01 +56 69 1.400000000000000e+01 +56 70 1.500000000000000e+01 +56 71 1.600000000000000e+01 +56 72 1.700000000000000e+01 +56 73 1.800000000000000e+01 +56 74 1.900000000000000e+01 +56 75 2.000000000000000e+01 +56 76 2.100000000000000e+01 +56 77 2.200000000000000e+01 +56 78 2.300000000000000e+01 +56 79 2.400000000000000e+01 +56 80 2.500000000000000e+01 +56 81 2.600000000000000e+01 +56 82 2.700000000000000e+01 +56 83 2.800000000000000e+01 +56 84 2.900000000000000e+01 +56 85 3.000000000000000e+01 +56 86 3.100000000000000e+01 +56 87 3.200000000000000e+01 +56 88 3.300000000000000e+01 +56 89 3.400000000000000e+01 +56 90 3.500000000000000e+01 +56 91 3.600000000000000e+01 +56 92 3.700000000000000e+01 +56 93 3.800000000000000e+01 +56 94 3.900000000000000e+01 +56 95 4.000000000000000e+01 +56 96 4.100000000000000e+01 +56 97 4.200000000000000e+01 +56 98 4.300000000000000e+01 +56 99 4.400000000000000e+01 +56 100 4.500000000000000e+01 +57 57 1.000000000000000e+00 +57 58 2.000000000000000e+00 +57 59 3.000000000000000e+00 +57 60 4.000000000000000e+00 +57 61 5.000000000000000e+00 +57 62 6.000000000000000e+00 +57 63 7.000000000000000e+00 +57 64 8.000000000000000e+00 +57 65 9.000000000000000e+00 +57 66 1.000000000000000e+01 +57 67 1.100000000000000e+01 +57 68 1.200000000000000e+01 +57 69 1.300000000000000e+01 +57 70 1.400000000000000e+01 +57 71 1.500000000000000e+01 +57 72 1.600000000000000e+01 +57 73 1.700000000000000e+01 +57 74 1.800000000000000e+01 +57 75 1.900000000000000e+01 +57 76 2.000000000000000e+01 +57 77 2.100000000000000e+01 +57 78 2.200000000000000e+01 +57 79 2.300000000000000e+01 +57 80 2.400000000000000e+01 +57 81 2.500000000000000e+01 +57 82 2.600000000000000e+01 +57 83 2.700000000000000e+01 +57 84 2.800000000000000e+01 +57 85 2.900000000000000e+01 +57 86 3.000000000000000e+01 +57 87 3.100000000000000e+01 +57 88 3.200000000000000e+01 +57 89 3.300000000000000e+01 +57 90 3.400000000000000e+01 +57 91 3.500000000000000e+01 +57 92 3.600000000000000e+01 +57 93 3.700000000000000e+01 +57 94 3.800000000000000e+01 +57 95 3.900000000000000e+01 +57 96 4.000000000000000e+01 +57 97 4.100000000000000e+01 +57 98 4.200000000000000e+01 +57 99 4.300000000000000e+01 +57 100 4.400000000000000e+01 +58 58 1.000000000000000e+00 +58 59 2.000000000000000e+00 +58 60 3.000000000000000e+00 +58 61 4.000000000000000e+00 +58 62 5.000000000000000e+00 +58 63 6.000000000000000e+00 +58 64 7.000000000000000e+00 +58 65 8.000000000000000e+00 +58 66 9.000000000000000e+00 +58 67 1.000000000000000e+01 +58 68 1.100000000000000e+01 +58 69 1.200000000000000e+01 +58 70 1.300000000000000e+01 +58 71 1.400000000000000e+01 +58 72 1.500000000000000e+01 +58 73 1.600000000000000e+01 +58 74 1.700000000000000e+01 +58 75 1.800000000000000e+01 +58 76 1.900000000000000e+01 +58 77 2.000000000000000e+01 +58 78 2.100000000000000e+01 +58 79 2.200000000000000e+01 +58 80 2.300000000000000e+01 +58 81 2.400000000000000e+01 +58 82 2.500000000000000e+01 +58 83 2.600000000000000e+01 +58 84 2.700000000000000e+01 +58 85 2.800000000000000e+01 +58 86 2.900000000000000e+01 +58 87 3.000000000000000e+01 +58 88 3.100000000000000e+01 +58 89 3.200000000000000e+01 +58 90 3.300000000000000e+01 +58 91 3.400000000000000e+01 +58 92 3.500000000000000e+01 +58 93 3.600000000000000e+01 +58 94 3.700000000000000e+01 +58 95 3.800000000000000e+01 +58 96 3.900000000000000e+01 +58 97 4.000000000000000e+01 +58 98 4.100000000000000e+01 +58 99 4.200000000000000e+01 +58 100 4.300000000000000e+01 +59 59 1.000000000000000e+00 +59 60 2.000000000000000e+00 +59 61 3.000000000000000e+00 +59 62 4.000000000000000e+00 +59 63 5.000000000000000e+00 +59 64 6.000000000000000e+00 +59 65 7.000000000000000e+00 +59 66 8.000000000000000e+00 +59 67 9.000000000000000e+00 +59 68 1.000000000000000e+01 +59 69 1.100000000000000e+01 +59 70 1.200000000000000e+01 +59 71 1.300000000000000e+01 +59 72 1.400000000000000e+01 +59 73 1.500000000000000e+01 +59 74 1.600000000000000e+01 +59 75 1.700000000000000e+01 +59 76 1.800000000000000e+01 +59 77 1.900000000000000e+01 +59 78 2.000000000000000e+01 +59 79 2.100000000000000e+01 +59 80 2.200000000000000e+01 +59 81 2.300000000000000e+01 +59 82 2.400000000000000e+01 +59 83 2.500000000000000e+01 +59 84 2.600000000000000e+01 +59 85 2.700000000000000e+01 +59 86 2.800000000000000e+01 +59 87 2.900000000000000e+01 +59 88 3.000000000000000e+01 +59 89 3.100000000000000e+01 +59 90 3.200000000000000e+01 +59 91 3.300000000000000e+01 +59 92 3.400000000000000e+01 +59 93 3.500000000000000e+01 +59 94 3.600000000000000e+01 +59 95 3.700000000000000e+01 +59 96 3.800000000000000e+01 +59 97 3.900000000000000e+01 +59 98 4.000000000000000e+01 +59 99 4.100000000000000e+01 +59 100 4.200000000000000e+01 +60 60 1.000000000000000e+00 +60 61 2.000000000000000e+00 +60 62 3.000000000000000e+00 +60 63 4.000000000000000e+00 +60 64 5.000000000000000e+00 +60 65 6.000000000000000e+00 +60 66 7.000000000000000e+00 +60 67 8.000000000000000e+00 +60 68 9.000000000000000e+00 +60 69 1.000000000000000e+01 +60 70 1.100000000000000e+01 +60 71 1.200000000000000e+01 +60 72 1.300000000000000e+01 +60 73 1.400000000000000e+01 +60 74 1.500000000000000e+01 +60 75 1.600000000000000e+01 +60 76 1.700000000000000e+01 +60 77 1.800000000000000e+01 +60 78 1.900000000000000e+01 +60 79 2.000000000000000e+01 +60 80 2.100000000000000e+01 +60 81 2.200000000000000e+01 +60 82 2.300000000000000e+01 +60 83 2.400000000000000e+01 +60 84 2.500000000000000e+01 +60 85 2.600000000000000e+01 +60 86 2.700000000000000e+01 +60 87 2.800000000000000e+01 +60 88 2.900000000000000e+01 +60 89 3.000000000000000e+01 +60 90 3.100000000000000e+01 +60 91 3.200000000000000e+01 +60 92 3.300000000000000e+01 +60 93 3.400000000000000e+01 +60 94 3.500000000000000e+01 +60 95 3.600000000000000e+01 +60 96 3.700000000000000e+01 +60 97 3.800000000000000e+01 +60 98 3.900000000000000e+01 +60 99 4.000000000000000e+01 +60 100 4.100000000000000e+01 +61 61 1.000000000000000e+00 +61 62 2.000000000000000e+00 +61 63 3.000000000000000e+00 +61 64 4.000000000000000e+00 +61 65 5.000000000000000e+00 +61 66 6.000000000000000e+00 +61 67 7.000000000000000e+00 +61 68 8.000000000000000e+00 +61 69 9.000000000000000e+00 +61 70 1.000000000000000e+01 +61 71 1.100000000000000e+01 +61 72 1.200000000000000e+01 +61 73 1.300000000000000e+01 +61 74 1.400000000000000e+01 +61 75 1.500000000000000e+01 +61 76 1.600000000000000e+01 +61 77 1.700000000000000e+01 +61 78 1.800000000000000e+01 +61 79 1.900000000000000e+01 +61 80 2.000000000000000e+01 +61 81 2.100000000000000e+01 +61 82 2.200000000000000e+01 +61 83 2.300000000000000e+01 +61 84 2.400000000000000e+01 +61 85 2.500000000000000e+01 +61 86 2.600000000000000e+01 +61 87 2.700000000000000e+01 +61 88 2.800000000000000e+01 +61 89 2.900000000000000e+01 +61 90 3.000000000000000e+01 +61 91 3.100000000000000e+01 +61 92 3.200000000000000e+01 +61 93 3.300000000000000e+01 +61 94 3.400000000000000e+01 +61 95 3.500000000000000e+01 +61 96 3.600000000000000e+01 +61 97 3.700000000000000e+01 +61 98 3.800000000000000e+01 +61 99 3.900000000000000e+01 +61 100 4.000000000000000e+01 +62 62 1.000000000000000e+00 +62 63 2.000000000000000e+00 +62 64 3.000000000000000e+00 +62 65 4.000000000000000e+00 +62 66 5.000000000000000e+00 +62 67 6.000000000000000e+00 +62 68 7.000000000000000e+00 +62 69 8.000000000000000e+00 +62 70 9.000000000000000e+00 +62 71 1.000000000000000e+01 +62 72 1.100000000000000e+01 +62 73 1.200000000000000e+01 +62 74 1.300000000000000e+01 +62 75 1.400000000000000e+01 +62 76 1.500000000000000e+01 +62 77 1.600000000000000e+01 +62 78 1.700000000000000e+01 +62 79 1.800000000000000e+01 +62 80 1.900000000000000e+01 +62 81 2.000000000000000e+01 +62 82 2.100000000000000e+01 +62 83 2.200000000000000e+01 +62 84 2.300000000000000e+01 +62 85 2.400000000000000e+01 +62 86 2.500000000000000e+01 +62 87 2.600000000000000e+01 +62 88 2.700000000000000e+01 +62 89 2.800000000000000e+01 +62 90 2.900000000000000e+01 +62 91 3.000000000000000e+01 +62 92 3.100000000000000e+01 +62 93 3.200000000000000e+01 +62 94 3.300000000000000e+01 +62 95 3.400000000000000e+01 +62 96 3.500000000000000e+01 +62 97 3.600000000000000e+01 +62 98 3.700000000000000e+01 +62 99 3.800000000000000e+01 +62 100 3.900000000000000e+01 +63 63 1.000000000000000e+00 +63 64 2.000000000000000e+00 +63 65 3.000000000000000e+00 +63 66 4.000000000000000e+00 +63 67 5.000000000000000e+00 +63 68 6.000000000000000e+00 +63 69 7.000000000000000e+00 +63 70 8.000000000000000e+00 +63 71 9.000000000000000e+00 +63 72 1.000000000000000e+01 +63 73 1.100000000000000e+01 +63 74 1.200000000000000e+01 +63 75 1.300000000000000e+01 +63 76 1.400000000000000e+01 +63 77 1.500000000000000e+01 +63 78 1.600000000000000e+01 +63 79 1.700000000000000e+01 +63 80 1.800000000000000e+01 +63 81 1.900000000000000e+01 +63 82 2.000000000000000e+01 +63 83 2.100000000000000e+01 +63 84 2.200000000000000e+01 +63 85 2.300000000000000e+01 +63 86 2.400000000000000e+01 +63 87 2.500000000000000e+01 +63 88 2.600000000000000e+01 +63 89 2.700000000000000e+01 +63 90 2.800000000000000e+01 +63 91 2.900000000000000e+01 +63 92 3.000000000000000e+01 +63 93 3.100000000000000e+01 +63 94 3.200000000000000e+01 +63 95 3.300000000000000e+01 +63 96 3.400000000000000e+01 +63 97 3.500000000000000e+01 +63 98 3.600000000000000e+01 +63 99 3.700000000000000e+01 +63 100 3.800000000000000e+01 +64 64 1.000000000000000e+00 +64 65 2.000000000000000e+00 +64 66 3.000000000000000e+00 +64 67 4.000000000000000e+00 +64 68 5.000000000000000e+00 +64 69 6.000000000000000e+00 +64 70 7.000000000000000e+00 +64 71 8.000000000000000e+00 +64 72 9.000000000000000e+00 +64 73 1.000000000000000e+01 +64 74 1.100000000000000e+01 +64 75 1.200000000000000e+01 +64 76 1.300000000000000e+01 +64 77 1.400000000000000e+01 +64 78 1.500000000000000e+01 +64 79 1.600000000000000e+01 +64 80 1.700000000000000e+01 +64 81 1.800000000000000e+01 +64 82 1.900000000000000e+01 +64 83 2.000000000000000e+01 +64 84 2.100000000000000e+01 +64 85 2.200000000000000e+01 +64 86 2.300000000000000e+01 +64 87 2.400000000000000e+01 +64 88 2.500000000000000e+01 +64 89 2.600000000000000e+01 +64 90 2.700000000000000e+01 +64 91 2.800000000000000e+01 +64 92 2.900000000000000e+01 +64 93 3.000000000000000e+01 +64 94 3.100000000000000e+01 +64 95 3.200000000000000e+01 +64 96 3.300000000000000e+01 +64 97 3.400000000000000e+01 +64 98 3.500000000000000e+01 +64 99 3.600000000000000e+01 +64 100 3.700000000000000e+01 +65 65 1.000000000000000e+00 +65 66 2.000000000000000e+00 +65 67 3.000000000000000e+00 +65 68 4.000000000000000e+00 +65 69 5.000000000000000e+00 +65 70 6.000000000000000e+00 +65 71 7.000000000000000e+00 +65 72 8.000000000000000e+00 +65 73 9.000000000000000e+00 +65 74 1.000000000000000e+01 +65 75 1.100000000000000e+01 +65 76 1.200000000000000e+01 +65 77 1.300000000000000e+01 +65 78 1.400000000000000e+01 +65 79 1.500000000000000e+01 +65 80 1.600000000000000e+01 +65 81 1.700000000000000e+01 +65 82 1.800000000000000e+01 +65 83 1.900000000000000e+01 +65 84 2.000000000000000e+01 +65 85 2.100000000000000e+01 +65 86 2.200000000000000e+01 +65 87 2.300000000000000e+01 +65 88 2.400000000000000e+01 +65 89 2.500000000000000e+01 +65 90 2.600000000000000e+01 +65 91 2.700000000000000e+01 +65 92 2.800000000000000e+01 +65 93 2.900000000000000e+01 +65 94 3.000000000000000e+01 +65 95 3.100000000000000e+01 +65 96 3.200000000000000e+01 +65 97 3.300000000000000e+01 +65 98 3.400000000000000e+01 +65 99 3.500000000000000e+01 +65 100 3.600000000000000e+01 +66 66 1.000000000000000e+00 +66 67 2.000000000000000e+00 +66 68 3.000000000000000e+00 +66 69 4.000000000000000e+00 +66 70 5.000000000000000e+00 +66 71 6.000000000000000e+00 +66 72 7.000000000000000e+00 +66 73 8.000000000000000e+00 +66 74 9.000000000000000e+00 +66 75 1.000000000000000e+01 +66 76 1.100000000000000e+01 +66 77 1.200000000000000e+01 +66 78 1.300000000000000e+01 +66 79 1.400000000000000e+01 +66 80 1.500000000000000e+01 +66 81 1.600000000000000e+01 +66 82 1.700000000000000e+01 +66 83 1.800000000000000e+01 +66 84 1.900000000000000e+01 +66 85 2.000000000000000e+01 +66 86 2.100000000000000e+01 +66 87 2.200000000000000e+01 +66 88 2.300000000000000e+01 +66 89 2.400000000000000e+01 +66 90 2.500000000000000e+01 +66 91 2.600000000000000e+01 +66 92 2.700000000000000e+01 +66 93 2.800000000000000e+01 +66 94 2.900000000000000e+01 +66 95 3.000000000000000e+01 +66 96 3.100000000000000e+01 +66 97 3.200000000000000e+01 +66 98 3.300000000000000e+01 +66 99 3.400000000000000e+01 +66 100 3.500000000000000e+01 +67 67 1.000000000000000e+00 +67 68 2.000000000000000e+00 +67 69 3.000000000000000e+00 +67 70 4.000000000000000e+00 +67 71 5.000000000000000e+00 +67 72 6.000000000000000e+00 +67 73 7.000000000000000e+00 +67 74 8.000000000000000e+00 +67 75 9.000000000000000e+00 +67 76 1.000000000000000e+01 +67 77 1.100000000000000e+01 +67 78 1.200000000000000e+01 +67 79 1.300000000000000e+01 +67 80 1.400000000000000e+01 +67 81 1.500000000000000e+01 +67 82 1.600000000000000e+01 +67 83 1.700000000000000e+01 +67 84 1.800000000000000e+01 +67 85 1.900000000000000e+01 +67 86 2.000000000000000e+01 +67 87 2.100000000000000e+01 +67 88 2.200000000000000e+01 +67 89 2.300000000000000e+01 +67 90 2.400000000000000e+01 +67 91 2.500000000000000e+01 +67 92 2.600000000000000e+01 +67 93 2.700000000000000e+01 +67 94 2.800000000000000e+01 +67 95 2.900000000000000e+01 +67 96 3.000000000000000e+01 +67 97 3.100000000000000e+01 +67 98 3.200000000000000e+01 +67 99 3.300000000000000e+01 +67 100 3.400000000000000e+01 +68 68 1.000000000000000e+00 +68 69 2.000000000000000e+00 +68 70 3.000000000000000e+00 +68 71 4.000000000000000e+00 +68 72 5.000000000000000e+00 +68 73 6.000000000000000e+00 +68 74 7.000000000000000e+00 +68 75 8.000000000000000e+00 +68 76 9.000000000000000e+00 +68 77 1.000000000000000e+01 +68 78 1.100000000000000e+01 +68 79 1.200000000000000e+01 +68 80 1.300000000000000e+01 +68 81 1.400000000000000e+01 +68 82 1.500000000000000e+01 +68 83 1.600000000000000e+01 +68 84 1.700000000000000e+01 +68 85 1.800000000000000e+01 +68 86 1.900000000000000e+01 +68 87 2.000000000000000e+01 +68 88 2.100000000000000e+01 +68 89 2.200000000000000e+01 +68 90 2.300000000000000e+01 +68 91 2.400000000000000e+01 +68 92 2.500000000000000e+01 +68 93 2.600000000000000e+01 +68 94 2.700000000000000e+01 +68 95 2.800000000000000e+01 +68 96 2.900000000000000e+01 +68 97 3.000000000000000e+01 +68 98 3.100000000000000e+01 +68 99 3.200000000000000e+01 +68 100 3.300000000000000e+01 +69 69 1.000000000000000e+00 +69 70 2.000000000000000e+00 +69 71 3.000000000000000e+00 +69 72 4.000000000000000e+00 +69 73 5.000000000000000e+00 +69 74 6.000000000000000e+00 +69 75 7.000000000000000e+00 +69 76 8.000000000000000e+00 +69 77 9.000000000000000e+00 +69 78 1.000000000000000e+01 +69 79 1.100000000000000e+01 +69 80 1.200000000000000e+01 +69 81 1.300000000000000e+01 +69 82 1.400000000000000e+01 +69 83 1.500000000000000e+01 +69 84 1.600000000000000e+01 +69 85 1.700000000000000e+01 +69 86 1.800000000000000e+01 +69 87 1.900000000000000e+01 +69 88 2.000000000000000e+01 +69 89 2.100000000000000e+01 +69 90 2.200000000000000e+01 +69 91 2.300000000000000e+01 +69 92 2.400000000000000e+01 +69 93 2.500000000000000e+01 +69 94 2.600000000000000e+01 +69 95 2.700000000000000e+01 +69 96 2.800000000000000e+01 +69 97 2.900000000000000e+01 +69 98 3.000000000000000e+01 +69 99 3.100000000000000e+01 +69 100 3.200000000000000e+01 +70 70 1.000000000000000e+00 +70 71 2.000000000000000e+00 +70 72 3.000000000000000e+00 +70 73 4.000000000000000e+00 +70 74 5.000000000000000e+00 +70 75 6.000000000000000e+00 +70 76 7.000000000000000e+00 +70 77 8.000000000000000e+00 +70 78 9.000000000000000e+00 +70 79 1.000000000000000e+01 +70 80 1.100000000000000e+01 +70 81 1.200000000000000e+01 +70 82 1.300000000000000e+01 +70 83 1.400000000000000e+01 +70 84 1.500000000000000e+01 +70 85 1.600000000000000e+01 +70 86 1.700000000000000e+01 +70 87 1.800000000000000e+01 +70 88 1.900000000000000e+01 +70 89 2.000000000000000e+01 +70 90 2.100000000000000e+01 +70 91 2.200000000000000e+01 +70 92 2.300000000000000e+01 +70 93 2.400000000000000e+01 +70 94 2.500000000000000e+01 +70 95 2.600000000000000e+01 +70 96 2.700000000000000e+01 +70 97 2.800000000000000e+01 +70 98 2.900000000000000e+01 +70 99 3.000000000000000e+01 +70 100 3.100000000000000e+01 +71 71 1.000000000000000e+00 +71 72 2.000000000000000e+00 +71 73 3.000000000000000e+00 +71 74 4.000000000000000e+00 +71 75 5.000000000000000e+00 +71 76 6.000000000000000e+00 +71 77 7.000000000000000e+00 +71 78 8.000000000000000e+00 +71 79 9.000000000000000e+00 +71 80 1.000000000000000e+01 +71 81 1.100000000000000e+01 +71 82 1.200000000000000e+01 +71 83 1.300000000000000e+01 +71 84 1.400000000000000e+01 +71 85 1.500000000000000e+01 +71 86 1.600000000000000e+01 +71 87 1.700000000000000e+01 +71 88 1.800000000000000e+01 +71 89 1.900000000000000e+01 +71 90 2.000000000000000e+01 +71 91 2.100000000000000e+01 +71 92 2.200000000000000e+01 +71 93 2.300000000000000e+01 +71 94 2.400000000000000e+01 +71 95 2.500000000000000e+01 +71 96 2.600000000000000e+01 +71 97 2.700000000000000e+01 +71 98 2.800000000000000e+01 +71 99 2.900000000000000e+01 +71 100 3.000000000000000e+01 +72 72 1.000000000000000e+00 +72 73 2.000000000000000e+00 +72 74 3.000000000000000e+00 +72 75 4.000000000000000e+00 +72 76 5.000000000000000e+00 +72 77 6.000000000000000e+00 +72 78 7.000000000000000e+00 +72 79 8.000000000000000e+00 +72 80 9.000000000000000e+00 +72 81 1.000000000000000e+01 +72 82 1.100000000000000e+01 +72 83 1.200000000000000e+01 +72 84 1.300000000000000e+01 +72 85 1.400000000000000e+01 +72 86 1.500000000000000e+01 +72 87 1.600000000000000e+01 +72 88 1.700000000000000e+01 +72 89 1.800000000000000e+01 +72 90 1.900000000000000e+01 +72 91 2.000000000000000e+01 +72 92 2.100000000000000e+01 +72 93 2.200000000000000e+01 +72 94 2.300000000000000e+01 +72 95 2.400000000000000e+01 +72 96 2.500000000000000e+01 +72 97 2.600000000000000e+01 +72 98 2.700000000000000e+01 +72 99 2.800000000000000e+01 +72 100 2.900000000000000e+01 +73 73 1.000000000000000e+00 +73 74 2.000000000000000e+00 +73 75 3.000000000000000e+00 +73 76 4.000000000000000e+00 +73 77 5.000000000000000e+00 +73 78 6.000000000000000e+00 +73 79 7.000000000000000e+00 +73 80 8.000000000000000e+00 +73 81 9.000000000000000e+00 +73 82 1.000000000000000e+01 +73 83 1.100000000000000e+01 +73 84 1.200000000000000e+01 +73 85 1.300000000000000e+01 +73 86 1.400000000000000e+01 +73 87 1.500000000000000e+01 +73 88 1.600000000000000e+01 +73 89 1.700000000000000e+01 +73 90 1.800000000000000e+01 +73 91 1.900000000000000e+01 +73 92 2.000000000000000e+01 +73 93 2.100000000000000e+01 +73 94 2.200000000000000e+01 +73 95 2.300000000000000e+01 +73 96 2.400000000000000e+01 +73 97 2.500000000000000e+01 +73 98 2.600000000000000e+01 +73 99 2.700000000000000e+01 +73 100 2.800000000000000e+01 +74 74 1.000000000000000e+00 +74 75 2.000000000000000e+00 +74 76 3.000000000000000e+00 +74 77 4.000000000000000e+00 +74 78 5.000000000000000e+00 +74 79 6.000000000000000e+00 +74 80 7.000000000000000e+00 +74 81 8.000000000000000e+00 +74 82 9.000000000000000e+00 +74 83 1.000000000000000e+01 +74 84 1.100000000000000e+01 +74 85 1.200000000000000e+01 +74 86 1.300000000000000e+01 +74 87 1.400000000000000e+01 +74 88 1.500000000000000e+01 +74 89 1.600000000000000e+01 +74 90 1.700000000000000e+01 +74 91 1.800000000000000e+01 +74 92 1.900000000000000e+01 +74 93 2.000000000000000e+01 +74 94 2.100000000000000e+01 +74 95 2.200000000000000e+01 +74 96 2.300000000000000e+01 +74 97 2.400000000000000e+01 +74 98 2.500000000000000e+01 +74 99 2.600000000000000e+01 +74 100 2.700000000000000e+01 +75 75 1.000000000000000e+00 +75 76 2.000000000000000e+00 +75 77 3.000000000000000e+00 +75 78 4.000000000000000e+00 +75 79 5.000000000000000e+00 +75 80 6.000000000000000e+00 +75 81 7.000000000000000e+00 +75 82 8.000000000000000e+00 +75 83 9.000000000000000e+00 +75 84 1.000000000000000e+01 +75 85 1.100000000000000e+01 +75 86 1.200000000000000e+01 +75 87 1.300000000000000e+01 +75 88 1.400000000000000e+01 +75 89 1.500000000000000e+01 +75 90 1.600000000000000e+01 +75 91 1.700000000000000e+01 +75 92 1.800000000000000e+01 +75 93 1.900000000000000e+01 +75 94 2.000000000000000e+01 +75 95 2.100000000000000e+01 +75 96 2.200000000000000e+01 +75 97 2.300000000000000e+01 +75 98 2.400000000000000e+01 +75 99 2.500000000000000e+01 +75 100 2.600000000000000e+01 +76 76 1.000000000000000e+00 +76 77 2.000000000000000e+00 +76 78 3.000000000000000e+00 +76 79 4.000000000000000e+00 +76 80 5.000000000000000e+00 +76 81 6.000000000000000e+00 +76 82 7.000000000000000e+00 +76 83 8.000000000000000e+00 +76 84 9.000000000000000e+00 +76 85 1.000000000000000e+01 +76 86 1.100000000000000e+01 +76 87 1.200000000000000e+01 +76 88 1.300000000000000e+01 +76 89 1.400000000000000e+01 +76 90 1.500000000000000e+01 +76 91 1.600000000000000e+01 +76 92 1.700000000000000e+01 +76 93 1.800000000000000e+01 +76 94 1.900000000000000e+01 +76 95 2.000000000000000e+01 +76 96 2.100000000000000e+01 +76 97 2.200000000000000e+01 +76 98 2.300000000000000e+01 +76 99 2.400000000000000e+01 +76 100 2.500000000000000e+01 +77 77 1.000000000000000e+00 +77 78 2.000000000000000e+00 +77 79 3.000000000000000e+00 +77 80 4.000000000000000e+00 +77 81 5.000000000000000e+00 +77 82 6.000000000000000e+00 +77 83 7.000000000000000e+00 +77 84 8.000000000000000e+00 +77 85 9.000000000000000e+00 +77 86 1.000000000000000e+01 +77 87 1.100000000000000e+01 +77 88 1.200000000000000e+01 +77 89 1.300000000000000e+01 +77 90 1.400000000000000e+01 +77 91 1.500000000000000e+01 +77 92 1.600000000000000e+01 +77 93 1.700000000000000e+01 +77 94 1.800000000000000e+01 +77 95 1.900000000000000e+01 +77 96 2.000000000000000e+01 +77 97 2.100000000000000e+01 +77 98 2.200000000000000e+01 +77 99 2.300000000000000e+01 +77 100 2.400000000000000e+01 +78 78 1.000000000000000e+00 +78 79 2.000000000000000e+00 +78 80 3.000000000000000e+00 +78 81 4.000000000000000e+00 +78 82 5.000000000000000e+00 +78 83 6.000000000000000e+00 +78 84 7.000000000000000e+00 +78 85 8.000000000000000e+00 +78 86 9.000000000000000e+00 +78 87 1.000000000000000e+01 +78 88 1.100000000000000e+01 +78 89 1.200000000000000e+01 +78 90 1.300000000000000e+01 +78 91 1.400000000000000e+01 +78 92 1.500000000000000e+01 +78 93 1.600000000000000e+01 +78 94 1.700000000000000e+01 +78 95 1.800000000000000e+01 +78 96 1.900000000000000e+01 +78 97 2.000000000000000e+01 +78 98 2.100000000000000e+01 +78 99 2.200000000000000e+01 +78 100 2.300000000000000e+01 +79 79 1.000000000000000e+00 +79 80 2.000000000000000e+00 +79 81 3.000000000000000e+00 +79 82 4.000000000000000e+00 +79 83 5.000000000000000e+00 +79 84 6.000000000000000e+00 +79 85 7.000000000000000e+00 +79 86 8.000000000000000e+00 +79 87 9.000000000000000e+00 +79 88 1.000000000000000e+01 +79 89 1.100000000000000e+01 +79 90 1.200000000000000e+01 +79 91 1.300000000000000e+01 +79 92 1.400000000000000e+01 +79 93 1.500000000000000e+01 +79 94 1.600000000000000e+01 +79 95 1.700000000000000e+01 +79 96 1.800000000000000e+01 +79 97 1.900000000000000e+01 +79 98 2.000000000000000e+01 +79 99 2.100000000000000e+01 +79 100 2.200000000000000e+01 +80 80 1.000000000000000e+00 +80 81 2.000000000000000e+00 +80 82 3.000000000000000e+00 +80 83 4.000000000000000e+00 +80 84 5.000000000000000e+00 +80 85 6.000000000000000e+00 +80 86 7.000000000000000e+00 +80 87 8.000000000000000e+00 +80 88 9.000000000000000e+00 +80 89 1.000000000000000e+01 +80 90 1.100000000000000e+01 +80 91 1.200000000000000e+01 +80 92 1.300000000000000e+01 +80 93 1.400000000000000e+01 +80 94 1.500000000000000e+01 +80 95 1.600000000000000e+01 +80 96 1.700000000000000e+01 +80 97 1.800000000000000e+01 +80 98 1.900000000000000e+01 +80 99 2.000000000000000e+01 +80 100 2.100000000000000e+01 +81 81 1.000000000000000e+00 +81 82 2.000000000000000e+00 +81 83 3.000000000000000e+00 +81 84 4.000000000000000e+00 +81 85 5.000000000000000e+00 +81 86 6.000000000000000e+00 +81 87 7.000000000000000e+00 +81 88 8.000000000000000e+00 +81 89 9.000000000000000e+00 +81 90 1.000000000000000e+01 +81 91 1.100000000000000e+01 +81 92 1.200000000000000e+01 +81 93 1.300000000000000e+01 +81 94 1.400000000000000e+01 +81 95 1.500000000000000e+01 +81 96 1.600000000000000e+01 +81 97 1.700000000000000e+01 +81 98 1.800000000000000e+01 +81 99 1.900000000000000e+01 +81 100 2.000000000000000e+01 +82 82 1.000000000000000e+00 +82 83 2.000000000000000e+00 +82 84 3.000000000000000e+00 +82 85 4.000000000000000e+00 +82 86 5.000000000000000e+00 +82 87 6.000000000000000e+00 +82 88 7.000000000000000e+00 +82 89 8.000000000000000e+00 +82 90 9.000000000000000e+00 +82 91 1.000000000000000e+01 +82 92 1.100000000000000e+01 +82 93 1.200000000000000e+01 +82 94 1.300000000000000e+01 +82 95 1.400000000000000e+01 +82 96 1.500000000000000e+01 +82 97 1.600000000000000e+01 +82 98 1.700000000000000e+01 +82 99 1.800000000000000e+01 +82 100 1.900000000000000e+01 +83 83 1.000000000000000e+00 +83 84 2.000000000000000e+00 +83 85 3.000000000000000e+00 +83 86 4.000000000000000e+00 +83 87 5.000000000000000e+00 +83 88 6.000000000000000e+00 +83 89 7.000000000000000e+00 +83 90 8.000000000000000e+00 +83 91 9.000000000000000e+00 +83 92 1.000000000000000e+01 +83 93 1.100000000000000e+01 +83 94 1.200000000000000e+01 +83 95 1.300000000000000e+01 +83 96 1.400000000000000e+01 +83 97 1.500000000000000e+01 +83 98 1.600000000000000e+01 +83 99 1.700000000000000e+01 +83 100 1.800000000000000e+01 +84 84 1.000000000000000e+00 +84 85 2.000000000000000e+00 +84 86 3.000000000000000e+00 +84 87 4.000000000000000e+00 +84 88 5.000000000000000e+00 +84 89 6.000000000000000e+00 +84 90 7.000000000000000e+00 +84 91 8.000000000000000e+00 +84 92 9.000000000000000e+00 +84 93 1.000000000000000e+01 +84 94 1.100000000000000e+01 +84 95 1.200000000000000e+01 +84 96 1.300000000000000e+01 +84 97 1.400000000000000e+01 +84 98 1.500000000000000e+01 +84 99 1.600000000000000e+01 +84 100 1.700000000000000e+01 +85 85 1.000000000000000e+00 +85 86 2.000000000000000e+00 +85 87 3.000000000000000e+00 +85 88 4.000000000000000e+00 +85 89 5.000000000000000e+00 +85 90 6.000000000000000e+00 +85 91 7.000000000000000e+00 +85 92 8.000000000000000e+00 +85 93 9.000000000000000e+00 +85 94 1.000000000000000e+01 +85 95 1.100000000000000e+01 +85 96 1.200000000000000e+01 +85 97 1.300000000000000e+01 +85 98 1.400000000000000e+01 +85 99 1.500000000000000e+01 +85 100 1.600000000000000e+01 +86 86 1.000000000000000e+00 +86 87 2.000000000000000e+00 +86 88 3.000000000000000e+00 +86 89 4.000000000000000e+00 +86 90 5.000000000000000e+00 +86 91 6.000000000000000e+00 +86 92 7.000000000000000e+00 +86 93 8.000000000000000e+00 +86 94 9.000000000000000e+00 +86 95 1.000000000000000e+01 +86 96 1.100000000000000e+01 +86 97 1.200000000000000e+01 +86 98 1.300000000000000e+01 +86 99 1.400000000000000e+01 +86 100 1.500000000000000e+01 +87 87 1.000000000000000e+00 +87 88 2.000000000000000e+00 +87 89 3.000000000000000e+00 +87 90 4.000000000000000e+00 +87 91 5.000000000000000e+00 +87 92 6.000000000000000e+00 +87 93 7.000000000000000e+00 +87 94 8.000000000000000e+00 +87 95 9.000000000000000e+00 +87 96 1.000000000000000e+01 +87 97 1.100000000000000e+01 +87 98 1.200000000000000e+01 +87 99 1.300000000000000e+01 +87 100 1.400000000000000e+01 +88 88 1.000000000000000e+00 +88 89 2.000000000000000e+00 +88 90 3.000000000000000e+00 +88 91 4.000000000000000e+00 +88 92 5.000000000000000e+00 +88 93 6.000000000000000e+00 +88 94 7.000000000000000e+00 +88 95 8.000000000000000e+00 +88 96 9.000000000000000e+00 +88 97 1.000000000000000e+01 +88 98 1.100000000000000e+01 +88 99 1.200000000000000e+01 +88 100 1.300000000000000e+01 +89 89 1.000000000000000e+00 +89 90 2.000000000000000e+00 +89 91 3.000000000000000e+00 +89 92 4.000000000000000e+00 +89 93 5.000000000000000e+00 +89 94 6.000000000000000e+00 +89 95 7.000000000000000e+00 +89 96 8.000000000000000e+00 +89 97 9.000000000000000e+00 +89 98 1.000000000000000e+01 +89 99 1.100000000000000e+01 +89 100 1.200000000000000e+01 +90 90 1.000000000000000e+00 +90 91 2.000000000000000e+00 +90 92 3.000000000000000e+00 +90 93 4.000000000000000e+00 +90 94 5.000000000000000e+00 +90 95 6.000000000000000e+00 +90 96 7.000000000000000e+00 +90 97 8.000000000000000e+00 +90 98 9.000000000000000e+00 +90 99 1.000000000000000e+01 +90 100 1.100000000000000e+01 +91 91 1.000000000000000e+00 +91 92 2.000000000000000e+00 +91 93 3.000000000000000e+00 +91 94 4.000000000000000e+00 +91 95 5.000000000000000e+00 +91 96 6.000000000000000e+00 +91 97 7.000000000000000e+00 +91 98 8.000000000000000e+00 +91 99 9.000000000000000e+00 +91 100 1.000000000000000e+01 +92 92 1.000000000000000e+00 +92 93 2.000000000000000e+00 +92 94 3.000000000000000e+00 +92 95 4.000000000000000e+00 +92 96 5.000000000000000e+00 +92 97 6.000000000000000e+00 +92 98 7.000000000000000e+00 +92 99 8.000000000000000e+00 +92 100 9.000000000000000e+00 +93 93 1.000000000000000e+00 +93 94 2.000000000000000e+00 +93 95 3.000000000000000e+00 +93 96 4.000000000000000e+00 +93 97 5.000000000000000e+00 +93 98 6.000000000000000e+00 +93 99 7.000000000000000e+00 +93 100 8.000000000000000e+00 +94 94 1.000000000000000e+00 +94 95 2.000000000000000e+00 +94 96 3.000000000000000e+00 +94 97 4.000000000000000e+00 +94 98 5.000000000000000e+00 +94 99 6.000000000000000e+00 +94 100 7.000000000000000e+00 +95 95 1.000000000000000e+00 +95 96 2.000000000000000e+00 +95 97 3.000000000000000e+00 +95 98 4.000000000000000e+00 +95 99 5.000000000000000e+00 +95 100 6.000000000000000e+00 +96 96 1.000000000000000e+00 +96 97 2.000000000000000e+00 +96 98 3.000000000000000e+00 +96 99 4.000000000000000e+00 +96 100 5.000000000000000e+00 +97 97 1.000000000000000e+00 +97 98 2.000000000000000e+00 +97 99 3.000000000000000e+00 +97 100 4.000000000000000e+00 +98 98 1.000000000000000e+00 +98 99 2.000000000000000e+00 +98 100 3.000000000000000e+00 +99 99 1.000000000000000e+00 +99 100 2.000000000000000e+00 +100 100 1.000000000000000e+00 From f8e2da22ba4d9d9d6b0bed5bf85fe50679ebcf8f Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Mon, 23 Aug 2021 14:07:13 -0600 Subject: [PATCH 20/70] Tpetra: Remove dead code from RowMatrixTransposer unit test --- .../RowMatrixTransposer/RowMatrixTransposer_UnitTests.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/tpetra/core/test/RowMatrixTransposer/RowMatrixTransposer_UnitTests.cpp b/packages/tpetra/core/test/RowMatrixTransposer/RowMatrixTransposer_UnitTests.cpp index 4146efe4b701..1e9449530477 100644 --- a/packages/tpetra/core/test/RowMatrixTransposer/RowMatrixTransposer_UnitTests.cpp +++ b/packages/tpetra/core/test/RowMatrixTransposer/RowMatrixTransposer_UnitTests.cpp @@ -74,10 +74,6 @@ namespace { typedef CrsMatrix<>::scalar_type Scalar; typedef CrsMatrix MAT; auto comm = Tpetra::getDefaultComm(); - int numProcs = comm->getSize(); - Tpetra::global_size_t numGlobal = 4*numProcs; - - auto rowMap = createUniformContigMapWithNode(numGlobal,comm); RCP matrix = Reader::readSparseFile("a.mtx", comm); RCP matrixT = Reader::readSparseFile("atrans.mtx", comm); From 0b001c3b927a69b054e535cbae037f423cb79115 Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Tue, 24 Aug 2021 08:32:57 -0600 Subject: [PATCH 21/70] Fixed up workset builder for UVM --- packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp | 5 +++-- packages/panzer/disc-fe/src/Panzer_Workset.cpp | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp b/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp index 911c2220e35d..e45c38db0bd1 100644 --- a/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp +++ b/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp @@ -1898,8 +1898,9 @@ getDivVectorBasis(const bool weighted, Kokkos::deep_copy(cell_div_basis_ref_host,cell_div_basis_ref.get_view()); using fst=Intrepid2::FunctionSpaceTools; - fst::HDIVtransformDIV(cell_div_basis_host,cell_jac_det_host,cell_div_basis_ref_host); - // PHX::Device().fence(); + fst::HDIVtransformDIV(cell_div_basis.get_view(),cell_jac_det.get_view(),cell_div_basis_ref.get_view()); + Kokkos::deep_copy(cell_div_basis_host, cell_div_basis.get_static_view()); + // Copy cell quantity back into main array for(int b=0; b workset_orientations(num_cells); // We can only apply orientations to owned and ghost cells - virtual cells are ignored (no orientations available) + auto local_cell_ids_host = Kokkos::create_mirror_view(local_cell_ids); + Kokkos::deep_copy(local_cell_ids_host, local_cell_ids); for(int i=0; i Date: Tue, 24 Aug 2021 14:42:52 -0600 Subject: [PATCH 22/70] Return the state of the ROL algorithm --- packages/piro/src/Piro_PerformAnalysis.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/piro/src/Piro_PerformAnalysis.cpp b/packages/piro/src/Piro_PerformAnalysis.cpp index f565ba63b125..6eae303d4aa8 100644 --- a/packages/piro/src/Piro_PerformAnalysis.cpp +++ b/packages/piro/src/Piro_PerformAnalysis.cpp @@ -596,6 +596,8 @@ Piro::PerformROLAnalysis( boundConstraint = rcp( new ROL::Bounds(ROL::makePtr >(p_lo), ROL::makePtr >(p_up))); } + int return_status = 0; + if ( useFullSpace ) { //ROL::Vector_SimOpt sopt_vec(ROL::makePtrFromRef(rol_x),ROL::makePtrFromRef(rol_p)); ROL::Vector_SimOpt sopt_vec(ROL::makePtrFromRef(rol_x_primal),ROL::makePtrFromRef(rol_p_primal)); @@ -608,23 +610,26 @@ Piro::PerformROLAnalysis( ROL::OptimizationProblem prob(ROL::makePtrFromRef(obj), ROL::makePtrFromRef(sopt_vec), bnd, ROL::makePtrFromRef(constr), r_ptr); ROL::OptimizationSolver optSolver(prob, rolParams.sublist("ROL Options")); optSolver.solve(*out); + return_status = optSolver.getAlgorithmState()->statusFlag; } else { ROL::OptimizationProblem prob(ROL::makePtrFromRef(obj), ROL::makePtrFromRef(sopt_vec), ROL::makePtrFromRef(constr), r_ptr); ROL::OptimizationSolver optSolver(prob, rolParams.sublist("ROL Options")); optSolver.solve(*out); + return_status = optSolver.getAlgorithmState()->statusFlag; } } else { if(boundConstrained) output = algo->run(rol_p_primal, reduced_obj, *boundConstraint, print, *out); else output = algo->run(rol_p_primal, reduced_obj, print, *out); + return_status = algo->getState()->statusFlag; } for ( unsigned i = 0; i < output.size(); i++ ) { *out << output[i]; } - return 0; + return return_status; #else (void)piroModel; (void)p; From a67b56eec524c44908cc1e91d8a41a15ec70417e Mon Sep 17 00:00:00 2001 From: Matthias Mayr Date: Wed, 25 Aug 2021 13:31:11 +0200 Subject: [PATCH 23/70] MueLu: collect region MG solver stuff in one file --- .../regionMG/src/SolveRegionHierarchy_def.hpp | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 packages/muelu/research/regionMG/src/SolveRegionHierarchy_def.hpp diff --git a/packages/muelu/research/regionMG/src/SolveRegionHierarchy_def.hpp b/packages/muelu/research/regionMG/src/SolveRegionHierarchy_def.hpp new file mode 100644 index 000000000000..bdee4ec493e2 --- /dev/null +++ b/packages/muelu/research/regionMG/src/SolveRegionHierarchy_def.hpp @@ -0,0 +1,204 @@ +// @HEADER +// +// *********************************************************************** +// +// MueLu: A package for multigrid based preconditioning +// Copyright 2012 Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact +// Jonathan Hu (jhu@sandia.gov) +// Ray Tuminaro (rstumin@sandia.gov) +// +// *********************************************************************** +// +// @HEADER +#ifndef MUELU_SOLVEREGIONHIERARCHY_DEF_HPP +#define MUELU_SOLVEREGIONHIERARCHY_DEF_HPP + +#include "SetupRegionHierarchy_def.hpp" + +using Teuchos::RCP; +using Teuchos::rcp; +using Teuchos::Array; + +template +void solveRegionProblem(const double tol, const bool scaleResidualHist, const int maxIts, + const std::string cycleType, const std::string convergenceLog, + RCP& coarseSolverData, + Array >& smootherParams, + RCP hierarchyData, + RCP > & regHierarchy, + RCP >& X, + RCP >& B) { + + using LO = LocalOrdinal; + using GO = GlobalOrdinal; + using NO = Node; + using SC = Scalar; + + using Map = Xpetra::Map; + using Import = Xpetra::Import; + using Matrix = Xpetra::Matrix; + using Vector = Xpetra::Vector; + using VectorFactory = Xpetra::VectorFactory; + + using Level = MueLu::Level; + + using STS = Teuchos::ScalarTraits; + using magnitude_type = typename STS::magnitudeType; + // const Scalar zero = STS::zero(); + const Scalar one = STS::one(); + + // we start by extracting some basic data from the hierarchy + const int numLevels = regHierarchy->GetNumLevels(); + RCP level0 = regHierarchy->GetLevel(0); + RCP regMat = level0->Get >("A"); + RCP revisedRowMap = regMat->getRowMap(); + RCP rowImport = level0->Get >("rowImport"); + RCP dofMap = X->getMap(); + const int myRank = dofMap->getComm()->getRank(); + + // Instead of checking each time for rank, create a rank 0 stream + RCP fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); + Teuchos::FancyOStream& out = *fancy; + out.setOutputToRootOnly(0); + + TEUCHOS_TEST_FOR_EXCEPT_MSG(!(numLevels>0), "We require numLevel > 0. Probably, numLevel has not been set, yet."); + + // We first use the non-level container variables to setup the fine grid problem. + // This is ok since the initial setup just mimics the application and the outer + // Krylov method. + // + // We switch to using the level container variables as soon as we enter the + // recursive part of the algorithm. + // + + // Composite residual vector + RCP compRes = VectorFactory::Build(dofMap, true); + + // transform composite vectors to regional layout + Teuchos::RCP quasiRegX; + Teuchos::RCP regX; + compositeToRegional(X, quasiRegX, regX, + revisedRowMap, rowImport); + + RCP quasiRegB; + RCP regB; + compositeToRegional(B, quasiRegB, regB, + revisedRowMap, rowImport); + + RCP regRes; + regRes = VectorFactory::Build(revisedRowMap, true); + + ///////////////////////////////////////////////////////////////////////// + // SWITCH TO RECURSIVE STYLE --> USE LEVEL CONTAINER VARIABLES + ///////////////////////////////////////////////////////////////////////// + + // Prepare output of residual norm to file + RCP log; + if (myRank == 0) + { + log = rcp(new std::ofstream(convergenceLog.c_str())); + (*log) << "# num procs = " << dofMap->getComm()->getSize() << "\n" + << "# iteration | res-norm (scaled=" << scaleResidualHist << ")\n" + << "#\n"; + *log << std::setprecision(16) << std::scientific; + } + + // Print type of residual norm to the screen + if (scaleResidualHist) + out << "Using scaled residual norm." << std::endl; + else + out << "Using unscaled residual norm." << std::endl; + + + // Richardson iterations + magnitude_type normResIni = Teuchos::ScalarTraits::zero(); + const int old_precision = std::cout.precision(); + std::cout << std::setprecision(8) << std::scientific; + int cycle = 0; + + Teuchos::RCP regCorrect; + regCorrect = VectorFactory::Build(revisedRowMap, true); + for (cycle = 0; cycle < maxIts; ++cycle) + { + const Scalar SC_ZERO = Teuchos::ScalarTraits::zero(); + regCorrect->putScalar(SC_ZERO); + // Get Stuff out of Hierarchy + RCP level = regHierarchy->GetLevel(0); + RCP regInterfaceScalings = level->Get >("regInterfaceScalings"); + // check for convergence + { + //////////////////////////////////////////////////////////////////////// + // SWITCH BACK TO NON-LEVEL VARIABLES + //////////////////////////////////////////////////////////////////////// + computeResidual(regRes, regX, regB, regMat, *smootherParams[0]); + scaleInterfaceDOFs(regRes, regInterfaceScalings, true); + + compRes = VectorFactory::Build(dofMap, true); + regionalToComposite(regRes, compRes, rowImport); + + typename Teuchos::ScalarTraits::magnitudeType normRes = compRes->norm2(); + if(cycle == 0) { normResIni = normRes; } + + if (scaleResidualHist) + normRes /= normResIni; + + // Output current residual norm to screen (on proc 0 only) + out << cycle << "\t" << normRes << std::endl; + if (myRank == 0) + (*log) << cycle << "\t" << normRes << "\n"; + + if (normRes < tol) + break; + } + + ///////////////////////////////////////////////////////////////////////// + // SWITCH TO RECURSIVE STYLE --> USE LEVEL CONTAINER VARIABLES + ///////////////////////////////////////////////////////////////////////// + + bool zeroInitGuess = true; + scaleInterfaceDOFs(regRes, regInterfaceScalings, false); + vCycle(0, numLevels, cycleType, regHierarchy, + regCorrect, regRes, + smootherParams, zeroInitGuess, coarseSolverData, hierarchyData); + + regX->update(one, *regCorrect, one); + } + out << "Number of iterations performed for this solve: " << cycle << std::endl; + + std::cout << std::setprecision(old_precision); + std::cout.unsetf(std::ios::fixed | std::ios::scientific); +} + +#endif // MUELU_SOLVEREGIONHIERARCHY_DEF_HPP From bac93aaa7c8d95e5e6ea67e9689fba9c6ff6c21e Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Wed, 25 Aug 2021 08:32:46 -0600 Subject: [PATCH 24/70] MueLu: Re-fix ML2MueLuTranslator --- .../MueLu_ML2MueLuParameterTranslator.cpp | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/muelu/src/Interface/MueLu_ML2MueLuParameterTranslator.cpp b/packages/muelu/src/Interface/MueLu_ML2MueLuParameterTranslator.cpp index c21497a9a502..3a9e875e807c 100644 --- a/packages/muelu/src/Interface/MueLu_ML2MueLuParameterTranslator.cpp +++ b/packages/muelu/src/Interface/MueLu_ML2MueLuParameterTranslator.cpp @@ -46,9 +46,12 @@ #include "MueLu_ConfigDefs.hpp" #if defined(HAVE_MUELU_ML) -#include -#include // for default values -#include +# include +# if defined(HAVE_ML_EPETRA) && defined(HAVE_ML_TEUCHOS) +# include +# include // for default values +# include +# endif #endif #include @@ -198,7 +201,7 @@ namespace MueLu { RCP out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); // TODO: use internal out (GetOStream()) -#if defined(HAVE_MUELU_ML) +#if defined(HAVE_MUELU_ML) && defined(HAVE_ML_EPETRA) && defined(HAVE_ML_TEUCHOS) // TODO alternative with standard parameterlist from ML user guide? @@ -218,9 +221,9 @@ namespace MueLu { #else if (defaultVals != "") { // If no validator available: issue a warning and set parameter value to false in the output list - *out << "Warning: MueLu_ENABLE_ML=OFF and/or MueLu_ENABLE_Epetra=OFF. No ML default values available." << std::endl; + *out << "Warning: MueLu_ENABLE_ML=OFF, ML_ENABLE_Epetra=OFF or ML_ENABLE_TEUCHOS=OFF. No ML default values available." << std::endl; } -#endif // HAVE_MUELU_ML +#endif // HAVE_MUELU_ML && HAVE_ML_EPETRA && HAVE_ML_TEUCHOS // // Move smoothers/aggregation/coarse parameters to sublists @@ -240,17 +243,17 @@ namespace MueLu { bool validate = paramList.get("ML validate parameter list", true); /* true = default in ML */ if (validate && defaultVals!="refmaxwell") { -#if defined(HAVE_MUELU_ML) +#if defined(HAVE_MUELU_ML) && defined(HAVE_ML_EPETRA) && defined(HAVE_ML_TEUCHOS) // Validate parameter list using ML validator int depth = paramList.get("ML validate depth", 5); /* 5 = default in ML */ TEUCHOS_TEST_FOR_EXCEPTION(! ML_Epetra::ValidateMLPParameters(paramList, depth), Exceptions::RuntimeError, "ERROR: ML's Teuchos::ParameterList contains incorrect parameter!"); #else // If no validator available: issue a warning and set parameter value to false in the output list - *out << "Warning: MueLu_ENABLE_ML=OFF and/or MueLu_ENABLE_Epetra=OFF. The parameter list cannot be validated." << std::endl; + *out << "Warning: MueLu_ENABLE_ML=OFF, ML_ENABLE_Epetra=OFF or ML_ENABLE_TEUCHOS=OFF. The parameter list cannot be validated." << std::endl; paramList.set("ML validate parameter list", false); -#endif // HAVE_MUELU_ML +#endif // HAVE_MUELU_ML && HAVE_ML_EPETRA && HAVE_ML_TEUCHOS } // if(validate) } // scope From 7f7cdbec118814477f0428941d8590fa003a5052 Mon Sep 17 00:00:00 2001 From: Matthias Mayr Date: Wed, 25 Aug 2021 16:49:46 +0200 Subject: [PATCH 25/70] MueLu: move region MG vCycle to "solve" file A vCycle is related to solving a problem, not to setup of the MG hierarchy. So, let's move the region MG vCycle function from the "setup" file to the "solve" file. --- .../regionMG/src/SetupRegionHierarchy_def.hpp | 259 ------------------ .../regionMG/src/SolveRegionHierarchy_def.hpp | 258 +++++++++++++++++ .../structured/Driver_Structured_Regions.cpp | 3 +- 3 files changed, 260 insertions(+), 260 deletions(-) diff --git a/packages/muelu/research/regionMG/src/SetupRegionHierarchy_def.hpp b/packages/muelu/research/regionMG/src/SetupRegionHierarchy_def.hpp index 56e145157a50..8d8f9669b36d 100644 --- a/packages/muelu/research/regionMG/src/SetupRegionHierarchy_def.hpp +++ b/packages/muelu/research/regionMG/src/SetupRegionHierarchy_def.hpp @@ -930,263 +930,4 @@ void createRegionHierarchy(const int numDimensions, } // createRegionHierarchy - -//! Recursive V-cycle in region fashion -template -void vCycle(const int l, ///< ID of current level - const int numLevels, ///< Total number of levels - const std::string cycleType, - RCP > & regHierarchy, - RCP >& fineRegX, ///< solution - RCP > fineRegB, ///< right hand side - Array > smootherParams, ///< region smoother parameter list - bool& zeroInitGuess, - RCP coarseSolverData = Teuchos::null, - RCP hierarchyData = Teuchos::null) -{ -#include "MueLu_UseShortNames.hpp" - using Teuchos::TimeMonitor; - const Scalar SC_ZERO = Teuchos::ScalarTraits::zero(); - const Scalar SC_ONE = Teuchos::ScalarTraits::one(); - - RCP level = regHierarchy->GetLevel(l); - RCP regMatrix = level->Get >("A", MueLu::NoFactory::get()); - RCP regRowMap = regMatrix->getRowMap(); - RCP > regRowImporter = level->Get > >("rowImport"); - RCP > regInterfaceScalings = level->Get > >("regInterfaceScalings"); - - int cycleCount = 1; - if(cycleType == "W" && l > 0) // W cycle and not on finest level - cycleCount=2; - if (l < numLevels - 1) { // fine or intermediate levels - for(int cycle=0; cycle < cycleCount; cycle++){ - -// std::cout << "level: " << l << std::endl; - - // extract data from hierarchy parameterlist - std::string levelName("level" + std::to_string(l)); - ParameterList levelList; - bool useCachedVectors = false; - // if(Teuchos::nonnull(hierarchyData) && hierarchyData->isSublist(levelName)) { - // levelList = hierarchyData->sublist(levelName); - // if(levelList.isParameter("residual") && levelList.isParameter("solution")) { - // useCachedVectors = true; - // } - // } - - RCP tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 1 - pre-smoother"))); - - // pre-smoothing - smootherApply(smootherParams[l], fineRegX, fineRegB, regMatrix, - regRowMap, regRowImporter, zeroInitGuess); - - tm = Teuchos::null; - tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 2 - compute residual"))); - - RCP regRes; - if(useCachedVectors) { - regRes = levelList.get >("residual"); - } else { - regRes = VectorFactory::Build(regRowMap, true); - } - computeResidual(regRes, fineRegX, fineRegB, regMatrix, *smootherParams[l]); - - tm = Teuchos::null; - tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 3 - scale interface"))); - - scaleInterfaceDOFs(regRes, regInterfaceScalings, true); - - tm = Teuchos::null; - tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 4 - create coarse vectors"))); - - // Transfer to coarse level - RCP coarseRegX; - RCP coarseRegB; - - { - RCP levelCoarse = regHierarchy->GetLevel(l+1); - RCP regProlongCoarse = levelCoarse->Get >("P", MueLu::NoFactory::get()); - RCP > regRowMapCoarse = regProlongCoarse->getColMap(); - // Get pre-communicated communication patterns for the fast MatVec - const ArrayRCP regionInterfaceLIDs = smootherParams[l+1]->get>("Fast MatVec: interface LIDs"); - const RCP regionInterfaceImporter = smootherParams[l+1]->get>("Fast MatVec: interface importer"); - - coarseRegX = VectorFactory::Build(regRowMapCoarse, true); - coarseRegB = VectorFactory::Build(regRowMapCoarse, true); - - regProlongCoarse->apply(*regRes, *coarseRegB, Teuchos::TRANS, SC_ONE, SC_ZERO, true, regionInterfaceImporter, regionInterfaceLIDs); - // TEUCHOS_ASSERT(regProlong[l+1][j]->getRangeMap()->isSameAs(*regRes[j]->getMap())); - // TEUCHOS_ASSERT(regProlong[l+1][j]->getDomainMap()->isSameAs(*coarseRegB[j]->getMap())); - } - - tm = Teuchos::null; - bool coarseZeroInitGuess = true; - - // Call V-cycle recursively - vCycle(l+1, numLevels, cycleType, regHierarchy, - coarseRegX, coarseRegB, - smootherParams, coarseZeroInitGuess, coarseSolverData, hierarchyData); - - tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 6 - transfer coarse to fine"))); - - // Transfer coarse level correction to fine level - RCP regCorrection; - { - RCP levelCoarse = regHierarchy->GetLevel(l+1); - RCP regProlongCoarse = levelCoarse->Get >("P", MueLu::NoFactory::get()); - // Get pre-communicated communication patterns for the fast MatVec - const ArrayRCP regionInterfaceLIDs = smootherParams[l]->get>("Fast MatVec: interface LIDs"); - const RCP regionInterfaceImporter = smootherParams[l]->get>("Fast MatVec: interface importer"); - - regCorrection = VectorFactory::Build(regRowMap, true); - regProlongCoarse->apply(*coarseRegX, *regCorrection, Teuchos::NO_TRANS, SC_ONE, SC_ZERO, false, regionInterfaceImporter, regionInterfaceLIDs); - // TEUCHOS_ASSERT(regProlong[l+1][j]->getDomainMap()->isSameAs(*coarseRegX[j]->getMap())); - // TEUCHOS_ASSERT(regProlong[l+1][j]->getRangeMap()->isSameAs(*regCorrection[j]->getMap())); - } - - tm = Teuchos::null; - tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 7 - add coarse grid correction"))); - - // apply coarse grid correction - fineRegX->update(SC_ONE, *regCorrection, SC_ONE); - if (coarseZeroInitGuess) zeroInitGuess = true; - -// std::cout << "level: " << l << std::endl; - - tm = Teuchos::null; - tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 8 - post-smoother"))); - - // post-smoothing - smootherApply(smootherParams[l], fineRegX, fineRegB, regMatrix, - regRowMap, regRowImporter, zeroInitGuess); - - tm = Teuchos::null; - - } - } else { - - // Coarsest grid solve - - RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); - fos->setOutputToRootOnly(0); - - RCP tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: * - coarsest grid solve"))); - - // std::cout << "Applying coarse colver" << std::endl; - - const std::string coarseSolverType = coarseSolverData->get("coarse solver type"); - if (coarseSolverType == "smoother") { - smootherApply(smootherParams[l], fineRegX, fineRegB, regMatrix, - regRowMap, regRowImporter, zeroInitGuess); - } else { - zeroInitGuess = false; - // First get the Xpetra vectors from region to composite format - RCP coarseRowMap = coarseSolverData->get >("compCoarseRowMap"); - RCP compX = VectorFactory::Build(coarseRowMap, true); - RCP compRhs = VectorFactory::Build(coarseRowMap, true); - { - RCP inverseInterfaceScaling = VectorFactory::Build(regInterfaceScalings->getMap()); - inverseInterfaceScaling->reciprocal(*regInterfaceScalings); - fineRegB->elementWiseMultiply(SC_ONE, *fineRegB, *inverseInterfaceScaling, SC_ZERO); - - regionalToComposite(fineRegB, compRhs, regRowImporter); - } - - if (coarseSolverType == "direct") - { -#if defined(HAVE_MUELU_TPETRA) && defined(HAVE_MUELU_AMESOS2) - - using DirectCoarseSolver = Amesos2::Solver, Tpetra::MultiVector >; - RCP coarseSolver = coarseSolverData->get >("direct solver object"); - - TEUCHOS_TEST_FOR_EXCEPT_MSG(coarseRowMap->lib()!=Xpetra::UseTpetra, - "Coarse solver requires Tpetra/Amesos2 stack."); - TEUCHOS_ASSERT(!coarseSolver.is_null()); - - // using Utilities = MueLu::Utilities; - - // From here on we switch to Tpetra for simplicity - // we could also implement a similar Epetra branch - using Tpetra_MultiVector = Tpetra::MultiVector; - - // *fos << "Attempting to use Amesos2 to solve the coarse grid problem" << std::endl; - RCP tX = Utilities::MV2NonConstTpetraMV2(*compX); - RCP tB = Utilities::MV2TpetraMV(compRhs); - - /* Solve! - * - * Calling solve() on the coarseSolver should just do a triangular solve, since symbolic - * and numeric factorization are supposed to have happened during hierarchy setup. - * Here, we just check if they're done and print message if not. - * - * We don't have to change the map of tX and tB since we have configured the Amesos2 solver - * during its construction to work with non-continuous maps. - */ - if (not coarseSolver->getStatus().symbolicFactorizationDone()) - *fos << "Symbolic factorization should have been done during hierarchy setup, " - "but actually is missing. Anyway ... just do it right now." << std::endl; - if (not coarseSolver->getStatus().numericFactorizationDone()) - *fos << "Numeric factorization should have been done during hierarchy setup, " - "but actually is missing. Anyway ... just do it right now." << std::endl; - coarseSolver->solve(tX.ptr(), tB.ptr()); -#else - *fos << "+++++++++++++++++++++++++++ WARNING +++++++++++++++++++++++++\n" - << "+ Coarse level direct solver requires Tpetra and Amesos2. +\n" - << "+ Skipping the coarse level solve. +\n" - << "+++++++++++++++++++++++++++ WARNING +++++++++++++++++++++++++" - << std::endl; -#endif - } - else if (coarseSolverType == "amg") // use AMG as coarse level solver - { - const bool coarseSolverRebalance = coarseSolverData->get("coarse solver rebalance"); - - // Extract the hierarchy from the coarseSolverData - RCP amgHierarchy = coarseSolverData->get>("amg hierarchy object"); - - // Run a single V-cycle - if(coarseSolverRebalance==false){ - amgHierarchy->Iterate(*compRhs, *compX, 1, true); - - } else { -#if defined(HAVE_MUELU_ZOLTAN2) && defined(HAVE_MPI) - RCP rebalanceImporter = coarseSolverData->get >("rebalanceImporter"); - - // TODO: These vectors could be cached to improve performance - RCP rebalancedRhs = VectorFactory::Build(rebalanceImporter->getTargetMap()); - RCP rebalancedX = VectorFactory::Build(rebalanceImporter->getTargetMap(), true); - rebalancedRhs->doImport(*compRhs, *rebalanceImporter, Xpetra::INSERT); - - rebalancedRhs->replaceMap(rebalancedRhs->getMap()->removeEmptyProcesses()); - rebalancedX->replaceMap(rebalancedX->getMap()->removeEmptyProcesses()); - - if(!amgHierarchy.is_null()){ - amgHierarchy->Iterate(*rebalancedRhs, *rebalancedX, 1, true); - } - - rebalancedX->replaceMap(rebalanceImporter->getTargetMap()); - compX->doExport(*rebalancedX, *rebalanceImporter, Xpetra::INSERT); -#else - amgHierarchy->Iterate(*compRhs, *compX, 1, true); -#endif - } - } - else - { - TEUCHOS_TEST_FOR_EXCEPT_MSG(false, "Unknown coarse solver type."); - } - - // Transform back to region format - RCP quasiRegX; - compositeToRegional(compX, quasiRegX, fineRegX, - regRowMap, - regRowImporter); - - tm = Teuchos::null; - } - } - - return; -} // vCycle - #endif // MUELU_SETUPREGIONHIERARCHY_DEF_HPP diff --git a/packages/muelu/research/regionMG/src/SolveRegionHierarchy_def.hpp b/packages/muelu/research/regionMG/src/SolveRegionHierarchy_def.hpp index bdee4ec493e2..ac5b9a72b663 100644 --- a/packages/muelu/research/regionMG/src/SolveRegionHierarchy_def.hpp +++ b/packages/muelu/research/regionMG/src/SolveRegionHierarchy_def.hpp @@ -51,6 +51,264 @@ using Teuchos::RCP; using Teuchos::rcp; using Teuchos::Array; +//! Recursive V-cycle in region fashion +template +void vCycle(const int l, ///< ID of current level + const int numLevels, ///< Total number of levels + const std::string cycleType, + RCP > & regHierarchy, + RCP >& fineRegX, ///< solution + RCP > fineRegB, ///< right hand side + Array > smootherParams, ///< region smoother parameter list + bool& zeroInitGuess, + RCP coarseSolverData = Teuchos::null, + RCP hierarchyData = Teuchos::null) +{ +#include "MueLu_UseShortNames.hpp" + using Teuchos::TimeMonitor; + const Scalar SC_ZERO = Teuchos::ScalarTraits::zero(); + const Scalar SC_ONE = Teuchos::ScalarTraits::one(); + + RCP level = regHierarchy->GetLevel(l); + RCP regMatrix = level->Get >("A", MueLu::NoFactory::get()); + RCP regRowMap = regMatrix->getRowMap(); + RCP > regRowImporter = level->Get > >("rowImport"); + RCP > regInterfaceScalings = level->Get > >("regInterfaceScalings"); + + int cycleCount = 1; + if(cycleType == "W" && l > 0) // W cycle and not on finest level + cycleCount=2; + if (l < numLevels - 1) { // fine or intermediate levels + for(int cycle=0; cycle < cycleCount; cycle++){ + +// std::cout << "level: " << l << std::endl; + + // extract data from hierarchy parameterlist + std::string levelName("level" + std::to_string(l)); + ParameterList levelList; + bool useCachedVectors = false; + // if(Teuchos::nonnull(hierarchyData) && hierarchyData->isSublist(levelName)) { + // levelList = hierarchyData->sublist(levelName); + // if(levelList.isParameter("residual") && levelList.isParameter("solution")) { + // useCachedVectors = true; + // } + // } + + RCP tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 1 - pre-smoother"))); + + // pre-smoothing + smootherApply(smootherParams[l], fineRegX, fineRegB, regMatrix, + regRowMap, regRowImporter, zeroInitGuess); + + tm = Teuchos::null; + tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 2 - compute residual"))); + + RCP regRes; + if(useCachedVectors) { + regRes = levelList.get >("residual"); + } else { + regRes = VectorFactory::Build(regRowMap, true); + } + computeResidual(regRes, fineRegX, fineRegB, regMatrix, *smootherParams[l]); + + tm = Teuchos::null; + tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 3 - scale interface"))); + + scaleInterfaceDOFs(regRes, regInterfaceScalings, true); + + tm = Teuchos::null; + tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 4 - create coarse vectors"))); + + // Transfer to coarse level + RCP coarseRegX; + RCP coarseRegB; + + { + RCP levelCoarse = regHierarchy->GetLevel(l+1); + RCP regProlongCoarse = levelCoarse->Get >("P", MueLu::NoFactory::get()); + RCP > regRowMapCoarse = regProlongCoarse->getColMap(); + // Get pre-communicated communication patterns for the fast MatVec + const ArrayRCP regionInterfaceLIDs = smootherParams[l+1]->get>("Fast MatVec: interface LIDs"); + const RCP regionInterfaceImporter = smootherParams[l+1]->get>("Fast MatVec: interface importer"); + + coarseRegX = VectorFactory::Build(regRowMapCoarse, true); + coarseRegB = VectorFactory::Build(regRowMapCoarse, true); + + regProlongCoarse->apply(*regRes, *coarseRegB, Teuchos::TRANS, SC_ONE, SC_ZERO, true, regionInterfaceImporter, regionInterfaceLIDs); + // TEUCHOS_ASSERT(regProlong[l+1][j]->getRangeMap()->isSameAs(*regRes[j]->getMap())); + // TEUCHOS_ASSERT(regProlong[l+1][j]->getDomainMap()->isSameAs(*coarseRegB[j]->getMap())); + } + + tm = Teuchos::null; + bool coarseZeroInitGuess = true; + + // Call V-cycle recursively + vCycle(l+1, numLevels, cycleType, regHierarchy, + coarseRegX, coarseRegB, + smootherParams, coarseZeroInitGuess, coarseSolverData, hierarchyData); + + tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 6 - transfer coarse to fine"))); + + // Transfer coarse level correction to fine level + RCP regCorrection; + { + RCP levelCoarse = regHierarchy->GetLevel(l+1); + RCP regProlongCoarse = levelCoarse->Get >("P", MueLu::NoFactory::get()); + // Get pre-communicated communication patterns for the fast MatVec + const ArrayRCP regionInterfaceLIDs = smootherParams[l]->get>("Fast MatVec: interface LIDs"); + const RCP regionInterfaceImporter = smootherParams[l]->get>("Fast MatVec: interface importer"); + + regCorrection = VectorFactory::Build(regRowMap, true); + regProlongCoarse->apply(*coarseRegX, *regCorrection, Teuchos::NO_TRANS, SC_ONE, SC_ZERO, false, regionInterfaceImporter, regionInterfaceLIDs); + // TEUCHOS_ASSERT(regProlong[l+1][j]->getDomainMap()->isSameAs(*coarseRegX[j]->getMap())); + // TEUCHOS_ASSERT(regProlong[l+1][j]->getRangeMap()->isSameAs(*regCorrection[j]->getMap())); + } + + tm = Teuchos::null; + tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 7 - add coarse grid correction"))); + + // apply coarse grid correction + fineRegX->update(SC_ONE, *regCorrection, SC_ONE); + if (coarseZeroInitGuess) zeroInitGuess = true; + +// std::cout << "level: " << l << std::endl; + + tm = Teuchos::null; + tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: 8 - post-smoother"))); + + // post-smoothing + smootherApply(smootherParams[l], fineRegX, fineRegB, regMatrix, + regRowMap, regRowImporter, zeroInitGuess); + + tm = Teuchos::null; + + } + } else { + + // Coarsest grid solve + + RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); + fos->setOutputToRootOnly(0); + + RCP tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("vCycle: * - coarsest grid solve"))); + + // std::cout << "Applying coarse colver" << std::endl; + + const std::string coarseSolverType = coarseSolverData->get("coarse solver type"); + if (coarseSolverType == "smoother") { + smootherApply(smootherParams[l], fineRegX, fineRegB, regMatrix, + regRowMap, regRowImporter, zeroInitGuess); + } else { + zeroInitGuess = false; + // First get the Xpetra vectors from region to composite format + RCP coarseRowMap = coarseSolverData->get >("compCoarseRowMap"); + RCP compX = VectorFactory::Build(coarseRowMap, true); + RCP compRhs = VectorFactory::Build(coarseRowMap, true); + { + RCP inverseInterfaceScaling = VectorFactory::Build(regInterfaceScalings->getMap()); + inverseInterfaceScaling->reciprocal(*regInterfaceScalings); + fineRegB->elementWiseMultiply(SC_ONE, *fineRegB, *inverseInterfaceScaling, SC_ZERO); + + regionalToComposite(fineRegB, compRhs, regRowImporter); + } + + if (coarseSolverType == "direct") + { +#if defined(HAVE_MUELU_TPETRA) && defined(HAVE_MUELU_AMESOS2) + + using DirectCoarseSolver = Amesos2::Solver, Tpetra::MultiVector >; + RCP coarseSolver = coarseSolverData->get >("direct solver object"); + + TEUCHOS_TEST_FOR_EXCEPT_MSG(coarseRowMap->lib()!=Xpetra::UseTpetra, + "Coarse solver requires Tpetra/Amesos2 stack."); + TEUCHOS_ASSERT(!coarseSolver.is_null()); + + // using Utilities = MueLu::Utilities; + + // From here on we switch to Tpetra for simplicity + // we could also implement a similar Epetra branch + using Tpetra_MultiVector = Tpetra::MultiVector; + + // *fos << "Attempting to use Amesos2 to solve the coarse grid problem" << std::endl; + RCP tX = Utilities::MV2NonConstTpetraMV2(*compX); + RCP tB = Utilities::MV2TpetraMV(compRhs); + + /* Solve! + * + * Calling solve() on the coarseSolver should just do a triangular solve, since symbolic + * and numeric factorization are supposed to have happened during hierarchy setup. + * Here, we just check if they're done and print message if not. + * + * We don't have to change the map of tX and tB since we have configured the Amesos2 solver + * during its construction to work with non-continuous maps. + */ + if (not coarseSolver->getStatus().symbolicFactorizationDone()) + *fos << "Symbolic factorization should have been done during hierarchy setup, " + "but actually is missing. Anyway ... just do it right now." << std::endl; + if (not coarseSolver->getStatus().numericFactorizationDone()) + *fos << "Numeric factorization should have been done during hierarchy setup, " + "but actually is missing. Anyway ... just do it right now." << std::endl; + coarseSolver->solve(tX.ptr(), tB.ptr()); +#else + *fos << "+++++++++++++++++++++++++++ WARNING +++++++++++++++++++++++++\n" + << "+ Coarse level direct solver requires Tpetra and Amesos2. +\n" + << "+ Skipping the coarse level solve. +\n" + << "+++++++++++++++++++++++++++ WARNING +++++++++++++++++++++++++" + << std::endl; +#endif + } + else if (coarseSolverType == "amg") // use AMG as coarse level solver + { + const bool coarseSolverRebalance = coarseSolverData->get("coarse solver rebalance"); + + // Extract the hierarchy from the coarseSolverData + RCP amgHierarchy = coarseSolverData->get>("amg hierarchy object"); + + // Run a single V-cycle + if(coarseSolverRebalance==false){ + amgHierarchy->Iterate(*compRhs, *compX, 1, true); + + } else { +#if defined(HAVE_MUELU_ZOLTAN2) && defined(HAVE_MPI) + RCP rebalanceImporter = coarseSolverData->get >("rebalanceImporter"); + + // TODO: These vectors could be cached to improve performance + RCP rebalancedRhs = VectorFactory::Build(rebalanceImporter->getTargetMap()); + RCP rebalancedX = VectorFactory::Build(rebalanceImporter->getTargetMap(), true); + rebalancedRhs->doImport(*compRhs, *rebalanceImporter, Xpetra::INSERT); + + rebalancedRhs->replaceMap(rebalancedRhs->getMap()->removeEmptyProcesses()); + rebalancedX->replaceMap(rebalancedX->getMap()->removeEmptyProcesses()); + + if(!amgHierarchy.is_null()){ + amgHierarchy->Iterate(*rebalancedRhs, *rebalancedX, 1, true); + } + + rebalancedX->replaceMap(rebalanceImporter->getTargetMap()); + compX->doExport(*rebalancedX, *rebalanceImporter, Xpetra::INSERT); +#else + amgHierarchy->Iterate(*compRhs, *compX, 1, true); +#endif + } + } + else + { + TEUCHOS_TEST_FOR_EXCEPT_MSG(false, "Unknown coarse solver type."); + } + + // Transform back to region format + RCP quasiRegX; + compositeToRegional(compX, quasiRegX, fineRegX, + regRowMap, + regRowImporter); + + tm = Teuchos::null; + } + } + + return; +} // vCycle + template void solveRegionProblem(const double tol, const bool scaleResidualHist, const int maxIts, const std::string cycleType, const std::string convergenceLog, diff --git a/packages/muelu/research/regionMG/test/structured/Driver_Structured_Regions.cpp b/packages/muelu/research/regionMG/test/structured/Driver_Structured_Regions.cpp index a50a1c8d5e7e..b7885288bfb9 100644 --- a/packages/muelu/research/regionMG/test/structured/Driver_Structured_Regions.cpp +++ b/packages/muelu/research/regionMG/test/structured/Driver_Structured_Regions.cpp @@ -128,6 +128,7 @@ #include "SetupRegionVector_def.hpp" #include "SetupRegionMatrix_def.hpp" #include "SetupRegionHierarchy_def.hpp" +#include "SolveRegionHierarchy_def.hpp" template @@ -700,7 +701,7 @@ int main_(Teuchos::CommandLineProcessor &clp, Xpetra::UnderlyingLib& lib, int ar keepCoarseCoords); hierarchyData->print(); - + comm->barrier(); From b609a041ee4865d77ef0be8b880d342b94fa4a4a Mon Sep 17 00:00:00 2001 From: Jacob Domagala Date: Wed, 25 Aug 2021 19:18:29 +0200 Subject: [PATCH 26/70] Fix Panzer's projection and ip_coordinates tests with UVM off (#9614) * Fix Panzer's projection test with UVM off * Extract views from MDField to avoid CUDA warnings * Remove no longer needed comments * Fix Panzer's ip_coordinates test with UVM off --- .../test/projection/projection.cpp | 135 ++++++++++++------ .../disc-fe/src/Panzer_L2Projection.cpp | 19 ++- .../panzer/disc-fe/src/Panzer_Workset.cpp | 7 +- ...nseScatterEvaluator_IPCoordinates_impl.hpp | 5 +- 4 files changed, 112 insertions(+), 54 deletions(-) diff --git a/packages/panzer/adapters-stk/test/projection/projection.cpp b/packages/panzer/adapters-stk/test/projection/projection.cpp index 23e1c059866a..5651fe7139e3 100644 --- a/packages/panzer/adapters-stk/test/projection/projection.cpp +++ b/packages/panzer/adapters-stk/test/projection/projection.cpp @@ -339,22 +339,27 @@ TEUCHOS_UNIT_TEST(L2Projection, ToNodal) const auto cellLocalIdsNoGhost = Kokkos::subview(workset.cell_local_ids_k,std::make_pair(0,workset.numOwnedCells())); sourceGlobalIndexer->getElementLIDs(cellLocalIdsNoGhost,localIds); + auto cellLocalIdsNoGhost_h = Kokkos::create_mirror_view(cellLocalIdsNoGhost); + Kokkos::deep_copy(cellLocalIdsNoGhost_h, cellLocalIdsNoGhost); + // Create vector to store if LID is owned timer->start("Create isOwned view"); PHX::View isOwned("projection unit test: isOwned", workset.numOwnedCells(),localIds.extent(1)); + auto isOwned_h = Kokkos::create_mirror_view(isOwned); { std::vector cellGIDs(localIds.extent(1)); std::vector cellOwnedIds(localIds.extent(1)); for (std::size_t cell=0; cell < cellLocalIdsNoGhost.extent(0); ++cell) { - // Assumes UVM - sourceGlobalIndexer->getElementGIDs(cellLocalIdsNoGhost(cell),cellGIDs); + sourceGlobalIndexer->getElementGIDs(cellLocalIdsNoGhost_h(cell),cellGIDs); sourceGlobalIndexer->ownedIndices(cellGIDs,cellOwnedIds); for (std::size_t i=0; i < cellOwnedIds.size(); ++i) - isOwned(cell,i) = cellOwnedIds[i]; + isOwned_h(cell,i) = cellOwnedIds[i]; } } timer->stop("Create isOwned view"); + Kokkos::deep_copy(isOwned, isOwned_h); + const auto offsetsPHI = sourceGlobalIndexer->getGIDFieldOffsetsKokkos(block,PHI_Index); const auto offsetsE = sourceGlobalIndexer->getGIDFieldOffsetsKokkos(block,E_Index); const auto offsetsB = sourceGlobalIndexer->getGIDFieldOffsetsKokkos(block,B_Index); @@ -608,18 +613,24 @@ TEUCHOS_UNIT_TEST(L2Projection, ToNodal) const auto cellLocalIdsNoGhost = Kokkos::subview(workset.cell_local_ids_k,std::make_pair(0,workset.numOwnedCells())); targetGlobalIndexer->getElementLIDs(cellLocalIdsNoGhost,localIds); + auto localIds_h = Kokkos::create_mirror_view(localIds); + Kokkos::deep_copy(localIds_h, localIds); + + auto cellLocalIdsNoGhost_h = Kokkos::create_mirror_view(cellLocalIdsNoGhost); + Kokkos::deep_copy(cellLocalIdsNoGhost_h, cellLocalIdsNoGhost); + // Create vector to store if LID is owned timer->start("Create isOwned view"); PHX::View isOwned("projection unit test: isOwned", workset.numOwnedCells(),localIds.extent(1)); + auto isOwned_h = Kokkos::create_mirror_view(isOwned); { std::vector cellGIDs(localIds.extent(1)); std::vector cellOwnedIds(localIds.extent(1)); for (std::size_t cell=0; cell < cellLocalIdsNoGhost.extent(0); ++cell) { - // Assumes UVM - targetGlobalIndexer->getElementGIDs(cellLocalIdsNoGhost(cell),cellGIDs); + targetGlobalIndexer->getElementGIDs(cellLocalIdsNoGhost_h(cell),cellGIDs); targetGlobalIndexer->ownedIndices(cellGIDs,cellOwnedIds); for (std::size_t i=0; i < cellOwnedIds.size(); ++i) - isOwned(cell,i) = cellOwnedIds[i]; + isOwned_h(cell,i) = cellOwnedIds[i]; } } timer->stop("Create isOwned view"); @@ -627,14 +638,16 @@ TEUCHOS_UNIT_TEST(L2Projection, ToNodal) const auto offsets = targetGlobalIndexer->getGIDFieldOffsets(block,0); const auto& basisValues = workset.getBasisValues(hgradBD,integrationDescriptor); const auto& coords = basisValues.getBasisCoordinates(); + auto coords_h = Kokkos::create_mirror_view(PHX::as_view(coords)); + Kokkos::deep_copy(coords_h, PHX::as_view(coords)); const int numBasis = static_cast(offsets.size()); for (int cell=0; cell < workset.numOwnedCells(); ++cell) { for (int basis=0; basis < numBasis; ++basis) { - if (isOwned(cell,offsets[basis])) { - const int lid = localIds(cell,offsets[basis]); - const double phiGold = 1.0 + coords(cell,basis,0) + 2.0 * coords(cell,basis,1) - + 3.0 * coords(cell,basis,2); + if (isOwned_h(cell,offsets[basis])) { + const int lid = localIds_h(cell,offsets[basis]); + const double phiGold = 1.0 + coords_h(cell,basis,0) + 2.0 * coords_h(cell,basis,1) + + 3.0 * coords_h(cell,basis,2); //Note: here we are relying on the fact that, for low order HGRAD basis, //the basis coefficients are the values of the function at the nodes @@ -715,18 +728,24 @@ TEUCHOS_UNIT_TEST(L2Projection, ToNodal) const auto cellLocalIdsNoGhost = Kokkos::subview(workset.cell_local_ids_k,std::make_pair(0,workset.numOwnedCells())); targetGlobalIndexer->getElementLIDs(cellLocalIdsNoGhost,localIds); + auto localIds_h = Kokkos::create_mirror_view(localIds); + Kokkos::deep_copy(localIds_h, localIds); + + auto cellLocalIdsNoGhost_h = Kokkos::create_mirror_view(cellLocalIdsNoGhost); + Kokkos::deep_copy(cellLocalIdsNoGhost_h, cellLocalIdsNoGhost); + // Create vector to store if LID is owned timer->start("Create isOwned view"); PHX::View isOwned("projection unit test: isOwned", workset.numOwnedCells(),localIds.extent(1)); + auto isOwned_h = Kokkos::create_mirror_view(isOwned); { std::vector cellGIDs(localIds.extent(1)); std::vector cellOwnedIds(localIds.extent(1)); for (std::size_t cell=0; cell < cellLocalIdsNoGhost.extent(0); ++cell) { - // Assumes UVM - targetGlobalIndexer->getElementGIDs(cellLocalIdsNoGhost(cell),cellGIDs); + targetGlobalIndexer->getElementGIDs(cellLocalIdsNoGhost_h(cell),cellGIDs); targetGlobalIndexer->ownedIndices(cellGIDs,cellOwnedIds); for (std::size_t i=0; i < cellOwnedIds.size(); ++i) - isOwned(cell,i) = cellOwnedIds[i]; + isOwned_h(cell,i) = cellOwnedIds[i]; } } timer->stop("Create isOwned view"); @@ -734,14 +753,16 @@ TEUCHOS_UNIT_TEST(L2Projection, ToNodal) const auto offsets = targetGlobalIndexer->getGIDFieldOffsets(block,0); const auto& basisValues = workset.getBasisValues(hgradBD,integrationDescriptor); const auto& coords = basisValues.getBasisCoordinates(); + auto coords_h = Kokkos::create_mirror_view(PHX::as_view(coords)); + Kokkos::deep_copy(coords_h, PHX::as_view(coords)); const int numBasis = static_cast(offsets.size()); for (int cell=0; cell < workset.numOwnedCells(); ++cell) { for (int basis=0; basis < numBasis; ++basis) { - if (isOwned(cell,offsets[basis])) { - const int lid = localIds(cell,offsets[basis]); - const double phiGold = 1.0 + coords(cell,basis,0) + 2.0 * coords(cell,basis,1) - + 3.0 * coords(cell,basis,2); + if (isOwned_h(cell,offsets[basis])) { + const int lid = localIds_h(cell,offsets[basis]); + const double phiGold = 1.0 + coords_h(cell,basis,0) + 2.0 * coords_h(cell,basis,1) + + 3.0 * coords_h(cell,basis,2); //Note: here we are relying on the fact that, for low order HGRAD basis, //the basis coefficients are the values of the function at the nodes @@ -892,18 +913,21 @@ TEUCHOS_UNIT_TEST(L2Projection, CurlMassMatrix) // fill in the mass matrix // the integral of the edge basis squared over one cell is 4/3 // the integral of the edge basis times the basis function across from it in the element is 2/3 - const auto localMass = ghostedMatrix->getLocalMatrixDevice(); - const int numElems = lids.extent(0); - Kokkos::parallel_for(numElems, KOKKOS_LAMBDA (const int& i) { - double row_values[2]={4.0/3.0,2.0/3.0}; - LO cols[2]; - for(int r = 0; r < 4; r++){ - cols[0] = lids(i,r); - cols[1] = lids(i,(r+2)%4); - localMass.sumIntoValues(lids(i,r),cols,2,row_values,false,true); - } - }); - typename PHX::Device().fence(); + { + const auto localMass = ghostedMatrix->getLocalMatrixDevice(); + const int numElems = lids.extent(0); + Kokkos::parallel_for(numElems, KOKKOS_LAMBDA (const int& i) { + double row_values[2]={4.0/3.0,2.0/3.0}; + LO cols[2]; + for(int r = 0; r < 4; r++){ + cols[0] = lids(i,r); + cols[1] = lids(i,(r+2)%4); + localMass.sumIntoValues(lids(i,r),cols,2,row_values,false,true); + } + }); + typename PHX::Device().fence(); + } + ghostedMatrix->fillComplete(); const auto exporter = factory.getGhostedExport(0); connMassMatrix->doExport(*ghostedMatrix, *exporter, Tpetra::ADD); @@ -1073,25 +1097,30 @@ TEUCHOS_UNIT_TEST(L2Projection, HighOrderTri) const auto cellLocalIdsNoGhost = Kokkos::subview(workset.cell_local_ids_k,std::make_pair(0,workset.numOwnedCells())); sourceGlobalIndexer->getElementLIDs(cellLocalIdsNoGhost,localIds); + auto cellLocalIdsNoGhost_h = Kokkos::create_mirror_view(cellLocalIdsNoGhost); + Kokkos::deep_copy(cellLocalIdsNoGhost_h, cellLocalIdsNoGhost); + // Create vector to store if LID is owned timer->start("Create isOwned view"); PHX::View isOwned("projection unit test: isOwned", workset.numOwnedCells(),localIds.extent(1)); + auto isOwned_h = Kokkos::create_mirror_view(isOwned); { std::vector cellGIDs(localIds.extent(1)); std::vector cellOwnedIds(localIds.extent(1)); for (std::size_t cell=0; cell < cellLocalIdsNoGhost.extent(0); ++cell) { - // Assumes UVM - sourceGlobalIndexer->getElementGIDs(cellLocalIdsNoGhost(cell),cellGIDs); + sourceGlobalIndexer->getElementGIDs(cellLocalIdsNoGhost_h(cell),cellGIDs); sourceGlobalIndexer->ownedIndices(cellGIDs,cellOwnedIds); for (std::size_t i=0; i < cellOwnedIds.size(); ++i) - isOwned(cell,i) = cellOwnedIds[i]; + isOwned_h(cell,i) = cellOwnedIds[i]; } } timer->stop("Create isOwned view"); + Kokkos::deep_copy(isOwned, isOwned_h); + const auto offsetsPHI = sourceGlobalIndexer->getGIDFieldOffsetsKokkos(block,PHI_Index); const auto& basisValues = workset.getBasisValues(hgradBD,integrationDescriptor); - const auto& coords = basisValues.getBasisCoordinates(); + const auto& coords = basisValues.getBasisCoordinates().get_view(); const auto& x = sourceValues->getLocalViewDevice(Tpetra::Access::OverwriteAll); const int numBasisPHI = static_cast(offsetsPHI.extent(0)); @@ -1195,18 +1224,24 @@ TEUCHOS_UNIT_TEST(L2Projection, HighOrderTri) const auto cellLocalIdsNoGhost = Kokkos::subview(workset.cell_local_ids_k,std::make_pair(0,workset.numOwnedCells())); targetGlobalIndexer->getElementLIDs(cellLocalIdsNoGhost,localIds); + auto localIds_h = Kokkos::create_mirror_view(localIds); + Kokkos::deep_copy(localIds_h, localIds); + + auto cellLocalIdsNoGhost_h = Kokkos::create_mirror_view(cellLocalIdsNoGhost); + Kokkos::deep_copy(cellLocalIdsNoGhost_h, cellLocalIdsNoGhost); + // Create vector to store if LID is owned timer->start("Create isOwned view"); PHX::View isOwned("projection unit test: isOwned", workset.numOwnedCells(),localIds.extent(1)); + auto isOwned_h = Kokkos::create_mirror_view(isOwned); { std::vector cellGIDs(localIds.extent(1)); std::vector cellOwnedIds(localIds.extent(1)); for (std::size_t cell=0; cell < cellLocalIdsNoGhost.extent(0); ++cell) { - // Assumes UVM - targetGlobalIndexer->getElementGIDs(cellLocalIdsNoGhost(cell),cellGIDs); + targetGlobalIndexer->getElementGIDs(cellLocalIdsNoGhost_h(cell),cellGIDs); targetGlobalIndexer->ownedIndices(cellGIDs,cellOwnedIds); for (std::size_t i=0; i < cellOwnedIds.size(); ++i) - isOwned(cell,i) = cellOwnedIds[i]; + isOwned_h(cell,i) = cellOwnedIds[i]; } } timer->stop("Create isOwned view"); @@ -1214,13 +1249,15 @@ TEUCHOS_UNIT_TEST(L2Projection, HighOrderTri) const auto offsets = targetGlobalIndexer->getGIDFieldOffsets(block,0); const auto& basisValues = workset.getBasisValues(hgradBD,integrationDescriptor); const auto& coords = basisValues.getBasisCoordinates(); + auto coords_h = Kokkos::create_mirror_view(PHX::as_view(coords)); + Kokkos::deep_copy(coords_h, PHX::as_view(coords)); const int numBasis = static_cast(offsets.size()); for (int cell=0; cell < workset.numOwnedCells(); ++cell) { for (int basis=0; basis < numBasis; ++basis) { - if (isOwned(cell,offsets[basis])) { - const int lid = localIds(cell,offsets[basis]); - const double phiGold = 1.0 + coords(cell,basis,0) + 2.0 * coords(cell,basis,1); + if (isOwned_h(cell,offsets[basis])) { + const int lid = localIds_h(cell,offsets[basis]); + const double phiGold = 1.0 + coords_h(cell,basis,0) + 2.0 * coords_h(cell,basis,1); TEST_FLOATING_EQUALITY(hostValues(lid,phiIndex), phiGold, tol); TEST_FLOATING_EQUALITY(hostValues(lid,dphiDxIndex), 1.0, tol); TEST_FLOATING_EQUALITY(hostValues(lid,dphiDyIndex), 2.0, tol); @@ -1285,18 +1322,24 @@ TEUCHOS_UNIT_TEST(L2Projection, HighOrderTri) const auto cellLocalIdsNoGhost = Kokkos::subview(workset.cell_local_ids_k,std::make_pair(0,workset.numOwnedCells())); targetGlobalIndexer->getElementLIDs(cellLocalIdsNoGhost,localIds); + auto localIds_h = Kokkos::create_mirror_view(localIds); + Kokkos::deep_copy(localIds_h, localIds); + + auto cellLocalIdsNoGhost_h = Kokkos::create_mirror_view(cellLocalIdsNoGhost); + Kokkos::deep_copy(cellLocalIdsNoGhost_h, cellLocalIdsNoGhost); + // Create vector to store if LID is owned timer->start("Create isOwned view"); PHX::View isOwned("projection unit test: isOwned", workset.numOwnedCells(),localIds.extent(1)); + auto isOwned_h = Kokkos::create_mirror_view(isOwned); { std::vector cellGIDs(localIds.extent(1)); std::vector cellOwnedIds(localIds.extent(1)); for (std::size_t cell=0; cell < cellLocalIdsNoGhost.extent(0); ++cell) { - // Assumes UVM - targetGlobalIndexer->getElementGIDs(cellLocalIdsNoGhost(cell),cellGIDs); + targetGlobalIndexer->getElementGIDs(cellLocalIdsNoGhost_h(cell),cellGIDs); targetGlobalIndexer->ownedIndices(cellGIDs,cellOwnedIds); for (std::size_t i=0; i < cellOwnedIds.size(); ++i) - isOwned(cell,i) = cellOwnedIds[i]; + isOwned_h(cell,i) = cellOwnedIds[i]; } } timer->stop("Create isOwned view"); @@ -1304,13 +1347,15 @@ TEUCHOS_UNIT_TEST(L2Projection, HighOrderTri) const auto offsets = targetGlobalIndexer->getGIDFieldOffsets(block,0); const auto& basisValues = workset.getBasisValues(hgradBD,integrationDescriptor); const auto& coords = basisValues.getBasisCoordinates(); + auto coords_h = Kokkos::create_mirror_view(PHX::as_view(coords)); + Kokkos::deep_copy(coords_h, PHX::as_view(coords)); const int numBasis = static_cast(offsets.size()); for (int cell=0; cell < workset.numOwnedCells(); ++cell) { for (int basis=0; basis < numBasis; ++basis) { - if (isOwned(cell,offsets[basis])) { - const int lid = localIds(cell,offsets[basis]); - const double phiGold = 1.0 + coords(cell,basis,0) + 2.0 * coords(cell,basis,1); + if (isOwned_h(cell,offsets[basis])) { + const int lid = localIds_h(cell,offsets[basis]); + const double phiGold = 1.0 + coords_h(cell,basis,0) + 2.0 * coords_h(cell,basis,1); TEST_FLOATING_EQUALITY(hostValues(lid,phiIndex), phiGold, looseTol); TEST_FLOATING_EQUALITY(hostValues(lid,dphiDxIndex), 1.0, superconvergedTol); TEST_FLOATING_EQUALITY(hostValues(lid,dphiDyIndex), 2.0, superconvergedTol); diff --git a/packages/panzer/disc-fe/src/Panzer_L2Projection.cpp b/packages/panzer/disc-fe/src/Panzer_L2Projection.cpp index 326445eefafa..76c51845bc80 100644 --- a/packages/panzer/disc-fe/src/Panzer_L2Projection.cpp +++ b/packages/panzer/disc-fe/src/Panzer_L2Projection.cpp @@ -94,13 +94,13 @@ namespace panzer { ghostedMatrix->resumeFill(); ghostedMatrix->setAllToScalar(0.0); - auto M = ghostedMatrix->getLocalMatrixDevice(); const int fieldIndex = targetGlobalIndexer_->getFieldNum(targetBasisDescriptor_.getType()); const bool is_scalar = targetBasisDescriptor_.getType()=="HGrad" || targetBasisDescriptor_.getType()=="Const" || targetBasisDescriptor_.getType()=="HVol"; // Loop over element blocks and fill mass matrix if(is_scalar){ + auto M = ghostedMatrix->getLocalMatrixDevice(); for (const auto& block : elementBlockNames_) { double ebMultiplier = 1.0; @@ -118,11 +118,14 @@ namespace panzer { const auto unweightedBasis = basisValues.getBasisValues(false).get_static_view(); const auto weightedBasis = basisValues.getBasisValues(true).get_static_view(); - // Offsets (this assumes UVM, need to fix) const std::vector& offsets = targetGlobalIndexer_->getGIDFieldOffsets(block,fieldIndex); PHX::View kOffsets("MassMatrix: Offsets",offsets.size()); + auto kOffsets_h = Kokkos::create_mirror_view(kOffsets); + for(const auto& i : offsets) - kOffsets(i) = offsets[i]; + kOffsets_h(i) = offsets[i]; + + Kokkos::deep_copy(kOffsets, kOffsets_h); // Local Ids PHX::View localIds("MassMatrix: LocalIds", workset.numOwnedCells()+workset.numGhostCells()+workset.numVirtualCells(), @@ -201,6 +204,7 @@ namespace panzer { } } } else { + auto M = ghostedMatrix->getLocalMatrixDevice(); for (const auto& block : elementBlockNames_) { double ebMultiplier = 1.0; @@ -218,11 +222,14 @@ namespace panzer { const auto unweightedBasis = basisValues.getVectorBasisValues(false).get_static_view(); const auto weightedBasis = basisValues.getVectorBasisValues(true).get_static_view(); - // Offsets (this assumes UVM, need to fix) const std::vector& offsets = targetGlobalIndexer_->getGIDFieldOffsets(block,fieldIndex); PHX::View kOffsets("MassMatrix: Offsets",offsets.size()); + auto kOffsets_h = Kokkos::create_mirror_view(kOffsets); + for(const auto& i : offsets) - kOffsets(i) = offsets[i]; + kOffsets_h(i) = offsets[i]; + + Kokkos::deep_copy(kOffsets, kOffsets_h); // Local Ids PHX::View localIds("MassMatrix: LocalIds", workset.numOwnedCells()+workset.numGhostCells()+workset.numVirtualCells(), @@ -348,7 +355,7 @@ namespace panzer { targetGlobalIndexer_->getElementGIDs(elements[elmt],row_gids); sourceGlobalIndexer.getElementGIDs(elements[elmt],col_gids); for(std::size_t row=0;rowgetLocalElement(row_gids[row]); nEntriesPerRow[lid] += col_gids.size(); } diff --git a/packages/panzer/disc-fe/src/Panzer_Workset.cpp b/packages/panzer/disc-fe/src/Panzer_Workset.cpp index d50926a15f42..4e2a04139976 100644 --- a/packages/panzer/disc-fe/src/Panzer_Workset.cpp +++ b/packages/panzer/disc-fe/src/Panzer_Workset.cpp @@ -83,9 +83,12 @@ applyBV2Orientations(const int num_cells, const auto & local_orientations = *orientations_interface->getOrientations(); std::vector workset_orientations(num_cells); + auto local_cell_ids_h = Kokkos::create_mirror_view(local_cell_ids); + Kokkos::deep_copy(local_cell_ids_h, local_cell_ids); + // We can only apply orientations to owned and ghost cells - virtual cells are ignored (no orientations available) for(int i=0; i(cell),ip,dim)); + tmpCoords_[dim].push_back(ip_coordinates_h(static_cast(cell),ip,dim)); } //********************************************************************** From 6dfadb277e7354400d18c3cc7cc82bf7d9f875dc Mon Sep 17 00:00:00 2001 From: bathmatt Date: Wed, 25 Aug 2021 13:33:58 -0400 Subject: [PATCH 27/70] panzer: Main driver1 uvm free for tpetra blocked (#9603) * fixed some device access errors * Have globa stats working * have residual calculation working in parallel * Fixed up some scatter operators * fixed resid response Co-authored-by: Matt Bettencourt Co-authored-by: Matt Bettencourt --- .../src/evaluators/Panzer_DOF_impl.hpp | 1 - .../Panzer_Dirichlet_Residual_impl.hpp | 11 +++++--- ...nzer_GatherSolution_BlockedTpetra_impl.hpp | 8 +++--- .../evaluators/Panzer_GlobalStatistics.cpp | 1 + .../Panzer_GlobalStatistics_impl.hpp | 25 +++++++++++++------ ...erDirichletResidual_BlockedTpetra_impl.hpp | 6 +++-- ...zer_ScatterResidual_BlockedTpetra_impl.hpp | 6 +++-- ...er_ResponseScatterEvaluator_Functional.cpp | 1 + ...sponseScatterEvaluator_Functional_impl.hpp | 4 ++- 9 files changed, 43 insertions(+), 20 deletions(-) diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_DOF_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_DOF_impl.hpp index 7f602ba24dca..0ca6f3b46e7f 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_DOF_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_DOF_impl.hpp @@ -208,7 +208,6 @@ DOF(const Teuchos::ParameterList & p) : auto offsets_array_h = Kokkos::create_mirror_view(offsets_array); for(std::size_t i=0;i:: evaluateFields( typename Traits::EvalData workset) { - for (index_t i = 0; i < workset.num_cells; ++i) - for (std::size_t j = 0; j < cell_data_size; ++j) - residual(i,j)=dof(i,j)-value(i,j); + auto residual_v = residual.get_static_view(); + auto dof_v = dof.get_static_view(); + auto value_v = value.get_static_view(); + auto local_cell_data_size = cell_data_size; + Kokkos::parallel_for (workset.num_cells, KOKKOS_LAMBDA (index_t i) { + for (std::size_t j = 0; j < local_cell_data_size; ++j) + residual_v(i,j)=dof_v(i,j)-value_v(i,j); + }); } //********************************************************************** diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedTpetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedTpetra_impl.hpp index 027533d4e542..d4bcea7b20d9 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedTpetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedTpetra_impl.hpp @@ -494,7 +494,7 @@ postRegistrationSetup(typename TRAITS::SetupData d, int blockOffset = globalIndexer_->getBlockGIDOffset(blockId,blk); hostBlockOffsets(blk) = blockOffset; } - blockOffsets_(numBlocks) = blockOffsets_(numBlocks-1) + blockGlobalIndexers[blockGlobalIndexers.size()-1]->getElementBlockGIDCount(blockId); + hostBlockOffsets(numBlocks) = hostBlockOffsets(numBlocks-1) + blockGlobalIndexers[blockGlobalIndexers.size()-1]->getElementBlockGIDCount(blockId); Kokkos::deep_copy(blockOffsets_,hostBlockOffsets); indexerNames_.clear(); // Don't need this anymore @@ -558,8 +558,10 @@ evaluateFields(typename TRAITS::EvalData workset) const PHX::View worksetLIDs = worksetLIDs_; const PHX::View fieldValues = gatherFields_[fieldIndex].get_static_view(); const PHX::View blockOffsets = blockOffsets_; - const int blockStart = blockOffsets(blockRowIndex); - const int numDerivatives = blockOffsets(numFieldBlocks); + auto blockOffsets_h = Kokkos::create_mirror_view(blockOffsets); + Kokkos::deep_copy(blockOffsets_h, blockOffsets); + const int blockStart = blockOffsets_h(blockRowIndex); + const int numDerivatives = blockOffsets_h(numFieldBlocks); Kokkos::parallel_for(Kokkos::RangePolicy(0,workset.num_cells), KOKKOS_LAMBDA (const int& cell) { for (int basis=0; basis < static_cast(fieldOffsets.size()); ++basis) { diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_GlobalStatistics.cpp b/packages/panzer/disc-fe/src/evaluators/Panzer_GlobalStatistics.cpp index 86bf86c9a35e..b5b718688ce7 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_GlobalStatistics.cpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_GlobalStatistics.cpp @@ -39,6 +39,7 @@ // Eric C. Cyr (eccyr@sandia.gov) // *********************************************************************** // @HEADER +#include "Kokkos_View_Fad.hpp" #include "PanzerDiscFE_config.hpp" diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_GlobalStatistics_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_GlobalStatistics_impl.hpp index d68342c963a9..eb754ae7f35a 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_GlobalStatistics_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_GlobalStatistics_impl.hpp @@ -115,9 +115,11 @@ postRegistrationSetup( PHX::FieldManager& /* fm */) { ir_index = panzer::getIntegrationRuleIndex(ir_order,(*sd.worksets_)[0], this->wda); - for (typename PHX::MDField::size_type cell = 0; cell < ones.extent(0); ++cell) - for (typename PHX::MDField::size_type ip = 0; ip < ones.extent(1); ++ip) - ones(cell,ip) = 1.0; + auto l_ones = ones.get_static_view(); + Kokkos::parallel_for("GlobalStatistics", l_ones.extent(0), KOKKOS_LAMBDA(int cell) { + for (std::size_t ip = 0; ip < l_ones.extent(1); ++ip) + l_ones(cell,ip) = 1.0; + }); } //********************************************************************** @@ -133,9 +135,11 @@ evaluateFields( Intrepid2::FunctionSpaceTools::integrate(volumes.get_view(), ones.get_view(), (this->wda(workset).int_rules[ir_index])->weighted_measure.get_view()); + auto volumes_h = Kokkos::create_mirror_view(as_view(volumes)); + Kokkos::deep_copy(volumes_h, as_view(volumes)); for (index_t cell = 0; cell < workset.num_cells; ++cell) - total_volume += volumes(cell); + total_volume += volumes_h(cell); typename std::vector >::size_type field_index = 0; for (typename std::vector >::iterator field = field_values.begin(); @@ -144,13 +148,18 @@ evaluateFields( Intrepid2::FunctionSpaceTools::integrate(tmp.get_view(), field->get_view(), (this->wda(workset).int_rules[ir_index])->weighted_measure.get_view()); - + auto tmp_h = Kokkos::create_mirror_view(tmp.get_static_view()); + auto field_h = Kokkos::create_mirror_view( field->get_static_view()); + Kokkos::deep_copy(tmp_h, tmp.get_static_view()); + Kokkos::deep_copy(field_h, field->get_static_view()); + + for (index_t cell = 0; cell < workset.num_cells; ++cell) { - averages[field_index] += tmp(cell); + averages[field_index] += tmp_h(cell); for (typename PHX::MDField::size_type ip = 0; ip < (field->extent(1)); ++ip) { - maxs[field_index] = std::max( (*field)(cell,ip), maxs[field_index]); - mins[field_index] = std::min( (*field)(cell,ip), mins[field_index]); + maxs[field_index] = std::max( field_h(cell,ip), maxs[field_index]); + mins[field_index] = std::min( field_h(cell,ip), mins[field_index]); } } diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedTpetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedTpetra_impl.hpp index fad83bb8466c..724a4f03ef9d 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedTpetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedTpetra_impl.hpp @@ -435,7 +435,7 @@ postRegistrationSetup(typename TRAITS::SetupData d, int blockOffset = globalIndexer_->getBlockGIDOffset(blockId,blk); hostBlockOffsets(blk) = blockOffset; } - blockOffsets_(numBlocks) = blockOffsets_(numBlocks-1) + blockGlobalIndexers[blockGlobalIndexers.size()-1]->getElementBlockGIDCount(blockId); + hostBlockOffsets(numBlocks) = hostBlockOffsets(numBlocks-1) + blockGlobalIndexers[blockGlobalIndexers.size()-1]->getElementBlockGIDCount(blockId); Kokkos::deep_copy(blockOffsets_,hostBlockOffsets); // Make sure the that hard coded derivative dimension in the @@ -552,8 +552,10 @@ evaluateFields(typename TRAITS::EvalData workset) // lids for the sub-block that it is scattering to. The subviews // below are to offset the LID blocks correctly. const auto& globalIndexers = globalIndexer_->getFieldDOFManagers(); + auto blockOffsets_h = Kokkos::create_mirror_view(blockOffsets_); + Kokkos::deep_copy(blockOffsets_h, blockOffsets_); for (size_t block=0; block < globalIndexers.size(); ++block) { - const auto subviewOfBlockLIDs = Kokkos::subview(worksetLIDs_,Kokkos::ALL(), std::make_pair(blockOffsets_(block),blockOffsets_(block+1))); + const auto subviewOfBlockLIDs = Kokkos::subview(worksetLIDs_,Kokkos::ALL(), std::make_pair(blockOffsets_h(block),blockOffsets_h(block+1))); globalIndexers[block]->getElementLIDs(localCellIds,subviewOfBlockLIDs); } diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_BlockedTpetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_BlockedTpetra_impl.hpp index c11cb9a8868b..6362a7617b46 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_BlockedTpetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_BlockedTpetra_impl.hpp @@ -318,7 +318,7 @@ postRegistrationSetup(typename TRAITS::SetupData d, int blockOffset = globalIndexer_->getBlockGIDOffset(blockId,blk); hostBlockOffsets(blk) = blockOffset; } - blockOffsets_(numBlocks) = blockOffsets_(numBlocks-1) + blockGlobalIndexers[blockGlobalIndexers.size()-1]->getElementBlockGIDCount(blockId); + hostBlockOffsets(numBlocks) = hostBlockOffsets(numBlocks-1) + blockGlobalIndexers[blockGlobalIndexers.size()-1]->getElementBlockGIDCount(blockId); Kokkos::deep_copy(blockOffsets_,hostBlockOffsets); // Make sure the that hard coded derivative dimension in the @@ -435,8 +435,10 @@ evaluateFields(typename TRAITS::EvalData workset) // lids for the sub-block that it is scattering to. The subviews // below are to offset the LID blocks correctly. const auto& globalIndexers = globalIndexer_->getFieldDOFManagers(); + auto blockOffsets_h = Kokkos::create_mirror_view(blockOffsets_); + Kokkos::deep_copy(blockOffsets_h, blockOffsets_); for (size_t block=0; block < globalIndexers.size(); ++block) { - const auto subviewOfBlockLIDs = Kokkos::subview(worksetLIDs_,Kokkos::ALL(), std::make_pair(blockOffsets_(block),blockOffsets_(block+1))); + const auto subviewOfBlockLIDs = Kokkos::subview(worksetLIDs_,Kokkos::ALL(), std::make_pair(blockOffsets_h(block),blockOffsets_h(block+1))); globalIndexers[block]->getElementLIDs(localCellIds,subviewOfBlockLIDs); } diff --git a/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional.cpp b/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional.cpp index 72d14da63289..ccee42edc62c 100644 --- a/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional.cpp +++ b/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional.cpp @@ -39,6 +39,7 @@ // Eric C. Cyr (eccyr@sandia.gov) // *********************************************************************** // @HEADER +#include "Kokkos_View_Fad.hpp" #include "PanzerDiscFE_config.hpp" diff --git a/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional_impl.hpp b/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional_impl.hpp index 26b514f012e0..aa6446e46a9d 100644 --- a/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional_impl.hpp +++ b/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional_impl.hpp @@ -136,8 +136,10 @@ template void ResponseScatterEvaluator_Functional:: evaluateFields(typename Traits::EvalData d) { + auto cellIntegral_h = Kokkos::create_mirror_view ( cellIntegral_.get_static_view()); + Kokkos::deep_copy(cellIntegral_h, cellIntegral_.get_static_view()); for(index_t i=0;ivalue += cellIntegral_(i); + responseObj_->value += cellIntegral_h(i); } } From c2a27a8405d8d01f73662444e2827fb86d50b276 Mon Sep 17 00:00:00 2001 From: bathmatt Date: Wed, 25 Aug 2021 18:44:57 -0400 Subject: [PATCH 28/70] Panzer epetra scatter (#9581) * Have one gather working * have initial resid gather working * Fixed dirichlet resid eval * have resid dirichlet scatter working * Have gather working for jacobian * Got past jacobian scatter * have epetra GS passing * Fixed warnings for gcc Co-authored-by: Matt Bettencourt Co-authored-by: Matt Bettencourt --- .../Panzer_GatherSolution_Epetra.cpp | 2 + .../Panzer_GatherSolution_Epetra_impl.hpp | 155 ++++++------------ ...Panzer_ScatterDirichletResidual_Epetra.cpp | 2 + ...r_ScatterDirichletResidual_Epetra_impl.hpp | 138 +++++++++------- .../Panzer_ScatterResidual_Epetra.cpp | 2 + .../Panzer_ScatterResidual_Epetra_impl.hpp | 53 +++--- 6 files changed, 167 insertions(+), 185 deletions(-) diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Epetra.cpp b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Epetra.cpp index 8fd8394423a3..7e53d7dd6b0f 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Epetra.cpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Epetra.cpp @@ -40,6 +40,8 @@ // *********************************************************************** // @HEADER +#include "Kokkos_View_Fad.hpp" + #include "PanzerDiscFE_config.hpp" #include "Panzer_ExplicitTemplateInstantiation.hpp" diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Epetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Epetra_impl.hpp index 4d9dd12ad4a8..f8e67957383b 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Epetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Epetra_impl.hpp @@ -253,58 +253,36 @@ evaluateFields( // NOTE: A reordering of these loops will likely improve performance. The // "getGIDFieldOffsets may be expensive. However the "getElementGIDs" // can be cheaper. However the lookup for LIDs may be more expensive! - if (x_.is_null()) - { - // Gather operation for each cell in the workset. - for (int cell(0); cell < numCells; ++cell) - { - size_t cellLocalId(localCellIds[cell]); - auto LIDs = globalIndexer_->getElementLIDs(cellLocalId); - // Loop over the fields to be gathered. - for (int fieldInd(0); fieldInd < numFields; ++fieldInd) - { - MDField& field = gatherFields_[fieldInd]; - int fieldNum(fieldIds_[fieldInd]); - const vector& elmtOffset = - globalIndexer_->getGIDFieldOffsets(blockId, fieldNum); - int numBases(elmtOffset.size()); - - // Loop over the basis functions and fill the fields. - for (int basis(0); basis < numBases; ++basis) - { - int offset(elmtOffset[basis]), lid(LIDs[offset]); - field(cell, basis) = (*xEvRoGed_)[lid]; - } // end loop over the basis functions - } // end loop over the fields to be gathered - } // end loop over localCellIds - } - else // if (not x_.is_null()) + auto LIDs = globalIndexer_->getLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); + // Loop over the fields to be gathered. + for (int fieldInd(0); fieldInd < numFields; ++fieldInd) { + MDField& field = gatherFields_[fieldInd]; + auto field_h = Kokkos::create_mirror_view(field.get_static_view()); + int fieldNum(fieldIds_[fieldInd]); + const vector& elmtOffset = + globalIndexer_->getGIDFieldOffsets(blockId, fieldNum); + int numBases(elmtOffset.size()); // Gather operation for each cell in the workset. for (int cell(0); cell < numCells; ++cell) { - size_t cellLocalId(localCellIds[cell]); - auto LIDs = globalIndexer_->getElementLIDs(cellLocalId); - - // Loop over the fields to be gathered. - for (int fieldInd(0); fieldInd < numFields; ++fieldInd) + size_t cellLocalId(localCellIds[cell]); + // Loop over the basis functions and fill the fields. + for (int basis(0); basis < numBases; ++basis) { - MDField& field = gatherFields_[fieldInd]; - int fieldNum(fieldIds_[fieldInd]); - const vector& elmtOffset = - globalIndexer_->getGIDFieldOffsets(blockId, fieldNum); - int numBases(elmtOffset.size()); - - // Loop over the basis functions and fill the fields. - for (int basis(0); basis < numBases; ++basis) - { - int offset(elmtOffset[basis]), lid(LIDs[offset]); - field(cell, basis) = (*x_)[lid]; - } // end loop over the basis functions - } // end loop over the fields to be gathered + int offset(elmtOffset[basis]), lid(LIDs_h(cellLocalId, offset)); + if (x_.is_null()) + field_h(cell, basis) = (*xEvRoGed_)[lid]; + else + field_h(cell, basis) = (*x_)[lid]; + } // end loop over the basis functions } // end loop over localCellIds - } // end if (x_.is_null()) or not + Kokkos::deep_copy(field.get_static_view(), field_h); + } // end loop over the fields to be gathered + } // end of evaluateFields() (Residual Specialization) /////////////////////////////////////////////////////////////////////////////// @@ -780,66 +758,35 @@ evaluateFields( // NOTE: A reordering of these loops will likely improve performance. The // "getGIDFieldOffsets may be expensive. However the "getElementGIDs" // can be cheaper. However the lookup for LIDs may be more expensive! - if (x_.is_null()) + auto LIDs = globalIndexer_->getLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); + // Loop over the fields to be gathered. + for (int fieldInd(0); fieldInd < numFields; ++fieldInd) { - // Loop over the fields to be gathered. - for (int fieldInd(0); fieldInd < numFields; ++fieldInd) + MDField& field = gatherFields_[fieldInd]; + auto field_h = Kokkos::create_mirror_view(field.get_static_view()); + int fieldNum(fieldIds_[fieldInd]); + const vector& elmtOffset = + globalIndexer_->getGIDFieldOffsets(blockId, fieldNum); + int numBases(elmtOffset.size()); + // Gather operation for each cell in the workset. + for (int cell(0); cell < numCells; ++cell) { - MDField& field = gatherFields_[fieldInd]; - int fieldNum(fieldIds_[fieldInd]); - const vector& elmtOffset = - globalIndexer_->getGIDFieldOffsets(blockId, fieldNum); - int numBases(elmtOffset.size()); - - // Gather operation for each cell in the workset. - for (int cell(0); cell < numCells; ++cell) + size_t cellLocalId(localCellIds[cell]); + // Loop over the basis functions and fill the fields. + for (int basis(0); basis < numBases; ++basis) { - size_t cellLocalId(localCellIds[cell]); - auto LIDs = globalIndexer_->getElementLIDs(cellLocalId); - - // Loop over the basis functions and fill the fields. - for (int basis(0); basis < numBases; ++basis) - { - int offset(elmtOffset[basis]), lid(LIDs[offset]); - field(cell, basis) = (*xEvRoGed_)[lid]; - } // end loop over the basis functions - } // end loop over localCellIds - } // end loop over the fields to be gathered - } - else // if (not x_.is_null()) - { - // Loop over the fields to be gathered. - for (int fieldInd(0); fieldInd < numFields; ++fieldInd) - { - MDField& field = gatherFields_[fieldInd]; - int fieldNum(fieldIds_[fieldInd]); - const vector& elmtOffset = - globalIndexer_->getGIDFieldOffsets(blockId, fieldNum); - int numBases(elmtOffset.size()); - - // Gather operation for each cell in the workset. - for (int cell(0); cell < numCells; ++cell) - { - size_t cellLocalId(localCellIds[cell]); - auto LIDs = globalIndexer_->getElementLIDs(cellLocalId); - - // Loop over the basis functions and fill the fields. - for (int basis(0); basis < numBases; ++basis) - { - int offset(elmtOffset[basis]), lid(LIDs[offset]); - field(cell, basis) = (*x_)[lid]; - } // end loop over the basis functions - } // end loop over localCellIds - } // end loop over the fields to be gathered - } // end if (x_.is_null()) or not + int offset(elmtOffset[basis]), lid(LIDs_h(cellLocalId, offset)); + if (x_.is_null()) + field_h(cell, basis) = (*xEvRoGed_)[lid]; + else + field_h(cell, basis) = (*x_)[lid]; + } // end loop over the basis functions + } // end loop over localCellIds // Deal with the sensitivities. - if (applySensitivities_) - { - // Loop over the fields to be gathered. - for (int fieldInd(0); fieldInd < numFields; ++fieldInd) - { - MDField& field = gatherFields_[fieldInd]; + if (applySensitivities_) { int fieldNum(fieldIds_[fieldInd]); const vector& elmtOffset = globalIndexer_->getGIDFieldOffsets(blockId, fieldNum); @@ -854,11 +801,13 @@ evaluateFields( // Seed the FAD object. int offset(elmtOffset[basis]); - field(cell, basis).fastAccessDx(dos + offset) = seedValue; + field_h(cell, basis).fastAccessDx(dos + offset) = seedValue; } // end loop over the basis functions } // end loop over localCellIds - } // end loop over the fields to be gathered - } // end if (applySensitivities_) + } // end if (applySensitivities_) + Kokkos::deep_copy(field.get_static_view(), field_h); + } // end loop over the fields to be gathered + } // end of evaluateFields() (Jacobian Specialization) #endif // __Panzer_GatherSolution_Epetra_impl_hpp__ diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Epetra.cpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Epetra.cpp index c61e70262a98..5838a4c50765 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Epetra.cpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Epetra.cpp @@ -40,6 +40,8 @@ // *********************************************************************** // @HEADER +#include "Kokkos_View_Fad.hpp" + #include "PanzerDiscFE_config.hpp" #include "Panzer_ExplicitTemplateInstantiation.hpp" diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Epetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Epetra_impl.hpp index a66bee4c87f5..5f68a50ff839 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Epetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Epetra_impl.hpp @@ -188,60 +188,66 @@ evaluateFields(typename TRAITS::EvalData workset) // may be more expensive! // scatter operation for each cell in workset - for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementLIDs(cellLocalId); - - // loop over each field to be scattered - for(std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) { - int fieldNum = fieldIds_[fieldIndex]; - - if (!scatterIC_) { - // this call "should" get the right ordering according to the Intrepid2 basis - const std::pair,std::vector > & indicePair - = globalIndexer_->getGIDFieldOffsets_closure(blockId,fieldNum, side_subcell_dim_, local_side_id_); - const std::vector & elmtOffset = indicePair.first; - const std::vector & basisIdMap = indicePair.second; - - // loop over basis functions - for(std::size_t basis=0;basisgetLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); + + + // loop over each field to be scattered + for(std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) { + int fieldNum = fieldIds_[fieldIndex]; + auto field = PHX::as_view(scatterFields_[fieldIndex]); + auto field_h = Kokkos::create_mirror_view(field); + Kokkos::deep_copy(field_h, field); + + for(std::size_t worksetCellIndex=0;worksetCellIndex,std::vector > & indicePair + = globalIndexer_->getGIDFieldOffsets_closure(blockId,fieldNum, side_subcell_dim_, local_side_id_); + const std::vector & elmtOffset = indicePair.first; + const std::vector & basisIdMap = indicePair.second; + + // loop over basis functions + for(std::size_t basis=0;basis & elmtOffset = globalIndexer_->getGIDFieldOffsets(blockId,fieldNum); - + // record that you set a dirichlet condition + if(dirichletCounter_!=Teuchos::null) + (*dirichletCounter_)[lid] = 1.0; + } + } else { + // this call "should" get the right ordering according to the Intrepid2 basis + const std::vector & elmtOffset = globalIndexer_->getGIDFieldOffsets(blockId,fieldNum); + // loop over basis functions - for(std::size_t basis=0;basis cLIDs, rLIDs; int gidCount(0); bool useColumnIndexer = colGlobalIndexer_!=Teuchos::null; + const Teuchos::RCP& + colGlobalIndexer = useColumnIndexer ? colGlobalIndexer_ : globalIndexer_; // for convenience pull out some objects from workset std::string blockId = this->wda(workset).block_id; @@ -592,18 +600,24 @@ evaluateFields(typename TRAITS::EvalData workset) // may be more expensive! // scatter operation for each cell in workset + + auto LIDs = globalIndexer_->getLIDs(); + auto colLIDs = colGlobalIndexer->getLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + auto colLIDs_h = Kokkos::create_mirror_view(colLIDs); + Kokkos::deep_copy(LIDs_h, LIDs); + Kokkos::deep_copy(colLIDs_h, colLIDs); + + std::vector scatterFields_h; + for ( std::size_t i=0; i< scatterFields_.size(); ++i) { + scatterFields_h.push_back(Kokkos::create_mirror_view(scatterFields_[i].get_static_view())); + Kokkos::deep_copy(scatterFields_h[i], scatterFields_[i].get_static_view()); + } + for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementLIDs(cellLocalId); - gidCount = globalIndexer_->getElementBlockGIDCount(blockId); - if(useColumnIndexer) - { - cLIDs = colGlobalIndexer_->getElementLIDs(cellLocalId); - gidCount = colGlobalIndexer_->getElementBlockGIDCount(blockId); - } - else - cLIDs = rLIDs; + gidCount = colGlobalIndexer->getElementBlockGIDCount(blockId); // loop over each field to be scattered for(std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) { @@ -618,7 +632,7 @@ evaluateFields(typename TRAITS::EvalData workset) // loop over basis functions for(std::size_t basis=0;basisReplaceMyValues(row, gidCount, scatterField.dx(), - &cLIDs[0]); + &colLIDs_h(cellLocalId,0)); TEUCHOS_ASSERT(err==0); } } diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_Epetra.cpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_Epetra.cpp index 8920bfb32f70..3caab6440264 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_Epetra.cpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_Epetra.cpp @@ -40,6 +40,8 @@ // *********************************************************************** // @HEADER +#include "Kokkos_View_Fad.hpp" + #include "PanzerDiscFE_config.hpp" #include "Panzer_ExplicitTemplateInstantiation.hpp" diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_Epetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_Epetra_impl.hpp index 6f0343d6d871..c88cb9a8c335 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_Epetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_Epetra_impl.hpp @@ -157,21 +157,25 @@ evaluateFields(typename TRAITS::EvalData workset) // may be more expensive! // scatter operation for each cell in workset - for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementLIDs(cellLocalId); - - // loop over each field to be scattered - for (std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) { - int fieldNum = fieldIds_[fieldIndex]; - const std::vector & elmtOffset = globalIndexer_->getGIDFieldOffsets(blockId,fieldNum); + auto LIDs = globalIndexer_->getLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); + // loop over each field to be scattered + for (std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) { + int fieldNum = fieldIds_[fieldIndex]; + auto field = PHX::as_view(scatterFields_[fieldIndex]); + auto field_h = Kokkos::create_mirror_view(field); + Kokkos::deep_copy(field_h, field); + + const std::vector & elmtOffset = globalIndexer_->getGIDFieldOffsets(blockId,fieldNum); + for(std::size_t worksetCellIndex=0;worksetCellIndexgetLIDs(); + auto colLIDs = colGlobalIndexer->getLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + auto colLIDs_h = Kokkos::create_mirror_view(colLIDs); + Kokkos::deep_copy(LIDs_h, LIDs); + Kokkos::deep_copy(colLIDs_h, colLIDs); + std::vector scatterFields_h; + for ( std::size_t i=0; i< scatterFields_.size(); ++i) { + scatterFields_h.push_back(Kokkos::create_mirror_view(scatterFields_[i].get_static_view())); + Kokkos::deep_copy(scatterFields_h[i], scatterFields_[i].get_static_view()); + } + for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementLIDs(cellLocalId); - auto initial_cLIDs = colGlobalIndexer->getElementLIDs(cellLocalId); std::vector cLIDs; - for (int i(0); i < static_cast(initial_cLIDs.extent(0)); ++i) - cLIDs.push_back(initial_cLIDs(i)); + for (int i(0); i < static_cast(colLIDs_h.extent(1)); ++i) + cLIDs.push_back(colLIDs_h(cellLocalId, i)); if (Teuchos::nonnull(workset.other)) { const std::size_t other_cellLocalId = workset.other->cell_local_ids[worksetCellIndex]; - auto other_cLIDs = colGlobalIndexer->getElementLIDs(other_cellLocalId); - for (int i(0); i < static_cast(other_cLIDs.extent(0)); ++i) - cLIDs.push_back(other_cLIDs(i)); + for (int i(0); i < static_cast(colLIDs_h.extent(1)); ++i) + cLIDs.push_back(colLIDs_h(other_cellLocalId, i)); } // loop over each field to be scattered @@ -423,9 +436,9 @@ evaluateFields(typename TRAITS::EvalData workset) // loop over the basis functions (currently they are nodes) for(std::size_t rowBasisNum = 0; rowBasisNum < elmtOffset.size(); rowBasisNum++) { - const ScalarT scatterField = (scatterFields_[fieldIndex])(worksetCellIndex,rowBasisNum); + const ScalarT scatterField = (scatterFields_h[fieldIndex])(worksetCellIndex,rowBasisNum); int rowOffset = elmtOffset[rowBasisNum]; - int row = rLIDs[rowOffset]; + int row = LIDs_h(cellLocalId,rowOffset); // Sum residual if(r!=Teuchos::null) From b6b992a810a1dc9ce097d079c3eb105436cea21c Mon Sep 17 00:00:00 2001 From: Paul Wolfenbarger Date: Thu, 26 Aug 2021 07:02:18 -0600 Subject: [PATCH 29/70] This resolves some cmake/std paths to the new packages/framework/pr_tools Basically this was showing up when the merge chjanges SHA1 results in the driver scripts. The scripts shoudl now be self-consistent. --- .../framework/pr_tools/PullRequestLinuxCudaDriver.sh | 6 +++--- .../pr_tools/PullRequestLinuxCudaVortexDriver.sh | 4 ++-- .../framework/pr_tools/PullRequestLinuxDriver.sh | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/framework/pr_tools/PullRequestLinuxCudaDriver.sh b/packages/framework/pr_tools/PullRequestLinuxCudaDriver.sh index 91b0e51f5b02..93b7abbca987 100755 --- a/packages/framework/pr_tools/PullRequestLinuxCudaDriver.sh +++ b/packages/framework/pr_tools/PullRequestLinuxCudaDriver.sh @@ -9,16 +9,16 @@ if [ "${Trilinos_CTEST_DO_ALL_AT_ONCE}" == "" ] ; then fi # comment out sh and add what we need individually. -#source $WORKSPACE/Trilinos/cmake/std/atdm/load-env.sh $JOB_NAME +#source $WORKSPACE/Trilinos/packages/framework/pr_tools/atdm/load-env.sh $JOB_NAME set -x #TODO: review appropriate job size #cuda 10.1.105 will be built with rhel7G if [[ "${JOB_NAME}" == "trilinos-folder/Trilinos_pullrequest_cuda_10.1.105"* ]] ; then - bsub -x -Is -q rhel7G -n 16 -J ${JOB_NAME} -W ${BSUB_CTEST_TIME_LIMIT} ${WORKSPACE}/Trilinos/cmake/std/PullRequestLinuxDriver.sh + bsub -x -Is -q rhel7G -n 16 -J ${JOB_NAME} -W ${BSUB_CTEST_TIME_LIMIT} ${WORKSPACE}/Trilinos/packages/framework/pr_tools/PullRequestLinuxDriver.sh else - bsub -x -Is -q rhel7F -n 16 -J ${JOB_NAME} -W ${BSUB_CTEST_TIME_LIMIT} ${WORKSPACE}/Trilinos/cmake/std/PullRequestLinuxDriver.sh + bsub -x -Is -q rhel7F -n 16 -J ${JOB_NAME} -W ${BSUB_CTEST_TIME_LIMIT} ${WORKSPACE}/Trilinos/packages/framework/pr_tools/PullRequestLinuxDriver.sh fi # NOTE: Above, this bsub command should grab a single rhel7F (Firestone, # Dual-Socket POWER8, 8 cores per socket, K80 GPUs) node. The option '-x' diff --git a/packages/framework/pr_tools/PullRequestLinuxCudaVortexDriver.sh b/packages/framework/pr_tools/PullRequestLinuxCudaVortexDriver.sh index a1f19010fbd2..3d85541e4a01 100755 --- a/packages/framework/pr_tools/PullRequestLinuxCudaVortexDriver.sh +++ b/packages/framework/pr_tools/PullRequestLinuxCudaVortexDriver.sh @@ -9,13 +9,13 @@ if [ "${Trilinos_CTEST_DO_ALL_AT_ONCE}" == "" ] ; then fi # comment out sh and add what we need individually. -#source $WORKSPACE/Trilinos/cmake/std/atdm/load-env.sh $JOB_NAME +#source $WORKSPACE/Trilinos/packages/framework/pr_tools/atdm/load-env.sh $JOB_NAME set -x #TODO: review appropriate job size bsub -Is -nnodes 2 -J ${JOB_NAME} -W ${BSUB_CTEST_TIME_LIMIT} \ - ${WORKSPACE}/Trilinos/cmake/std/PullRequestLinuxDriver.sh + ${WORKSPACE}/Trilinos/packages/framework/pr_tools/PullRequestLinuxDriver.sh # NOTE: Above, this bsub command should grab a two # nodes. The option '-x' makes sure that only this diff --git a/packages/framework/pr_tools/PullRequestLinuxDriver.sh b/packages/framework/pr_tools/PullRequestLinuxDriver.sh index ca6b81349b99..bd986e07c1e2 100755 --- a/packages/framework/pr_tools/PullRequestLinuxDriver.sh +++ b/packages/framework/pr_tools/PullRequestLinuxDriver.sh @@ -70,10 +70,10 @@ test -d ${REPO_ROOT:?}/.git || REPO_ROOT=`readlink -f ${WORKSPACE:?}/Trilinos` message_std "PRDriver> " "REPO_ROOT : ${REPO_ROOT}" # Get the md5 checksum of this script: -sig_script_old=$(get_md5sum ${REPO_ROOT:?}/cmake/std/PullRequestLinuxDriver.sh) +sig_script_old=$(get_md5sum ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriver.sh) # Get the md5 checksum of the Merge script -sig_merge_old=$(get_md5sum ${REPO_ROOT:?}/cmake/std/PullRequestLinuxDriverMerge.py) +sig_merge_old=$(get_md5sum ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriverMerge.py) print_banner "Merge Source into Target" @@ -87,7 +87,7 @@ merge_cmd_options=( ${TRILINOS_SOURCE_SHA:?} ${WORKSPACE:?} ) -merge_cmd="${PYTHON_EXE:?} ${REPO_ROOT:?}/cmake/std/PullRequestLinuxDriverMerge.py ${merge_cmd_options[@]}" +merge_cmd="${PYTHON_EXE:?} ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriverMerge.py ${merge_cmd_options[@]}" # Call the script to handle merging the incoming branch into @@ -105,13 +105,13 @@ print_banner "Merge completed" # Get the md5 checksum of this script: -sig_script_new=$(get_md5sum ${REPO_ROOT:?}/cmake/std/PullRequestLinuxDriver.sh) +sig_script_new=$(get_md5sum ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriver.sh) message_std "PRDriver> " "Old md5 checksum ${sig_script_old:?} for ${SCRIPTFILE:?}" message_std "PRDriver> " "New md5 checksum ${sig_script_new:?} for ${SCRIPTFILE:?}" message_std "PRDriver> " "" # Get the md5 checksum of the Merge script -sig_merge_new=$(get_md5sum ${REPO_ROOT:?}/cmake/std/PullRequestLinuxDriverMerge.py) +sig_merge_new=$(get_md5sum ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriverMerge.py) message_std "PRDriver> " "Old md5 checksum ${sig_merge_old:?} for ${SCRIPTPATH}/PullRequestLinuxDriverMerge.py" message_std "PRDriver> " "New md5 checksum ${sig_merge_new:?} for ${SCRIPTPATH}/PullRequestLinuxDriverMerge.py" @@ -120,7 +120,7 @@ then message_std "PRDriver> " "" message_std "PRDriver> " "Driver or Merge script change detected. Re-launching PR Driver" message_std "PRDriver> " "" - ${REPO_ROOT:?}/cmake/std/PullRequestLinuxDriver.sh + ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriver.sh exit $? fi From 4bdf69589cf86099ee8925bff31a77364c443352 Mon Sep 17 00:00:00 2001 From: Paul Wolfenbarger Date: Thu, 26 Aug 2021 07:04:34 -0600 Subject: [PATCH 30/70] Remove an outdated file that should not have been copied from the old location --- .../pr_tools/PullRequestLinuxDriver-old.sh | 311 ------------------ 1 file changed, 311 deletions(-) delete mode 100755 packages/framework/pr_tools/PullRequestLinuxDriver-old.sh diff --git a/packages/framework/pr_tools/PullRequestLinuxDriver-old.sh b/packages/framework/pr_tools/PullRequestLinuxDriver-old.sh deleted file mode 100755 index 918ea282ba14..000000000000 --- a/packages/framework/pr_tools/PullRequestLinuxDriver-old.sh +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/env bash -# set -x # echo commands - -# -# This script drives a PR testing build. It assume that Trilinos is already -# cloned under $PWD/Trilinos and that the 'origin' remote points to -# $TRILINOS_TARGET_REPO (but that is not checked here). -# -# As long as the ${PWD}/Trilinos git repo has the correct 'origin', this -# script will automatically set it up to do the merge correctly, no matter -# what its state before this script is called (i.e. from a past PR -# attempt). Unless the Trilinos/.git directory becomes corrupted, there should -# *NEVER* be any need to delete and reclone this Trilinos git repo. -# -# This script can be run in a mode where the driver scripts are run from one -# Trilinos git repo and operate on another Trilinos git repo that gets -# manipulated in order to merge the "source" topic branch into the "target" -# branch. This makes it easy to test changes to the PR scripts. But if this -# script is run from ${PWD}/Trilinos, then these repos are one and the same -# and we get the correct behavior for PR testing. -# - -export https_proxy=http://proxy.sandia.gov:80 -export http_proxy=http://proxy.sandia.gov:80 -no_proxy='localhost,localnets,.sandia.gov,127.0.0.1,169.254.0.0/16,forge.sandia.gov' - -whoami -which -a env - -echo -e "" -echo -e "Jenkins Environment Variables:" -echo -e "- JOB_BASE_NAME: ${JOB_BASE_NAME}" -echo -e "" - -## Rather than do proper option handling right now I am just going to -## test that all these environment variables are set. Because getopt ;-> -: ${TRILINOS_SOURCE_REPO:?} -: ${TRILINOS_SOURCE_BRANCH:?} -: ${TRILINOS_TARGET_REPO:?} -: ${TRILINOS_TARGET_BRANCH:?} -: ${TRILINOS_SOURCE_SHA:?} -: ${PULLREQUESTNUM:?} -: ${JOB_BASE_NAME:?} -: ${BUILD_NUMBER:?} -: ${WORKSPACE:?} - -source /projects/sems/modulefiles/utils/sems-modules-init.sh - -declare -i ierror=0 -#Have to keep loading git -module load sems-git/2.10.1 - - -#-------------------------------------------- -# Get Trilinos scripts and PR merge repo dirs -#-------------------------------------------- - -# The Trilinos dir that the PR merge will be done in -PR_TRILINOS_DIR=$PWD/Trilinos -echo "PR_TRILINOS_DIR = $PR_TRILINOS_DIR" - -# Get the Trilinos scripts driver dir where this script is run from. First,t -# ry to grab from the symlink (only works on Linux) -_ABS_FILE_PATH=`readlink -f $0` || \ - echo "Could not follow symlink to set TRILINOS_DRIVER_SRC_DIR!" -if [ "$_ABS_FILE_PATH" != "" ] ; then - _SCRIPT_DIR=`dirname $_ABS_FILE_PATH` - TRILINOS_DRIVER_SRC_DIR=$_SCRIPT_DIR/../.. -else - # If that did not work, then we are not on Linux so give up and assume the - # standard location - TRILINOS_DRIVER_SRC_DIR=${PR_TRILINOS_DIR} -fi -echo "TRILINOS_DRIVER_SRC_DIR = $TRILINOS_DRIVER_SRC_DIR" - - -#------------------------------ -# Doing merge of pull request -#------------------------------ - -ls -cd Trilinos - -# Check for existence of source_remote and remove if it exists -git_remote_text=`git remote -v | grep "source_remote"` -if [[ "$git_remote_text" != "" ]]; then - echo "git remote exists, removing it." - git remote rm source_remote -fi - -# Add the necessary remote -git remote add source_remote ${TRILINOS_SOURCE_REPO:?} -ierror=$? -if [[ $ierror != 0 ]]; then - echo "There was a problem adding the remote for the source repo. The error code was: $ierror" - #just in case somehow a previously defined source_remote caused this failure - #would be better to check prior to the add. Don't want to issue command that - #will be known to fail typically. git remote rm source_remote - exit $ierror -fi - -git remote -v - -num_retries=3 - -for i in `seq ${num_retries}` -do - git fetch source_remote ${TRILINOS_SOURCE_BRANCH:?} - ierror=$? - if [[ $ierror != 0 ]]; then - echo "Source remote fetch failed. The error code was: $ierror" - if [[ $i != $num_retries ]] - then - echo "retry $i" - sleep $(($i*20)) - else - exit $ierror - fi - else - break - fi -done - -git fetch origin ${TRILINOS_TARGET_BRANCH:?} -ierror=$? -if [[ $ierror != 0 ]]; then - echo "Origin target remote fetch failed. The error code was: $ierror" - exit $ierror -fi - -git status - -# Clean out any non-committed changes so that we can checkout the correct -# branch -git reset --hard HEAD -ierror=$? -if [[ $ierror != 0 ]]; then - echo "There was an error clearing out any local uncommitted changes. The error code was: $ierror" - exit $ierror -fi - -git status - -# Get on the right local branch -git checkout -B ${TRILINOS_TARGET_BRANCH:?} origin/${TRILINOS_TARGET_BRANCH:?} -ierror=$? -if [[ $ierror != 0 ]]; then - echo "There was an error checking out and updating to the target remote branch. The error code was: $ierror" - exit $ierror -fi - -git status - -# Merge the souce branch into the local target branch -git merge --no-edit source_remote/${TRILINOS_SOURCE_BRANCH:?} -ierror=$? -if [[ $ierror != 0 ]]; then - echo "There was an issue merging changes from ${TRILINOS_SOURCE_REPO:?} ${TRILINOS_SOURCE_BRANCH:?} onto ${TRILINOS_TARGET_REPO:?} ${TRILINOS_TARGET_BRANCH:?}. The error code was: $ierror" - exit $ierror -fi - -#Need to compare expected SOURCE SHA to actual SHA! This will prevent a security hole. -#first get the most recent SHA on the source branch -declare source_sha=$(git rev-parse source_remote/${TRILINOS_SOURCE_BRANCH:?}) -echo "The most recent SHA for repo: ${TRILINOS_SOURCE_REPO:?} on branch: ${TRILINOS_SOURCE_BRANCH:?} is: $source_sha" -#Now see if the two shas match, unless TRILINOS_SOURCE_SHA is the default value of ABC -if [[ ABC != ${TRILINOS_SOURCE_SHA:?} ]]; then - if [[ $source_sha != ${TRILINOS_SOURCE_SHA:?} ]]; then - echo "The SHA ($source_sha) for the last commit on branch ${TRILINOS_SOURCE_BRANCH:?} in repo ${TRILINOS_SOURCE_REPO:?} is different than the expected SHA, which is: ${TRILINOS_SOURCE_SHA:?}. The error code was: $ierror" - exit -1 - fi -fi - -# may eventually want to verify the same target SHA too, but there would be -# advantages to testing newer versions of target instead of older if -# not all test jobs kick off right away - -#------------------------------ -# PR merge is complete -#------------------------------ - -# TODO: Split this script in two here (See Issue 3625 for reasons) - -#------------------------------ -# Doing setup for build -#------------------------------ - -cd ../ - -# Set up the full environment for the build -if [ "Trilinos_pullrequest_gcc_4.8.4" == "${JOB_BASE_NAME:?}" ] ; then - source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestGCC4.8.4TestingEnv.sh - ierror=$? - if [[ $ierror != 0 ]]; then - echo "There was an issue loading the gcc environment. The error code was: $ierror" - exit $ierror - fi -elif [ "Trilinos_pullrequest_gcc_4.9.3" == "${JOB_BASE_NAME:?}" ] ; then - source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestGCC4.9.3TestingEnv.sh - ierror=$? - if [[ $ierror != 0 ]]; then - echo "There was an issue loading the gcc environment. The error code was: $ierror" - exit $ierror - fi -elif [ "Trilinos_pullrequest_gcc_4.9.3_SERIAL" == "${JOB_BASE_NAME:?}" ] ; then - # TODO: Update this to use a 4.9.3 SERIAL testing environment script. - source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestGCC4.9.3TestingEnvSERIAL.sh - ierror=$? - if [[ $ierror != 0 ]]; then - echo "There was an issue loading the gcc environment. The error code was: $ierror" - exit $ierror - fi -elif [ "Trilinos_pullrequest_intel_17.0.1" == "${JOB_BASE_NAME:?}" ] ; then - source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestIntel17.0.1TestingEnv.sh - ierror=$? - if [[ $ierror != 0 ]]; then - echo "There was an issue loading the intel environment. The error code was: $ierror" - exit $ierror - fi -else - ierror=42 - echo "There was an issue loading the proper environment. The error code was: $ierror" - exit $ierror -fi - -# The single submit requires at least cmake 3.10.* -cmake --version - -module list - -# This crashes for the serial case since MPI variables are not set -# - See Issue #3625 -# - wcm: bugfix #3673 -regex=".*(_SERIAL)$" -if [[ ! ${JOB_BASE_NAME:?} =~ ${regex} ]]; then - echo "MPI type = sems-${SEMS_MPI_NAME:?}/${SEMS_MPI_VERSION:?}" -else - echo "Job is SERIAL" -fi - -CDASH_TRACK="Pull Request" -echo "CDash Track = ${CDASH_TRACK:?}" - - -#------------------------------------- -# Doing configure/build/test/submit -#------------------------------------- - -rm packageEnables.cmake -env \ - TRILINOS_DIR=${PR_TRILINOS_DIR} \ - TRILINOS_SCRIPTS_DIR=${TRILINOS_DRIVER_SRC_DIR} \ -${TRILINOS_DRIVER_SRC_DIR}/commonTools/framework/get-changed-trilinos-packages.sh \ - origin/${TRILINOS_TARGET_BRANCH:?} HEAD packageEnables.cmake -# NOTE: Above we use the git diff origin/..HEAD to give us the -# correct list of changed files. This works because this is done after -# merging the target branch and the soruce branch. With that, git diff will -# show all of the changes in the merged copy from what is in . -# This with give the correct set of changed files even if older versions of -# were merged multiple times into . It turns -# out that the only way to have git show the correct set of diffs in that case -# is to actually do the merge and then do the diff. - -ierror=$? -if [[ $ierror != 0 ]]; then - echo "There was an issue generating packageEnables.cmake. The error code was: $ierror" - exit $ierror -fi - -echo "Enabled packages:" -cmake -P packageEnables.cmake - -build_name="PR-$PULLREQUESTNUM-test-$JOB_BASE_NAME-$BUILD_NUMBER" - -#This should be runnable from anywhere, but all the tests so far have been from the -#same dir the simple_testing.cmake file was in. -cd TFW_testing_single_configure_prototype - -if [ "icc" == ${CC:?} ] ; then - CONFIG_SCRIPT=PullRequestLinuxIntelTestingSettings.cmake -else - if [ "Trilinos_pullrequest_gcc_4.8.4" == "${JOB_BASE_NAME:?}" ]; then - CONFIG_SCRIPT=PullRequestLinuxGCC4.8.4TestingSettings.cmake - elif [ "Trilinos_pullrequest_gcc_4.9.3" == "${JOB_BASE_NAME:?}" ]; then - CONFIG_SCRIPT=PullRequestLinuxGCC4.9.3TestingSettings.cmake - elif [ "Trilinos_pullrequest_gcc_4.9.3_SERIAL" == "${JOB_BASE_NAME:?}" ]; then - # TODO: Update this to use a 4.9.3 SERIAL testing environment script. - CONFIG_SCRIPT=PullRequestLinuxGCC4.9.3TestingSettingsSERIAL.cmake - fi -fi - -ctest -S simple_testing.cmake \ - -Dbuild_name=${build_name:?} \ - -Dskip_by_parts_submit=OFF \ - -Dskip_update_step=ON \ - -Ddashboard_model=Experimental \ - -Ddashboard_track="${CDASH_TRACK:?}" \ - -DPARALLEL_LEVEL=18 \ - -Dbuild_dir="${WORKSPACE:?}/pull_request_test" \ - -Dconfigure_script=${TRILINOS_DRIVER_SRC_DIR}/cmake/std/${CONFIG_SCRIPT:?} \ - -Dpackage_enables=../packageEnables.cmake \ - -Dsubprojects_file=../TFW_single_configure_support_scripts/package_subproject_list.cmake - -ierror=$? -if [[ $ierror != 0 ]]; then - echo "Single configure/build/test failed. The error code was: $ierror" - exit $ierror -fi - -#pushd Trilinos/cmake/ctest/drivers/parameterized -#ctest -S ctest_linux_nightly_generic.cmake From 0b978e81dbecd8b9a6531c8d594d5eddb186cc2c Mon Sep 17 00:00:00 2001 From: Jacob Domagala Date: Thu, 26 Aug 2021 15:47:28 +0200 Subject: [PATCH 31/70] Fix Panzer's tests and examples with UVM off (#9618) * Fix Panzer's tests and examples with UVM off * Extract views from MDField to avoid CUDA warnings --- .../example/square_mesh/square_mesh.cpp | 13 ++-- .../example/square_mesh/square_mesh_bc.cpp | 45 +++++++------- ...anzer_Integrator_BasisTimesScalar_impl.hpp | 6 +- .../src/evaluators/Panzer_Parameter_impl.hpp | 15 +++-- ...erDirichletResidual_BlockedTpetra_impl.hpp | 2 +- ...zer_ScatterResidual_BlockedTpetra_impl.hpp | 62 +++++++++---------- 6 files changed, 76 insertions(+), 67 deletions(-) diff --git a/packages/panzer/adapters-stk/example/square_mesh/square_mesh.cpp b/packages/panzer/adapters-stk/example/square_mesh/square_mesh.cpp index f8c9c706449a..81868d3b6645 100644 --- a/packages/panzer/adapters-stk/example/square_mesh/square_mesh.cpp +++ b/packages/panzer/adapters-stk/example/square_mesh/square_mesh.cpp @@ -57,7 +57,7 @@ typedef Kokkos::DynRankView FieldContainer; /** This example whows how to get vertex IDs for all the elements */ int main( int argc, char **argv ) -{ +{ using Teuchos::RCP; Teuchos::oblackholestream blackhole; @@ -80,7 +80,7 @@ int main( int argc, char **argv ) mesh->writeToExodus("blocked_mesh.exo"); unsigned dim = mesh->getDimension(); - std::vector eBlocks; + std::vector eBlocks; mesh->getElementBlockNames(eBlocks); // loop over all blocks @@ -91,7 +91,8 @@ int main( int argc, char **argv ) std::vector localIds; mesh->getMyElements(blockName,elements); - FieldContainer vertices("vertices",elements.size(),4,dim); + FieldContainer vertices("vertices",elements.size(),4,dim); + auto vertices_h = Kokkos::create_mirror_view(vertices); // loop over elements of this block for(std::size_t elm=0;elmgetNodeCoordinates(nodes[v]); - - for(unsigned d=0;dwriteToExodus("blocked_mesh_bc.exo"); unsigned dim = mesh->getDimension(); - std::vector sideSets; - std::vector elementBlocks; + std::vector sideSets; + std::vector elementBlocks; mesh->getSidesetNames(sideSets); mesh->getElementBlockNames(elementBlocks); // loop over all sidesets for(std::size_t blk=0;blk sideEntities; + + std::vector sideEntities; mesh->getMySides(sideName,eBlockId,sideEntities); - + // don't try to build worksets for sides that don't have // any entities - if(sideEntities.size()==0) { + if(sideEntities.size()==0) { std::cout << "SIDE = " << sideName << "/" << eBlockId << " " << std::endl; continue; } - + std::vector elements; std::vector localSideIds; panzer_stk::workset_utils::getSideElements(*mesh,eBlockId,sideEntities,localSideIds,elements); TEUCHOS_ASSERT(localSideIds.size()==elements.size()); - - FieldContainer vertices("vertices",elements.size(),4,dim); - + + FieldContainer vertices("vertices",elements.size(),4,dim); + auto vertices_h = Kokkos::create_mirror_view(vertices); + // loop over elements of this block std::vector localIds; for(std::size_t elm=0;elm nodes; stk::mesh::Entity element = elements[elm]; - + localIds.push_back(mesh->elementLocalId(element)); mesh->getNodeIdsForElement(element,nodes); - + TEUCHOS_ASSERT(nodes.size()==4); - + for(std::size_t v=0;vgetNodeCoordinates(nodes[v]); - - for(unsigned d=0;dwda); diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp index f056c0654b1e..fbf909903ac3 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp @@ -59,13 +59,13 @@ Parameter(const std::string parameter_name, const std::string field_name, const Teuchos::RCP& data_layout, panzer::ParamLib& param_lib) -{ +{ target_field = PHX::MDField(field_name, data_layout); - + this->addEvaluatedField(target_field); - + //param = panzer::accessScalarParameter(parameter_name,param_lib); - param = panzer::createAndRegisterScalarParameter(parameter_name,param_lib); + param = panzer::createAndRegisterScalarParameter(parameter_name,param_lib); // no initialization, this will be done by someone else (possibly the ME) later std::string n = "Parameter Evaluator"; @@ -76,16 +76,19 @@ Parameter(const std::string parameter_name, template void Parameter:: evaluateFields(typename TRAITS::EvalData workset) -{ +{ //std::cout << "Parameter::evalauteFields() ParamValue = " << param->getValue() << std::endl; + auto target_field_h = Kokkos::create_mirror_view(target_field.get_view()); for (index_t cell = 0; cell < workset.num_cells; ++cell) { for (typename PHX::MDField::size_type pt = 0; pt < target_field.extent(1); ++pt) { - target_field(cell,pt) = param->getValue(); + target_field_h(cell,pt) = param->getValue(); } } + Kokkos::deep_copy(PHX::as_view(target_field), target_field_h); + } //********************************************************************** diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedTpetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedTpetra_impl.hpp index 724a4f03ef9d..4883000198c6 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedTpetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedTpetra_impl.hpp @@ -279,7 +279,7 @@ evaluateFields(typename TRAITS::EvalData workset) const auto fieldOffsets = fieldOffsets_[fieldIndex]; const auto basisIndices = basisIndexForMDFieldOffsets_[fieldIndex]; const auto worksetLIDs = worksetLIDs_; - const auto fieldValues = scatterFields_[fieldIndex]; + const auto fieldValues = scatterFields_[fieldIndex].get_static_view(); const auto applyBC = applyBC_[fieldIndex].get_static_view(); const bool checkApplyBC = checkApplyBC_; diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_BlockedTpetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_BlockedTpetra_impl.hpp index 6362a7617b46..d553d5c2f75a 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_BlockedTpetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_BlockedTpetra_impl.hpp @@ -71,18 +71,18 @@ template panzer::ScatterResidual_BlockedTpetra:: ScatterResidual_BlockedTpetra(const Teuchos::RCP & /* indexer */, const Teuchos::ParameterList& p) -{ +{ std::string scatterName = p.get("Scatter Name"); Teuchos::RCP scatterHolder = Teuchos::rcp(new PHX::Tag(scatterName,Teuchos::rcp(new PHX::MDALayout(0)))); // get names to be evaluated - const std::vector& names = + const std::vector& names = *(p.get< Teuchos::RCP< std::vector > >("Dependent Names")); - Teuchos::RCP dl = + Teuchos::RCP dl = p.get< Teuchos::RCP >("Basis")->functional; - + // build the vector of fields that this is dependent on for (std::size_t eq = 0; eq < names.size(); ++eq) { PHX::MDField field = PHX::MDField(names[eq],dl); @@ -105,23 +105,23 @@ template panzer::ScatterResidual_BlockedTpetra:: ScatterResidual_BlockedTpetra(const Teuchos::RCP & indexer, const Teuchos::ParameterList& p) - : globalIndexer_(indexer) + : globalIndexer_(indexer) , globalDataKey_("Residual Scatter Container") -{ +{ std::string scatterName = p.get("Scatter Name"); - scatterHolder_ = + scatterHolder_ = Teuchos::rcp(new PHX::Tag(scatterName,Teuchos::rcp(new PHX::MDALayout(0)))); // get names to be evaluated - const std::vector& names = + const std::vector& names = *(p.get< Teuchos::RCP< std::vector > >("Dependent Names")); // grab map from evaluated names to field names fieldMap_ = p.get< Teuchos::RCP< std::map > >("Dependent Map"); - Teuchos::RCP dl = + Teuchos::RCP dl = p.get< Teuchos::RCP >("Basis")->functional; - + // build the vector of fields that this is dependent on scatterFields_.resize(names.size()); for (std::size_t eq = 0; eq < names.size(); ++eq) { @@ -143,7 +143,7 @@ ScatterResidual_BlockedTpetra(const Teuchos::RCP & inde // ********************************************************************** template void panzer::ScatterResidual_BlockedTpetra:: -postRegistrationSetup(typename TRAITS::SetupData d, +postRegistrationSetup(typename TRAITS::SetupData d, PHX::FieldManager& /* fm */) { const Workset & workset_0 = (*d.worksets_)[0]; @@ -192,12 +192,12 @@ preEvaluate(typename TRAITS::PreEvalData d) template void panzer::ScatterResidual_BlockedTpetra:: evaluateFields(typename TRAITS::EvalData workset) -{ +{ using Teuchos::RCP; using Teuchos::rcp_dynamic_cast; using Thyra::VectorBase; using Thyra::ProductVectorBase; - + const auto& localCellIds = this->wda(workset).cell_local_ids_k; const RCP> thyraBlockResidual = rcp_dynamic_cast >(blockedContainer_->get_f(),true); @@ -216,12 +216,12 @@ evaluateFields(typename TRAITS::EvalData workset) // Class data fields for lambda capture const auto& fieldOffsets = fieldOffsets_[fieldIndex]; const auto& worksetLIDs = worksetLIDs_; - const auto& fieldValues = scatterFields_[fieldIndex]; + const auto& fieldValues = scatterFields_[fieldIndex].get_static_view(); - Kokkos::parallel_for(Kokkos::RangePolicy(0,workset.num_cells), KOKKOS_LAMBDA (const int& cell) { + Kokkos::parallel_for(Kokkos::RangePolicy(0,workset.num_cells), KOKKOS_LAMBDA (const int& cell) { for(int basis=0; basis < static_cast(fieldOffsets.size()); ++basis) { - const int lid = worksetLIDs(cell,fieldOffsets(basis)); - Kokkos::atomic_add(&kokkosResidual(lid,0), fieldValues(cell,basis)); + const int lid = worksetLIDs(cell,fieldOffsets(basis)); + Kokkos::atomic_add(&kokkosResidual(lid,0), fieldValues(cell,basis)); } }); } @@ -237,21 +237,21 @@ ScatterResidual_BlockedTpetra(const Teuchos::RCP & inde const Teuchos::ParameterList& p) : globalIndexer_(indexer) , globalDataKey_("Residual Scatter Container") -{ +{ std::string scatterName = p.get("Scatter Name"); - scatterHolder_ = + scatterHolder_ = Teuchos::rcp(new PHX::Tag(scatterName,Teuchos::rcp(new PHX::MDALayout(0)))); // get names to be evaluated - const std::vector& names = + const std::vector& names = *(p.get< Teuchos::RCP< std::vector > >("Dependent Names")); // grab map from evaluated names to field names fieldMap_ = p.get< Teuchos::RCP< std::map > >("Dependent Map"); - Teuchos::RCP dl = + Teuchos::RCP dl = p.get< Teuchos::RCP >("Basis")->functional; - + // build the vector of fields that this is dependent on scatterFields_.resize(names.size()); for (std::size_t eq = 0; eq < names.size(); ++eq) { @@ -278,7 +278,7 @@ postRegistrationSetup(typename TRAITS::SetupData d, { const Workset & workset_0 = (*d.worksets_)[0]; const std::string blockId = this->wda(workset_0).block_id; - + fieldIds_.resize(scatterFields_.size()); fieldOffsets_.resize(scatterFields_.size()); productVectorBlockIndex_.resize(scatterFields_.size()); @@ -356,7 +356,7 @@ preEvaluate(typename TRAITS::PreEvalData d) template void panzer::ScatterResidual_BlockedTpetra:: evaluateFields(typename TRAITS::EvalData workset) -{ +{ using Teuchos::RCP; using Teuchos::rcp_dynamic_cast; using Thyra::VectorBase; @@ -364,7 +364,7 @@ evaluateFields(typename TRAITS::EvalData workset) using Thyra::BlockedLinearOpBase; const auto& localCellIds = this->wda(workset).cell_local_ids_k; - + const int numFieldBlocks = globalIndexer_->getNumFieldBlocks(); const RCP blockedContainer = blockedContainer_; const RCP> thyraBlockResidual = rcp_dynamic_cast >(blockedContainer_->get_f()); @@ -404,7 +404,7 @@ evaluateFields(typename TRAITS::EvalData workset) const auto tpetraCrsMatrix = rcp_dynamic_cast>(thyraTpetraOperator->getTpetraOperator(),true); const auto managedMatrix = tpetraCrsMatrix->getLocalMatrixDevice(); const auto managedGraph = managedMatrix.graph; - + // Create runtime unmanaged versions using StaticCrsGraphType = typename LocalMatrixType::StaticCrsGraphType; StaticCrsGraphType unmanagedGraph; @@ -416,7 +416,7 @@ evaluateFields(typename TRAITS::EvalData workset) LocalMatrixType unmanagedMatrix(managedMatrix.values.label(), managedMatrix.numCols(), unmanagedValues, unmanagedGraph); new (&hostJacTpetraBlocks(row,col)) LocalMatrixType(unmanagedMatrix); } - + hostBlockExistsInJac(row,col) = 1; } else { @@ -425,7 +425,7 @@ evaluateFields(typename TRAITS::EvalData workset) } } typename PHX::View - jacTpetraBlocks("panzer::ScatterResidual_BlockedTpetra::jacTpetraBlocks",numFieldBlocks,numFieldBlocks); + jacTpetraBlocks("panzer::ScatterResidual_BlockedTpetra::jacTpetraBlocks",numFieldBlocks,numFieldBlocks); Kokkos::deep_copy(jacTpetraBlocks,hostJacTpetraBlocks); Kokkos::deep_copy(blockExistsInJac,hostBlockExistsInJac); @@ -455,7 +455,7 @@ evaluateFields(typename TRAITS::EvalData workset) // Class data fields for lambda capture const PHX::View fieldOffsets = fieldOffsets_[fieldIndex]; const PHX::View worksetLIDs = worksetLIDs_; - const PHX::View fieldValues = scatterFields_[fieldIndex].get_static_view(); + const PHX::View fieldValues = scatterFields_[fieldIndex].get_static_view(); const PHX::View blockOffsets = blockOffsets_; Kokkos::parallel_for(Kokkos::RangePolicy(0,workset.num_cells), KOKKOS_LAMBDA (const int& cell) { @@ -464,7 +464,7 @@ evaluateFields(typename TRAITS::EvalData workset) for(int basis=0; basis < static_cast(fieldOffsets.size()); ++basis) { typedef PHX::MDField FieldType; - typename FieldType::array_type::reference_type tmpFieldVal = fieldValues(cell,basis); + typename FieldType::array_type::reference_type tmpFieldVal = fieldValues(cell,basis); const int rowLID = worksetLIDs(cell,fieldOffsets(basis)); if (haveResidual) @@ -479,7 +479,7 @@ evaluateFields(typename TRAITS::EvalData workset) for (int i=0; i < sensSize; ++i) { cLIDs[i] = worksetLIDs(cell,start+i); vals[i] = tmpFieldVal.fastAccessDx(start+i); - } + } jacTpetraBlocks(blockRowIndex,blockColIndex).sumIntoValues(rowLID,cLIDs,sensSize,vals,true,true); } } From 1c68dc2b58dc96651217e2ff6a2ba26d3a8c244d Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Thu, 26 Aug 2021 07:46:53 -0600 Subject: [PATCH 32/70] fixed parameter evaluator to run in parallel --- .../src/evaluators/Panzer_Parameter_impl.hpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp index fbf909903ac3..055ae3bb5adf 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp @@ -78,16 +78,13 @@ void Parameter:: evaluateFields(typename TRAITS::EvalData workset) { //std::cout << "Parameter::evalauteFields() ParamValue = " << param->getValue() << std::endl; - auto target_field_h = Kokkos::create_mirror_view(target_field.get_view()); + auto param_val = param->getValue(); + auto target_field_v = target_field.get_static_view(); - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (typename PHX::MDField::size_type pt = 0; - pt < target_field.extent(1); ++pt) { - target_field_h(cell,pt) = param->getValue(); - } - } - - Kokkos::deep_copy(PHX::as_view(target_field), target_field_h); + Kokkos::parallel_for ("Parameter", workset.num_cells, KOKKOS_LAMBDA (const int cell) { + for (std::size_t pt=0; pt Date: Thu, 26 Aug 2021 08:35:17 -0600 Subject: [PATCH 33/70] Fixed missing fence --- .../panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp index 055ae3bb5adf..417ada800cda 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp @@ -85,7 +85,7 @@ evaluateFields(typename TRAITS::EvalData workset) for (std::size_t pt=0; pt Date: Thu, 26 Aug 2021 09:47:32 -0600 Subject: [PATCH 34/70] conversions for UVM removal --- .../evaluators/Panzer_ConstantFlux_impl.hpp | 14 +++++-- .../Panzer_Neumann_Residual_impl.hpp | 37 ++++++++++++------- .../src/evaluators/Panzer_SubcellSum.cpp | 2 + .../src/evaluators/Panzer_SubcellSum_impl.hpp | 10 +++-- 4 files changed, 42 insertions(+), 21 deletions(-) diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ConstantFlux_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ConstantFlux_impl.hpp index ecbe371338cf..212c90645c9d 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ConstantFlux_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ConstantFlux_impl.hpp @@ -77,10 +77,16 @@ postRegistrationSetup( TEUCHOS_ASSERT(static_cast(flux.extent(2)) == values.size()); - for (int cell = 0; cell < flux.extent_int(0); ++cell) - for (int ip = 0; ip < flux.extent_int(1); ++ip) - for (int dim = 0; dim < flux.extent_int(2); ++dim) - flux(cell,ip,dim) = values[dim]; + auto flux_v = flux.get_static_view(); + + for (int dim = 0; dim < flux_v.extent_int(2); ++dim) { + auto val = values[dim]; + Kokkos::parallel_for ("ConstantFlux", flux.extent_int(0), KOKKOS_LAMBDA( const int cell) { + for (int ip = 0; ip < flux_v.extent_int(1); ++ip) + flux_v(cell,ip,dim) = val; + }); + } + Kokkos::fence(); } //********************************************************************** diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_Neumann_Residual_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_Neumann_Residual_impl.hpp index f37e3af32537..383e3286f9a2 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_Neumann_Residual_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_Neumann_Residual_impl.hpp @@ -112,31 +112,40 @@ evaluateFields( typename Traits::EvalData workset) { residual.deep_copy(ScalarT(0.0)); - - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (std::size_t ip = 0; ip < num_ip; ++ip) { - normal_dot_flux(cell,ip) = ScalarT(0.0); - for (std::size_t dim = 0; dim < num_dim; ++dim) { - normal_dot_flux(cell,ip) += normal(cell,ip,dim) * flux(cell,ip,dim); + auto normal_dot_flux_v = normal_dot_flux.get_static_view(); + auto normal_v = normal.get_static_view(); + auto flux_v = flux.get_static_view(); + + std::size_t l_num_ip = num_ip, l_num_dim = num_dim; + Kokkos::parallel_for("NeumannResidual", workset.num_cells, KOKKOS_LAMBDA (const index_t cell) { + for (std::size_t ip = 0; ip < l_num_ip; ++ip) { + normal_dot_flux_v(cell,ip) = ScalarT(0.0); + for (std::size_t dim = 0; dim < l_num_dim; ++dim) { + normal_dot_flux_v(cell,ip) += normal_v(cell,ip,dim) * flux_v(cell,ip,dim); + } } - } - } + }); // const Kokkos::DynRankView & weighted_basis = this->wda(workset).bases[basis_index]->weighted_basis; const Teuchos::RCP > bv = this->wda(workset).bases[basis_index]; - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (std::size_t basis = 0; basis < residual.extent(1); ++basis) { - for (std::size_t qp = 0; qp < num_ip; ++qp) { - residual(cell,basis) += normal_dot_flux(cell,qp)*bv->weighted_basis_scalar(cell,basis,qp); + auto weighted_basis_scalar = this->wda(workset).bases[basis_index]->weighted_basis_scalar.get_static_view(); + auto residual_v = residual.get_static_view(); + Kokkos::parallel_for("NeumannResidual", workset.num_cells, KOKKOS_LAMBDA (const index_t cell) { + for (std::size_t basis = 0; basis < residual_v.extent(1); ++basis) { + for (std::size_t qp = 0; qp < l_num_ip; ++qp) { + residual_v(cell,basis) += normal_dot_flux_v(cell,qp)*weighted_basis_scalar(cell,basis,qp); + } } - } - } + }); if(workset.num_cells>0) Intrepid2::FunctionSpaceTools:: integrate(residual.get_view(), normal_dot_flux.get_view(), (this->wda(workset).bases[basis_index])->weighted_basis_scalar.get_view()); + + Kokkos::fence(); + } //********************************************************************** diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_SubcellSum.cpp b/packages/panzer/disc-fe/src/evaluators/Panzer_SubcellSum.cpp index cd80267e68e8..39800d890988 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_SubcellSum.cpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_SubcellSum.cpp @@ -40,6 +40,8 @@ // *********************************************************************** // @HEADER +#include "Kokkos_View_Fad.hpp" + #include "PanzerDiscFE_config.hpp" #ifdef HAVE_PANZER_EXPLICIT_INSTANTIATION diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_SubcellSum_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_SubcellSum_impl.hpp index 0c76fb300294..7dcb17501c9d 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_SubcellSum_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_SubcellSum_impl.hpp @@ -98,16 +98,20 @@ evaluateFields( else indices = fieldPattern_->getSubcellIndices(workset.subcell_dim,this->wda(workset).subcell_index); + auto outField_h = Kokkos::create_mirror_view(outField.get_static_view()); + auto inField_h = Kokkos::create_mirror_view(inField.get_static_view()); + Kokkos::deep_copy(inField_h, inField.get_static_view()); for(index_t c=0;c Date: Thu, 26 Aug 2021 13:15:23 -0600 Subject: [PATCH 35/70] got poisson example UVM free --- .../Example_SimpleSolution_impl.hpp | 24 ++++++++----- .../Example_SimpleSource_impl.hpp | 18 ++++++---- .../adapters-stk/src/Panzer_STK_Utilities.cpp | 5 ++- .../src/evaluators/Panzer_DotProduct_impl.hpp | 35 ++++++++++++------- 4 files changed, 53 insertions(+), 29 deletions(-) diff --git a/packages/panzer/adapters-stk/example/PoissonExample/Example_SimpleSolution_impl.hpp b/packages/panzer/adapters-stk/example/PoissonExample/Example_SimpleSolution_impl.hpp index 2dca1cc86a28..a007470311d2 100644 --- a/packages/panzer/adapters-stk/example/PoissonExample/Example_SimpleSolution_impl.hpp +++ b/packages/panzer/adapters-stk/example/PoissonExample/Example_SimpleSolution_impl.hpp @@ -85,17 +85,23 @@ template void SimpleSolution::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < solution.extent_int(1); ++point) { + auto ip_coordinates = this->wda(workset).int_rules[ir_index]->ip_coordinates.get_static_view(); + auto solution_v = solution.get_static_view(); + auto solution_grad_v = solution_grad.get_static_view(); - const double & x = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,0); - const double & y = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,1); + Kokkos::parallel_for (workset.num_cells, KOKKOS_LAMBDA (const index_t cell) { + for (int point = 0; point < solution_v.extent_int(1); ++point) { - solution(cell,point) = std::sin(2*M_PI*x)*std::sin(2*M_PI*y); - solution_grad(cell,point,0) = 2.0*M_PI*std::cos(2*M_PI*x)*std::sin(2*M_PI*y); - solution_grad(cell,point,1) = 2.0*M_PI*std::sin(2*M_PI*x)*std::cos(2*M_PI*y); - } - } + const double & x = ip_coordinates(cell,point,0); + const double & y = ip_coordinates(cell,point,1); + + solution_v(cell,point) = std::sin(2*M_PI*x)*std::sin(2*M_PI*y); + solution_grad_v(cell,point,0) = 2.0*M_PI*std::cos(2*M_PI*x)*std::sin(2*M_PI*y); + solution_grad_v(cell,point,1) = 2.0*M_PI*std::sin(2*M_PI*x)*std::cos(2*M_PI*y); + } + } ); + + Kokkos::fence(); } //********************************************************************** diff --git a/packages/panzer/adapters-stk/example/PoissonExample/Example_SimpleSource_impl.hpp b/packages/panzer/adapters-stk/example/PoissonExample/Example_SimpleSource_impl.hpp index fe60c90fbd8f..e6de3a6a97ef 100644 --- a/packages/panzer/adapters-stk/example/PoissonExample/Example_SimpleSource_impl.hpp +++ b/packages/panzer/adapters-stk/example/PoissonExample/Example_SimpleSource_impl.hpp @@ -82,14 +82,18 @@ template void SimpleSource::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < source.extent_int(1); ++point) { - const double& x = workset.int_rules[ir_index]->ip_coordinates(cell,point,0); - const double& y = workset.int_rules[ir_index]->ip_coordinates(cell,point,1); + auto ip_coordinates = workset.int_rules[ir_index]->ip_coordinates.get_static_view(); + auto source_v = source.get_static_view(); - source(cell,point) = 8.0*M_PI*M_PI*std::sin(2.0*M_PI*x)*std::sin(2.0*M_PI*y); - } - } + Kokkos::parallel_for ("SimpleSource", workset.num_cells, KOKKOS_LAMBDA (const index_t cell) { + for (int point = 0; point < source_v.extent_int(1); ++point) { + const double& x = ip_coordinates(cell,point,0); + const double& y = ip_coordinates(cell,point,1); + + source_v(cell,point) = 8.0*M_PI*M_PI*std::sin(2.0*M_PI*x)*std::sin(2.0*M_PI*y); + } + }); + Kokkos::fence(); } //********************************************************************** diff --git a/packages/panzer/adapters-stk/src/Panzer_STK_Utilities.cpp b/packages/panzer/adapters-stk/src/Panzer_STK_Utilities.cpp index 67bcdeffedec..87915fbaf47f 100644 --- a/packages/panzer/adapters-stk/src/Panzer_STK_Utilities.cpp +++ b/packages/panzer/adapters-stk/src/Panzer_STK_Utilities.cpp @@ -123,6 +123,8 @@ void gather_in_block(const std::string & blockId, const panzer::GlobalIndexer& d // grab the field const std::vector & elmtOffset = dofMngr.getGIDFieldOffsets(blockId,fieldNum); fc[fieldStr] = Kokkos::DynRankView("fc",localCellIds.size(),elmtOffset.size()); + auto field = Kokkos::create_mirror_view(fc[fieldStr]); + // gather operation for each cell in workset for(std::size_t worksetCellIndex=0;worksetCellIndex:: evaluateFields( typename Traits::EvalData workset) { - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int p = 0; p < num_pts; ++p) { - vec_a_dot_vec_b(cell,p) = ScalarT(0.0); - for (int dim = 0; dim < num_dim; ++dim) - vec_a_dot_vec_b(cell,p) += vec_a(cell,p,dim) * vec_b(cell,p,dim); - - if(multiplier_field_on) - vec_a_dot_vec_b(cell,p) *= multiplier_value*multiplier_field(cell,p); - else - vec_a_dot_vec_b(cell,p) *= multiplier_value; - } - } + + auto vec_a_v = vec_a.get_static_view(); + auto vec_b_v = vec_b.get_static_view(); + auto vec_a_dot_vec_b_v = vec_a_dot_vec_b.get_static_view(); + auto multiplier_field_v = multiplier_field.get_static_view(); + + int l_num_pts = num_pts, l_num_dim = num_dim; + auto l_multiplier_field_on = multiplier_field_on; + auto l_multiplier_value = multiplier_value; + + Kokkos::parallel_for (workset.num_cells, KOKKOS_LAMBDA (const int cell) { + for (int p = 0; p < l_num_pts; ++p) { + vec_a_dot_vec_b_v(cell,p) = ScalarT(0.0); + for (int dim = 0; dim < l_num_dim; ++dim) + vec_a_dot_vec_b_v(cell,p) += vec_a_v(cell,p,dim) * vec_b_v(cell,p,dim); + + if(l_multiplier_field_on) + vec_a_dot_vec_b_v(cell,p) *= l_multiplier_value*multiplier_field_v(cell,p); + else + vec_a_dot_vec_b_v(cell,p) *= l_multiplier_value; + } + }); + Kokkos::fence(); } //********************************************************************** From bc20b5d519c13c25613de291d5f08e8e5cac4c8d Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Fri, 27 Aug 2021 11:13:05 -0600 Subject: [PATCH 36/70] got through discrete curl and grad --- .../src/evaluators/Panzer_DOFCurl_impl.hpp | 6 +-- .../src/solvers/MiniEM_DiscreteCurl.hpp | 37 ++++++++++++------- .../src/solvers/MiniEM_DiscreteGradient.hpp | 10 +++-- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_DOFCurl_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_DOFCurl_impl.hpp index 56f8430db4b3..407ed082fa9b 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_DOFCurl_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_DOFCurl_impl.hpp @@ -447,11 +447,11 @@ DOFCurl(const Teuchos::ParameterList & p) : offsets = *p.get > >("Jacobian Offsets Vector"); // allocate and copy offsets vector to Kokkos array - PHX::View offsets_array_nc - = PHX::View("offsets",offsets.size()); + PHX::View offsets_array("offsets",offsets.size()); + auto offsets_array_nc = Kokkos::create_mirror_view(offsets_array); for(std::size_t i=0;i dofCoords("dofCoords", 1, hdivCardinality, dim); Kokkos::DynRankView basisCoeffsLI("basisCoeffsLI", 1, hcurlCardinality, hdivCardinality); - Kokkos::DynRankView elemOrts("elemOrts", 1); + typename Kokkos::DynRankView::HostMirror elemOrts("elemOrts", 1); typename Kokkos::DynRankView::HostMirror elemNodes("elemNodes", 1, numElemVertices); - Kokkos::DynRankView fOrt("fOrt", hdivCardinality); - Kokkos::DynRankView ortJacobian("ortJacobian", 2, 2); + typename Kokkos::DynRankView::HostMirror fOrt("fOrt", hdivCardinality); + typename Kokkos::DynRankView::HostMirror ortJacobian("ortJacobian", 2, 2); // the ranks of these depend on dimension Kokkos::DynRankView dofCoeffs; @@ -155,13 +155,17 @@ void addDiscreteCurlToRequestHandler( curlAtDofCoords = Kokkos::DynRankView("curlAtDofCoords", 1, hcurlCardinality, hdivCardinality); } face_basis->getDofCoeffs(refDofCoeffs); - + auto dofCoeffs_h = Kokkos::create_mirror_view(dofCoeffs); + auto refDofCoeffs_h = Kokkos::create_mirror_view(refDofCoeffs); + Kokkos::deep_copy(dofCoeffs_h, dofCoeffs); + Kokkos::deep_copy(refDofCoeffs_h, refDofCoeffs); // set up the topology of each face in an element for computing DOF coefficients // in 2D coefficients are same as reference coefficients - Kokkos::DynRankView sub_topologies(Kokkos::ViewAllocateWithoutInitializing("sub_topologies"), hdivCardinality); + typename Kokkos::DynRankView::HostMirror + sub_topologies(Kokkos::ViewAllocateWithoutInitializing("sub_topologies"), hdivCardinality); if(dim < 3) for(int i = 0; i < hdivCardinality; i++) - dofCoeffs(0,i) = refDofCoeffs(i); + dofCoeffs_h(0,i) = refDofCoeffs_h(i); else { for(int iface = 0; iface < hdivCardinality; iface++){ shards::CellTopology sub_topology(topology.getCellTopologyData(dim-1,iface)); @@ -200,27 +204,34 @@ void addDiscreteCurlToRequestHandler( Intrepid2::Impl::OrientationTools::getJacobianOfOrientationMap(ortJacobian, sub_topologies(iface), fOrt(iface)); auto ortJacobianDet = ortJacobian(0,0)*ortJacobian(1,1)-ortJacobian(1,0)*ortJacobian(0,1); for(int idim = 0; idim < dim; idim++) - dofCoeffs(0,iface,idim) = refDofCoeffs(iface,idim)*ortJacobianDet; + dofCoeffs_h(0,iface,idim) = refDofCoeffs_h(iface,idim)*ortJacobianDet; } } //orient basis + Kokkos::DynRankView elemOrts_d("elemOrts_d", 1); + Kokkos::deep_copy(elemOrts_d, elemOrts); ots::modifyBasisByOrientation(curlAtDofCoords, curlAtDofCoordsNonOriented, - elemOrts, + elemOrts_d, edge_basis.get()); //get basis coefficients (dofs) for(int curlIter=0; curlIter fGIDs; face_ugi->getElementGIDs(elementIds[elemIter],fGIDs); std::vector eGIDs; edge_ugi->getElementGIDs(elementIds[elemIter],eGIDs); - auto fLIDs = face_ugi->getElementLIDs(elementIds[elemIter]); - auto eLIDs = edge_ugi->getElementLIDs(elementIds[elemIter]); + auto eLIDs_k = edge_ugi->getElementLIDs(elementIds[elemIter]); + auto fLIDs_k = face_ugi->getElementLIDs(elementIds[elemIter]); + auto eLIDs = Kokkos::create_mirror_view(eLIDs_k); + auto fLIDs = Kokkos::create_mirror_view(fLIDs_k); + Kokkos::deep_copy(eLIDs, eLIDs_k); + Kokkos::deep_copy(fLIDs, fLIDs_k); // need to know which faces are owned by this proc std::vector isOwned(fGIDs.size()); @@ -248,8 +259,8 @@ void addDiscreteCurlToRequestHandler( auto it = std::find(edges_on_face.begin(), edges_on_face.end(), curlIter); if(it!=edges_on_face.end()){ // normalize the values - if(std::abs(basisCoeffsLI(0,curlIter,fIter)) > 1.0e-10) - values[curlIter] = basisCoeffsLI(0,curlIter,fIter)*area_scaling; + if(std::abs(basisCoeffsLI_h(0,curlIter,fIter)) > 1.0e-10) + values[curlIter] = basisCoeffsLI_h(0,curlIter,fIter)*area_scaling; } } diff --git a/packages/panzer/mini-em/src/solvers/MiniEM_DiscreteGradient.hpp b/packages/panzer/mini-em/src/solvers/MiniEM_DiscreteGradient.hpp index f3ed29ba4109..c8f4b22b8934 100644 --- a/packages/panzer/mini-em/src/solvers/MiniEM_DiscreteGradient.hpp +++ b/packages/panzer/mini-em/src/solvers/MiniEM_DiscreteGradient.hpp @@ -116,9 +116,13 @@ void addDiscreteGradientToRequestHandler( eUgi->getElementGIDs(elementIds[elemIter],eGIDs); std::vector nGIDs; nUgi->getElementGIDs(elementIds[elemIter],nGIDs); - auto eLIDs = eUgi->getElementLIDs(elementIds[elemIter]); - auto nLIDs = nUgi->getElementLIDs(elementIds[elemIter]); - + auto eLIDs_k = eUgi->getElementLIDs(elementIds[elemIter]); + auto nLIDs_k = nUgi->getElementLIDs(elementIds[elemIter]); + auto eLIDs = Kokkos::create_mirror_view(eLIDs_k); + auto nLIDs = Kokkos::create_mirror_view(nLIDs_k); + Kokkos::deep_copy(eLIDs, eLIDs_k); + Kokkos::deep_copy(nLIDs, nLIDs_k); + std::vector isOwned; eUgi->ownedIndices(eGIDs,isOwned); From e72baf2f41c069ef0b6f56bb55c6da65f89e42de Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Fri, 27 Aug 2021 11:27:01 -0600 Subject: [PATCH 37/70] Fixed up some view of view errors --- .../evaluators/Panzer_Integrator_BasisTimesVector_impl.hpp | 4 +++- .../evaluators/Panzer_Integrator_CurlBasisDotVector_impl.hpp | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_Integrator_BasisTimesVector_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_Integrator_BasisTimesVector_impl.hpp index 8d6ae7d97def..cd9694495e57 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_Integrator_BasisTimesVector_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_Integrator_BasisTimesVector_impl.hpp @@ -261,8 +261,10 @@ namespace panzer using std::size_t; // Get the PHX::Views of the field multipliers. + auto kokkosFieldMults_h = Kokkos::create_mirror_view(kokkosFieldMults_); for (size_t i(0); i < fieldMults_.size(); ++i) - kokkosFieldMults_(i) = fieldMults_[i].get_static_view(); + kokkosFieldMults_h(i) = fieldMults_[i].get_static_view(); + Kokkos::deep_copy(kokkosFieldMults_, kokkosFieldMults_h); // Determine the number of quadrature points and the dimensionality of the // vector that we're integrating. diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_Integrator_CurlBasisDotVector_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_Integrator_CurlBasisDotVector_impl.hpp index 1e4a9be96f3f..f580a6cc599b 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_Integrator_CurlBasisDotVector_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_Integrator_CurlBasisDotVector_impl.hpp @@ -291,9 +291,10 @@ namespace panzer using std::vector; // Get the PHX::Views of the field multipliers. + auto kokkosFieldMults_h = Kokkos::create_mirror_view(kokkosFieldMults_); for (size_t i(0); i < fieldMults_.size(); ++i) - kokkosFieldMults_(i) = fieldMults_[i].get_static_view(); - + kokkosFieldMults_h(i) = fieldMults_[i].get_static_view(); + Kokkos::deep_copy(kokkosFieldMults_, kokkosFieldMults_h); // Determine the index in the Workset bases for our particular basis // name. if (not useDescriptors_) From ce6798b75462b6faf86e7003ade5c2f479edffb3 Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Mon, 30 Aug 2021 11:46:21 -0600 Subject: [PATCH 38/70] have test updated partially --- .../test/evaluator_tests/gather_solution.cpp | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/packages/panzer/adapters-stk/test/evaluator_tests/gather_solution.cpp b/packages/panzer/adapters-stk/test/evaluator_tests/gather_solution.cpp index e134ce6b7de7..e81a34e2abc9 100644 --- a/packages/panzer/adapters-stk/test/evaluator_tests/gather_solution.cpp +++ b/packages/panzer/adapters-stk/test/evaluator_tests/gather_solution.cpp @@ -53,6 +53,8 @@ // Epetra #include "Epetra_MpiComm.h" +// Kokkos +#include "Kokkos_View_Fad.hpp" // Panzer #include "PanzerAdaptersSTK_config.hpp" @@ -492,28 +494,36 @@ namespace panzer { TEST_EQUALITY(fieldData2_q1.extent(1),4); TEST_EQUALITY(fieldData1_q1.size(),Teuchos::as(4*4/numProcs)); TEST_EQUALITY(fieldData2_q1.size(),Teuchos::as(4*4/numProcs)); + + auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view()); + auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view()); + Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view()); + Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view()); for(unsigned int i=0;i fieldData_qedge1(fieldName_qedge1,basis_qedge1->functional); fm.getFieldData(fieldData_qedge1); - + + auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view()); + Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view()); + TEST_EQUALITY(fieldData_qedge1.extent(0),Teuchos::as(4/numProcs)); TEST_EQUALITY(fieldData_qedge1.extent(1),4); TEST_EQUALITY(fieldData_qedge1.size(),Teuchos::as(4*4/numProcs)); for(unsigned int cell=0;cell(fieldData1_q1); fm.getFieldData(fieldData2_q1); + auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view()); + auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view()); + Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view()); + Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view()); + for(unsigned int cell=0;cell(fieldData_qedge1); + auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view()); + Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view()); + for(unsigned int cell=0;cell Date: Mon, 30 Aug 2021 11:48:39 -0600 Subject: [PATCH 39/70] have some of gather soln working --- .../Panzer_GatherSolution_Epetra_impl.hpp | 116 +++++++----------- 1 file changed, 44 insertions(+), 72 deletions(-) diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Epetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Epetra_impl.hpp index f8e67957383b..e78236458af6 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Epetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Epetra_impl.hpp @@ -472,86 +472,58 @@ evaluateFields( // NOTE: A reordering of these loops will likely improve performance. The // "getGIDFieldOffsets may be expensive. However the "getElementGIDs" // can be cheaper. However the lookup for LIDs may be more expensive! - if (x_.is_null()) - { - // Gather operation for each cell in the workset. - for (int cell(0); cell < numCells; ++cell) - { - size_t cellLocalId(localCellIds[cell]); - auto LIDs = globalIndexer_->getElementLIDs(cellLocalId); - - // Loop over the fields to be gathered. - for (int fieldInd(0); fieldInd < numFields; ++fieldInd) - { - MDField& field = gatherFields_[fieldInd]; - int fieldNum(fieldIds_[fieldInd]); - const vector& elmtOffset = - globalIndexer_->getGIDFieldOffsets(blockId, fieldNum); - int numBases(elmtOffset.size()); - - // Loop over the basis functions and fill the fields. - for (int basis(0); basis < numBases; ++basis) - { - int offset(elmtOffset[basis]), lid(LIDs[offset]); - field(cell, basis) = (*xEvRoGed_)[lid]; - } // end loop over the basis functions - } // end loop over the fields to be gathered - } // end loop over localCellIds - } - else // if (not x_.is_null()) + auto LIDs = globalIndexer_->getLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); + // Loop over the fields to be gathered. + for (int fieldInd(0); fieldInd < numFields; ++fieldInd) { + MDField& field = gatherFields_[fieldInd]; + auto field_h = Kokkos::create_mirror_view(field.get_static_view()); + int fieldNum(fieldIds_[fieldInd]); + const vector& elmtOffset = + globalIndexer_->getGIDFieldOffsets(blockId, fieldNum); + int numBases(elmtOffset.size()); // Gather operation for each cell in the workset. for (int cell(0); cell < numCells; ++cell) { - size_t cellLocalId(localCellIds[cell]); - auto LIDs = globalIndexer_->getElementLIDs(cellLocalId); - - // Loop over the fields to be gathered. - for (int fieldInd(0); fieldInd < numFields; ++fieldInd) + size_t cellLocalId(localCellIds[cell]); + // Loop over the basis functions and fill the fields. + for (int basis(0); basis < numBases; ++basis) { - MDField& field = gatherFields_[fieldInd]; - int fieldNum(fieldIds_[fieldInd]); - const vector& elmtOffset = - globalIndexer_->getGIDFieldOffsets(blockId, fieldNum); - int numBases(elmtOffset.size()); - - // Loop over the basis functions and fill the fields. - for (int basis(0); basis < numBases; ++basis) - { - int offset(elmtOffset[basis]), lid(LIDs[offset]); - field(cell, basis) = (*x_)[lid]; - } // end loop over the basis functions - } // end loop over the fields to be gathered + int offset(elmtOffset[basis]), lid(LIDs_h(cellLocalId, offset)); + if (x_.is_null()) + field_h(cell, basis) = (*xEvRoGed_)[lid]; + else + field_h(cell, basis) = (*x_)[lid]; + } // end loop over the basis functions } // end loop over localCellIds - } // end if (x_.is_null()) or not - // Deal with the tangent fields. - if (hasTangentFields_) - { - // Loop over the cells in the workset. - for (int cell(0); cell < numCells; ++cell) - { - // Loop over the fields to be gathered. - for (int fieldInd(0); fieldInd < numFields; ++fieldInd) - { - MDField& field = gatherFields_[fieldInd]; - int fieldNum(fieldIds_[fieldInd]); - const vector& elmtOffset = - globalIndexer_->getGIDFieldOffsets(blockId, fieldNum); - int numBases(elmtOffset.size()); - - // Loop over the basis functions. - for (int basis(0); basis < numBases; ++basis) - { - // Loop over the tangent fields and fill them in. - int numTangentFields(tangentFields_[fieldInd].size()); - for (int i(0); i < numTangentFields; ++i) - field(cell, basis).fastAccessDx(i) = + // Deal with the tangent fields. + if (hasTangentFields_) { + // Loop over the tangent fields and fill them in. + int numTangentFields(tangentFields_[fieldInd].size()); + for (int i(0); i < numTangentFields; ++i){ + auto tf = Kokkos::create_mirror_view(tangentFields_[fieldInd][i].get_static_view()); + Kokkos::deep_copy(tf, tangentFields_[fieldInd][i].get_static_view()); + // Loop over the cells in the workset. + for (int cell(0); cell < numCells; ++cell) { + // Loop over the fields to be gathered. + int fieldNum(fieldIds_[fieldInd]); + const vector& elmtOffset = + globalIndexer_->getGIDFieldOffsets(blockId, fieldNum); + int numBases(elmtOffset.size()); + + // Loop over the basis functions. + for (int basis(0); basis < numBases; ++basis){ + field_h(cell, basis).fastAccessDx(i) = tangentFields_[fieldInd][i](cell, basis).val(); - } // end loop over the basis functions - } // end loop over the fields to be gathered - } // end loop over the cells in the workset - } // end if (hasTangentFields_) + } // end loop over the basis functions + } // end loop over the cells in the workset + } // end loop over numTangentFields + } // end if (hasTangentFields_) + Kokkos::deep_copy(field.get_static_view(), field_h); + } // end loop over the fields to be gathered } // end of evaluateFields() (Tangent Specialization) /////////////////////////////////////////////////////////////////////////////// From 137445eb2078e164f719ad7d2da7be5c0519f33e Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Mon, 30 Aug 2021 11:50:44 -0600 Subject: [PATCH 40/70] have some of gather tan working --- .../Panzer_GatherTangent_Epetra.cpp | 2 ++ .../Panzer_GatherTangent_Epetra_impl.hpp | 25 +++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherTangent_Epetra.cpp b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherTangent_Epetra.cpp index b95c763e42a5..8c40c9592b9d 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherTangent_Epetra.cpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherTangent_Epetra.cpp @@ -40,6 +40,8 @@ // *********************************************************************** // @HEADER +#include "Kokkos_View_Fad.hpp" + #include "PanzerDiscFE_config.hpp" #ifdef HAVE_PANZER_EXPLICIT_INSTANTIATION diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherTangent_Epetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherTangent_Epetra_impl.hpp index c50d3e80b1f2..dc14f924306b 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherTangent_Epetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherTangent_Epetra_impl.hpp @@ -217,15 +217,17 @@ evaluateFields( // can be cheaper. However the lookup for LIDs may be more expensive! // Gather operation for each cell in the workset. - for (int cell(0); cell < numCells; ++cell) + auto LIDs = globalIndexer_->getLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); + // Loop over the fields to be gathered. + for (int fieldIndex(0); fieldIndex < numFields; ++fieldIndex) { - size_t cellLocalId(localCellIds[cell]); - auto LIDs = globalIndexer_->getElementLIDs(cellLocalId); - - // Loop over the fields to be gathered. - for (int fieldIndex(0); fieldIndex < numFields; ++fieldIndex) + MDField& field = gatherFields_[fieldIndex]; + auto field_h = Kokkos::create_mirror_view(field.get_static_view()); + for (int cell(0); cell < numCells; ++cell) { - MDField& field = gatherFields_[fieldIndex]; + size_t cellLocalId(localCellIds[cell]); int fieldNum(fieldIds_[fieldIndex]); const vector& elmtOffset = globalIndexer_->getGIDFieldOffsets(blockId, fieldNum); @@ -234,11 +236,12 @@ evaluateFields( // Loop over the basis functions and fill the fields. for (int basis(0); basis < numBases; ++basis) { - int offset(elmtOffset[basis]), lid(LIDs[offset]); - field(cell, basis) = (*dxdpEvRoGed_)[lid]; + int offset(elmtOffset[basis]), lid(LIDs_h(cellLocalId, offset)); + field_h(cell, basis) = (*dxdpEvRoGed_)[lid]; } // end loop over the basis functions - } // end loop over the fields to be gathered - } // end loop over the cells in the workset + } // end loop over the cells in the workset + Kokkos::deep_copy(field.get_static_view(), field_h); + } // end loop over the fields to be gathered } // end of evaluateFields() #endif // __Panzer_GatherTangent_Epetra_impl_hpp__ From fba1ecfc2b8d3f8289ec0ae10eccc92cefb73dbe Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Mon, 30 Aug 2021 12:04:09 -0600 Subject: [PATCH 41/70] Fixed one unit test --- .../test/evaluator_tests/gather_solution.cpp | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/packages/panzer/adapters-stk/test/evaluator_tests/gather_solution.cpp b/packages/panzer/adapters-stk/test/evaluator_tests/gather_solution.cpp index e81a34e2abc9..9a1a222efa38 100644 --- a/packages/panzer/adapters-stk/test/evaluator_tests/gather_solution.cpp +++ b/packages/panzer/adapters-stk/test/evaluator_tests/gather_solution.cpp @@ -581,17 +581,22 @@ namespace panzer { fm.getFieldData(fieldData1_q1); fm.getFieldData(fieldData2_q1); + auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view()); + auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view()); + Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view()); + Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view()); + for(unsigned int cell=0;cell(fieldData_qedge1); + auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view()); + Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view()); + for(unsigned int cell=0;cell Date: Mon, 30 Aug 2021 12:17:47 -0600 Subject: [PATCH 42/70] completed conversion --- .../adapters-stk/test/evaluator_tests/gather_solution.cpp | 4 ++-- .../src/evaluators/Panzer_GatherSolution_Epetra_impl.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/panzer/adapters-stk/test/evaluator_tests/gather_solution.cpp b/packages/panzer/adapters-stk/test/evaluator_tests/gather_solution.cpp index 9a1a222efa38..1b877be5fb78 100644 --- a/packages/panzer/adapters-stk/test/evaluator_tests/gather_solution.cpp +++ b/packages/panzer/adapters-stk/test/evaluator_tests/gather_solution.cpp @@ -603,8 +603,8 @@ namespace panzer { for(unsigned int cell=0;cell Date: Mon, 30 Aug 2021 15:48:31 -0600 Subject: [PATCH 43/70] Automatic snapshot commit from seacas at f2312cbcb8 Origin repo remote tracking branch: 'origin/master' Origin repo remote repo URL: 'origin = https://github.com/gsjaardema/seacas' At commit: commit f2312cbcb8f81ba04b186435fc66c1e01844a2cd Author: Greg Sjaardema Date: Mon Aug 30 13:25:51 2021 -0600 Summary: IOSS: Eliminate some compiler warnings * Clang-format changes * EJOIN: Remove unused variables * EPU: - clang-format - remove older exodus storage format write capability - fix time step subset code * EXODIFF: - Convert most raw data access to use std::vector; eliminate new/delete/leaks - Make error/warning handling more consistent - Change 'Elmt' to 'Element' in most places - Move FileInfo class to suplib_cpp - Output now calculates max length of all variable names for better alignment of diff output - Consolidate repeated code - Separated out the 'diff' reporting code from the 'summary' reporting code to reduce function complexity * MAPVAR: Address codacy static analyzer recommendations, spelling fix * NEM_SLICE: Address codacy static analyzer recommendations * ZELLIJ: Add missing include * Doxygen -- update Doxyfile's * CMAKE: Move `InstallSymLink.cmake` into seacas package hierarchy * IOSS: - Add initial support for Catalyst2 IOSS database type - Refactor the Compare class for better more understandable output - Codacy static analyzer recommendations - Add support for Hex9 element * APREPRO: Address codacy recommendations * EXODUS: - Better const correctness - Codacy recommendations * SUPES: -- codacy fixes * SUPLIB: -- codacy fixes -- Windows build improvements --- packages/seacas/IossProperties.md | 9 +- packages/seacas/MAPVAR.md | 27 +- packages/seacas/NetCDF-Mapping.md | 82 +- packages/seacas/README.md | 72 +- .../applications/conjoin/CJ_Internals.C | 77 +- packages/seacas/applications/ejoin/EJoin.C | 12 +- .../seacas/applications/epu/EP_Internals.C | 76 +- packages/seacas/applications/epu/EP_Version.h | 4 +- packages/seacas/applications/epu/epu.C | 13 +- .../applications/exodiff/CMakeLists.txt | 1 - .../applications/exodiff/ED_SystemInterface.C | 22 - .../seacas/applications/exodiff/ED_Version.h | 4 +- packages/seacas/applications/exodiff/check.C | 220 +-- .../seacas/applications/exodiff/create_file.C | 40 +- .../seacas/applications/exodiff/edge_block.C | 26 +- .../seacas/applications/exodiff/edge_block.h | 3 - .../seacas/applications/exodiff/exoII_read.C | 94 +- .../seacas/applications/exodiff/exoII_read.h | 63 +- .../seacas/applications/exodiff/exo_block.C | 48 +- .../seacas/applications/exodiff/exo_block.h | 22 +- .../seacas/applications/exodiff/exo_entity.C | 10 +- .../seacas/applications/exodiff/exo_entity.h | 18 +- .../seacas/applications/exodiff/exodiff.C | 1555 ++++++++--------- .../seacas/applications/exodiff/face_block.C | 26 +- .../seacas/applications/exodiff/face_block.h | 5 +- packages/seacas/applications/exodiff/map.C | 380 ++-- packages/seacas/applications/exodiff/map.h | 22 +- .../seacas/applications/exodiff/node_set.C | 18 +- .../seacas/applications/exodiff/node_set.h | 4 +- .../seacas/applications/exodiff/side_set.C | 29 +- .../seacas/applications/exodiff/side_set.h | 4 +- .../seacas/applications/exodiff/stringx.h | 5 +- packages/seacas/applications/exodiff/util.C | 38 +- packages/seacas/applications/exodiff/util.h | 5 +- .../seacas/applications/mapvar-kd/optkd.c | 52 +- .../seacas/applications/nem_slice/elb_graph.C | 7 +- .../applications/nem_slice/elb_loadbal.C | 22 +- .../seacas/applications/nem_slice/elb_main.C | 4 - packages/seacas/applications/slice/Slice.C | 1 + packages/seacas/applications/zellij/Cell.h | 1 + packages/seacas/applications/zellij/Doxyfile | 215 +-- .../seacas/applications/zellij/UnitCell.h | 1 + packages/seacas/applications/zellij/Zellij.C | 2 +- packages/seacas/applications/zellij/Zellij.md | 100 +- packages/seacas/cmake/InstallSymLink.cmake | 41 + .../seacas/cmake/tpls/FindTPLCatalyst2.cmake | 89 + .../seacas/libraries/aprepro_lib/Doxyfile | 209 ++- .../libraries/aprepro_lib/apr_aprepro.cc | 4 +- .../libraries/aprepro_lib/apr_builtin.cc | 4 +- .../libraries/aprepro_lib/apr_getline_int.c | 95 +- .../seacas/libraries/chaco/eigen/scale_diag.c | 11 +- .../seacas/libraries/chaco/eigen/sturmcnt.c | 41 +- .../seacas/libraries/chaco/util/seconds.c | 4 +- .../libraries/exoIIv2for32/src/exo_jack_32.c | 12 +- packages/seacas/libraries/exodus/Doxyfile | 211 ++- .../seacas/libraries/exodus/customdoxygen.css | 686 -------- .../libraries/exodus/include/exodusII.h | 174 +- .../libraries/exodus/include/exodusII_int.h | 172 +- .../libraries/exodus/include/polyhedra.md | 69 +- .../seacas/libraries/exodus/src/ex_close.c | 4 +- .../seacas/libraries/exodus/src/ex_conv.c | 20 +- .../seacas/libraries/exodus/src/ex_copy.c | 115 +- .../seacas/libraries/exodus/src/ex_create.c | 5 + .../libraries/exodus/src/ex_create_par.c | 5 + .../libraries/exodus/src/ex_get_attr_names.c | 6 +- .../libraries/exodus/src/ex_get_cmap_params.c | 17 +- .../libraries/exodus/src/ex_get_coord.c | 16 +- .../seacas/libraries/exodus/src/ex_get_map.c | 15 +- .../exodus/src/ex_int_get_block_param.c | 7 +- .../seacas/libraries/exodus/src/ex_ne_util.c | 10 +- .../exodus/src/ex_put_all_var_param.c | 10 +- .../libraries/exodus/src/ex_put_attribute.c | 6 +- .../libraries/exodus/src/ex_put_cmap_params.c | 5 +- .../exodus/src/ex_put_cmap_params_cc.c | 7 +- .../exodus/src/ex_put_concat_elem_block.c | 2 +- .../seacas/libraries/exodus/src/ex_put_conn.c | 5 +- .../libraries/exodus/src/ex_put_coord_names.c | 2 +- .../exodus/src/ex_put_coordinate_frames.c | 4 +- .../exodus/src/ex_put_eb_info_global.c | 2 +- .../libraries/exodus/src/ex_put_elem_cmap.c | 4 +- .../seacas/libraries/exodus/src/ex_put_info.c | 2 +- .../libraries/exodus/src/ex_put_init_ext.c | 13 +- .../libraries/exodus/src/ex_put_init_info.c | 2 +- .../exodus/src/ex_put_loadbal_param_cc.c | 8 +- .../libraries/exodus/src/ex_put_names.c | 2 +- .../libraries/exodus/src/ex_put_node_cmap.c | 4 +- .../exodus/src/ex_put_ns_param_global.c | 3 +- .../libraries/exodus/src/ex_put_num_map.c | 34 +- .../exodus/src/ex_put_partial_node_cmap.c | 4 +- .../exodus/src/ex_put_partial_num_map.c | 6 +- .../exodus/src/ex_put_processor_elem_maps.c | 3 +- .../exodus/src/ex_put_processor_node_maps.c | 4 +- .../src/ex_put_reduction_variable_names.c | 2 +- .../exodus/src/ex_put_ss_param_global.c | 3 +- .../libraries/exodus/src/ex_put_truth_table.c | 3 +- .../exodus/src/ex_put_variable_names.c | 2 +- .../seacas/libraries/exodus/src/ex_utils.c | 203 +-- .../libraries/exodus/test/ReadEdgeFace.c | 115 +- .../libraries/exodus/test/test-compress.dmp | 1 - .../libraries/exodus/test/testwt-assembly.c | 39 +- .../libraries/exodus/test/testwt-groups.c | 161 +- .../exodus/test/testwt-localization.C | 10 +- .../exodus/test/testwt-nface-nside.c | 84 +- .../seacas/libraries/exodus/test/testwt.c | 150 +- .../libraries/exodus_for/src/exo_jack.c | 41 +- packages/seacas/libraries/ioss/Doxyfile | 213 ++- .../libraries/ioss/cmake/Dependencies.cmake | 4 +- .../ioss/cmake/SEACASIoss_config.h.in | 2 + .../seacas/libraries/ioss/src/CMakeLists.txt | 10 +- .../seacas/libraries/ioss/src/Ioss_Assembly.C | 7 +- .../seacas/libraries/ioss/src/Ioss_Compare.C | 1179 ++++++++----- .../libraries/ioss/src/Ioss_CopyDatabase.C | 10 +- .../libraries/ioss/src/Ioss_DatabaseIO.C | 4 +- .../libraries/ioss/src/Ioss_Decomposition.C | 9 +- .../libraries/ioss/src/Ioss_FaceGenerator.C | 3 +- .../seacas/libraries/ioss/src/Ioss_Field.C | 54 +- .../seacas/libraries/ioss/src/Ioss_Field.h | 14 +- .../seacas/libraries/ioss/src/Ioss_Glob.h | 40 +- .../seacas/libraries/ioss/src/Ioss_Hex8.C | 2 +- .../seacas/libraries/ioss/src/Ioss_Hex9.C | 170 ++ .../seacas/libraries/ioss/src/Ioss_Hex9.h | 58 + .../libraries/ioss/src/Ioss_IOFactory.C | 6 + .../libraries/ioss/src/Ioss_Initializer.C | 1 + .../libraries/ioss/src/Ioss_MeshCopyOptions.h | 21 +- .../seacas/libraries/ioss/src/Ioss_Property.C | 4 +- .../seacas/libraries/ioss/src/Ioss_Property.h | 6 +- .../seacas/libraries/ioss/src/Ioss_Region.C | 2 + .../libraries/ioss/src/Ioss_ScopeGuard.h | 2 +- .../ioss/src/Ioss_StandardElementTypes.h | 1 + .../seacas/libraries/ioss/src/Ioss_Utils.C | 6 +- .../seacas/libraries/ioss/src/Ioss_Utils.h | 1 - .../ioss/src/catalyst/CMakeLists.txt | 24 + .../ioss/src/catalyst/Iocatalyst_DatabaseIO.C | 783 +++++++++ .../ioss/src/catalyst/Iocatalyst_DatabaseIO.h | 170 ++ .../ioss/src/catalyst/Iocatalyst_IOFactory.C | 55 + .../ioss/src/catalyst/Iocatalyst_IOFactory.h | 35 + .../src/catalyst/Iocatalyst_Initializer.C | 35 + .../src/catalyst/Iocatalyst_Initializer.h | 28 + .../src/catalyst/cmake/Dependencies.cmake | 3 + .../ioss/src/cgns/Iocgns_DatabaseIO.C | 14 +- .../ioss/src/cgns/Iocgns_ParallelDatabaseIO.C | 4 +- .../libraries/ioss/src/cgns/Iocgns_Utils.C | 5 +- .../ioss/src/exodus/Ioex_BaseDatabaseIO.C | 4 +- .../ioss/src/exodus/Ioex_Internals.C | 1462 ++++++++-------- .../ioss/src/faodel/Iofaodel_Utils.C | 5 +- .../ioss/src/generated/Iogn_GeneratedMesh.C | 197 ++- .../ioss/src/generated/Iogn_GeneratedMesh.h | 10 + .../libraries/ioss/src/init/CMakeLists.txt | 10 +- .../ioss/src/init/Ionit_Initializer.C | 9 +- .../libraries/ioss/src/main/cgns_decomp.C | 3 +- .../seacas/libraries/ioss/src/main/io_info.C | 22 +- .../libraries/ioss/src/main/io_modify.C | 11 +- .../seacas/libraries/ioss/src/main/io_shell.C | 11 +- .../libraries/ioss/src/main/io_shell_ts.C | 4 +- .../seacas/libraries/ioss/src/main/skinner.C | 2 +- .../ioss/src/pamgen/Iopg_DatabaseIO.C | 2 +- .../libraries/ioss/src/utest/CMakeLists.txt | 3 +- .../ioss/src/visualization/utils/Iovs_Utils.C | 17 +- packages/seacas/libraries/mapvarlib/flgrad.f | 4 +- .../libraries/supes/ext_lib/getline_int.c | 162 +- .../seacas/libraries/suplib_c/add_to_log.c | 4 +- packages/seacas/libraries/suplib_c/adler.c | 5 +- .../libraries/suplib_cpp/CMakeLists.txt | 1 + .../suplib_cpp}/FileInfo.C | 0 .../suplib_cpp}/FileInfo.h | 0 .../seacas/libraries/suplib_cpp/scopeguard.h | 7 +- packages/seacas/scripts/exodus3.in.py | 37 +- packages/seacas/scripts/tests/test_exodus3.py | 46 +- 168 files changed, 6174 insertions(+), 5276 deletions(-) create mode 100644 packages/seacas/cmake/InstallSymLink.cmake create mode 100644 packages/seacas/cmake/tpls/FindTPLCatalyst2.cmake delete mode 100644 packages/seacas/libraries/exodus/customdoxygen.css create mode 100644 packages/seacas/libraries/ioss/src/Ioss_Hex9.C create mode 100644 packages/seacas/libraries/ioss/src/Ioss_Hex9.h create mode 100644 packages/seacas/libraries/ioss/src/catalyst/CMakeLists.txt create mode 100644 packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_DatabaseIO.C create mode 100644 packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_DatabaseIO.h create mode 100644 packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_IOFactory.C create mode 100644 packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_IOFactory.h create mode 100644 packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_Initializer.C create mode 100644 packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_Initializer.h create mode 100644 packages/seacas/libraries/ioss/src/catalyst/cmake/Dependencies.cmake rename packages/seacas/{applications/exodiff => libraries/suplib_cpp}/FileInfo.C (100%) rename packages/seacas/{applications/exodiff => libraries/suplib_cpp}/FileInfo.h (100%) diff --git a/packages/seacas/IossProperties.md b/packages/seacas/IossProperties.md index 7567948760e1..948d80107745 100644 --- a/packages/seacas/IossProperties.md +++ b/packages/seacas/IossProperties.md @@ -1,8 +1,3 @@ -## Copyright(C) 1999-2020 National Technology & Engineering Solutions -## of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with -## NTESS, the U.S. Government retains certain rights in this software. -## -## See packages/seacas/LICENSE for details ## General Properties Property | Value | Description @@ -115,13 +110,13 @@ of the environment variable `IOSS_PROPERTIES`. The value of the variable is one or more colon-separated property/property-value pairs. For example, to set the `DECOMPOSITION_METHOD` and the `FILE_TYPE` externally, the following would be used: -``` +```bash export IOSS_PROPERTIES="DECOMPOSITION_METHOD=rib:FILE_TYPE=netcdf4" ``` If the environment variable is set correctly, there should be an informational message output during running of the application similar to: -``` +```bash IOSS: Adding property 'DECOMPOSITION_METHOD' with value 'rib' IOSS: Adding property 'FILE_TYPE' with value 'netcdf4' ``` diff --git a/packages/seacas/MAPVAR.md b/packages/seacas/MAPVAR.md index 87a284d380bd..e8185262c149 100644 --- a/packages/seacas/MAPVAR.md +++ b/packages/seacas/MAPVAR.md @@ -6,19 +6,16 @@ models. As I see it, it does the following for the interpolation: - * For all blocks + * For all blocks + * For all time steps + * For all nodal variables + * Iterate all nodes in this block; map from A->B + * Write values for *all* B nodes at this step for this variable - * For all time steps - - * For all nodal variables - * Iterate all nodes in this block; map from A->B - * Write values for *all* B nodes at this step for this variable - - * For all time steps - - * For all element variables - * Iterate all elements in this block; map from A->B - * Write values for all elements in this block at this step for this variable + * For all time steps + * For all element variables + * Iterate all elements in this block; map from A->B + * Write values for all elements in this block at this step for this variable This works for element variables since the exodus API can output elements a block and variable at a time. For nodes, it doesn't work @@ -36,9 +33,9 @@ shared between multiple element blocks -- it will only get the interpolated value from the last block. Now, what to do... - * I think that the Percept code can do some mapping from mesh to mesh... - * Klugy, but can do a timestep at a time and then rejoin all timesteps using `conjoin` - * Klugy, but can subset down to one block / mesh and then run mapvar on each submesh and then join using `ejoin` +* I think that the Percept code can do some mapping from mesh to mesh... +* Klugy, but can do a timestep at a time and then rejoin all timesteps using `conjoin` +* Klugy, but can subset down to one block / mesh and then run mapvar on each submesh and then join using `ejoin` Sorry for the bearer of bad news, but hopefully there is a path to get done what you need... diff --git a/packages/seacas/NetCDF-Mapping.md b/packages/seacas/NetCDF-Mapping.md index 08756f555619..f67b6f110350 100644 --- a/packages/seacas/NetCDF-Mapping.md +++ b/packages/seacas/NetCDF-Mapping.md @@ -1,20 +1,15 @@ -# Copyright(C) 1999-2020 National Technology & Engineering Solutions -# of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with -# NTESS, the U.S. Government retains certain rights in this software. -# -# See packages/seacas/LICENSE for details # Mapping of exodus entities onto NetCDF If you are using NetCDF-4.5.1 or later, then you can ignore the information in this file. The distributed version of netcdf sets the following limits on dimensions and variables: - * `#define NC_MAX_DIMS 1024` - * `#define NC_MAX_VARS 8192` +* `#define NC_MAX_DIMS 1024` +* `#define NC_MAX_VARS 8192` For use with Exodus, it is recommended that these be increased to: - * `#define NC_MAX_DIMS 65536` - * `#define NC_MAX_VARS 524288` +* `#define NC_MAX_DIMS 65536` +* `#define NC_MAX_VARS 524288` The reason for these increases is due to the mapping of Exodus onto NetCDF. The sections below show the number of Dimensions (controlled @@ -22,51 +17,44 @@ by NC_MAX_DIMS) and Variables (controlled by NC_MAX_VARS) that are used in an Exodus file. ## Entities - * A mesh-entity is an individual node, edge, face, or element. - * An entity is a set or block consisting of a single mesh-entity type. - * Each entity can have variables, maps, and attributes which contain an entry per mesh-entity. - * Each entity has an optional name and a required id (32-bit or 64-bit )which is non-negative. - * A mesh-entity can be in one and only one entity block, - * A mesh-entity can be in zero or more entity sets. - * Currently there is only a single implicit node block containing all nodes in the model. +* A mesh-entity is an individual node, edge, face, or element. +* An entity is a set or block consisting of a single mesh-entity type. +* Each entity can have variables, maps, and attributes which contain an entry per mesh-entity. +* Each entity has an optional name and a required id (32-bit or 64-bit )which is non-negative. +* A mesh-entity can be in one and only one entity block, +* A mesh-entity can be in zero or more entity sets. +* Currently there is only a single implicit node block containing all nodes in the model. ## Dimensions: (NC_MAX_DIMS) -* There are about 10 standard dimensions in every file. -* plus one for each set plus one if any attributes -* plus two for each block plus one if any attributes -* plus one for each transient variable on an entity (node, node set, element block, element set, ...) +* There are about 10 standard dimensions in every file. +* plus one for each set plus one if any attributes +* plus two for each block plus one if any attributes +* plus one for each transient variable on an entity (node, node set, element block, element set, ...) ## Variables: (NC_MAX_VARS) -* There are a few standard dimensions - * times - * names of each entity type (block set) - * ids of each entity type (block set) - * status of each entity type (block set) - * #ndim coordinates (1,2,3) - -* Each block adds 1 + 2*#attr_on_block + #var_on_block - -* Each set adds 2 + 2*#attr_on_set + #var_on_set - -* Each sideset add 3 + 2*#attr_on_sset + #var_on_sset - -* Each map adds 1 +* There are a few standard dimensions + * times + * names of each entity type (block set) + * ids of each entity type (block set) + * status of each entity type (block set) + * #ndim coordinates (1,2,3) +* Each block adds 1 + 2*#attr_on_block + #var_on_block +* Each set adds 2 + 2*#attr_on_set + #var_on_set +* Each sideset add 3 + 2*#attr_on_sset + #var_on_sset +* Each map adds 1 ## Example If we have an exodus file with: - * Nodes - - * 5 Element blocks - * 4 transient variables per element block - * 2 attributes per element block - - * 4 Node Sets - * Distribution Factors defined on each set - * 3 transient variables - - * 3 Side Sets - * Distribution Factors defined on each set - * 2 transient variables +* Nodes +* 5 Element blocks + * 4 transient variables per element block + * 2 attributes per element block +* 4 Node Sets + * Distribution Factors defined on each set + * 3 transient variables +* 3 Side Sets + * Distribution Factors defined on each set + * 2 transient variables Then there would be about: `10 + 5*(2+1) + 4*(2) + 3*(2) + 1 + 1 + 1 = 42` Dimensions diff --git a/packages/seacas/README.md b/packages/seacas/README.md index 74d7c04dd44d..25aab8cd574f 100644 --- a/packages/seacas/README.md +++ b/packages/seacas/README.md @@ -1,14 +1,7 @@ -# Copyright(C) 1999-2020 National Technology & Engineering Solutions -# of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with -# NTESS, the U.S. Government retains certain rights in this software. -# -# See packages/seacas/LICENSE for details -# SEACAS [[Documentation](http://gsjaardema.github.io/seacas/)] - -* [Build instructions](#build-instructions) -* [Configure, Build, and Install SEACAS](#configure-build-and-install-seacas) -* [Testing](#testing) -* [Contact information](#contact-information) +* [Build instructions](#build-instructions) +* [Configure, Build, and Install SEACAS](#configure-build-and-install-seacas) +* [Testing](#testing) +* [Contact information](#contact-information) ## Build instructions @@ -17,12 +10,12 @@ There are a few externally developed third-party libraries (TPL) that are required to build SEACAS - * [Zoltan](#zoltan) -- required, part of Trilinos - * [HDF5](#hdf5) -- optional, but highly recommended - * [Parallel-NetCDF](#parallel-netcdf) -- optional for parallel - * [NetCDF](#netcdf) -- required with modifications - * [MatIO](#matio) -- optional - * [Faodel](#faodel) -- optional +* [Zoltan](#zoltan) -- required, part of Trilinos +* [HDF5](#hdf5) -- optional, but highly recommended +* [Parallel-NetCDF](#parallel-netcdf) -- optional for parallel +* [NetCDF](#netcdf) -- required with modifications +* [MatIO](#matio) -- optional +* [Faodel](#faodel) -- optional #### Zoltan Zoltan is a package in Trilinos and it must be enabled for a SEACAS build. @@ -39,47 +32,48 @@ large models (>150 million elements); if you are not planning to create or read models of this size and do not want compression support, you do not have to build hdf5. - * Download HDF5 from +* Download HDF5 from - * untar it, creating a directory will will refer to as `hdf5-X.X.X` +* untar it, creating a directory will will refer to as `hdf5-X.X.X` - * `cd` to that directory and enter the command: - * Serial: +* `cd` to that directory and enter the command: + + * Serial: ```bash ./configure --prefix=${WHERE_TO_INSTALL} --enable-shared --enable-production --enable-debug=no --enable-static-exec ``` - * Parallel: + * Parallel: ```bash CC=mpicc ./configure --prefix=${WHERE_TO_INSTALL} --enable-shared --enable-production --enable-debug=no --enable-static-exec --enable-parallel ``` - * `make && make install` + * `make && make install` #### Parallel-NetCDF For a parallel build of Trilinos, especially to use the auto-decomposition support of the Ioss library, you will need the parallel-netcdf library, also known as pnetcdf. - * Download +* Download - * `tar zxvf parallel-netcdf-1.6.1.tar.gz` +* `tar zxvf parallel-netcdf-1.6.1.tar.gz` - * `cd` to the `parallel-netcdf-1.6.1` directory and enter the command: +* `cd` to the `parallel-netcdf-1.6.1` directory and enter the command: ```bash CC=mpicc ./configure --disable-fortran --prefix ${WHERE_TO_INSTALL} ``` - * `make && make install` +* `make && make install` #### NetCDF The most recent released version is recommended. For use with Exodus, some local modifications to the netcdf.h include file are required. See [NetCDF-Mapping.md](NetCDF-Mapping.md) for an explanation of why these modifications are required (or highly recommended) - * Download the latest netcdf-c release from +* Download the latest netcdf-c release from - * `tar zxvf netcdf-4.6.3.tar.gz` (or whatever the latest version is) +* `tar zxvf netcdf-4.6.3.tar.gz` (or whatever the latest version is) - * If the version is *prior* to 4.5.1, then you need to modify the +* If the version is *prior* to 4.5.1, then you need to modify the following defines in seacas/TPL/netcdf/netcdf-4.6.3/include/netcdf.h. Versions *4.5.1 or later* do not check these limits and can be run unmodified. @@ -89,9 +83,9 @@ The most recent released version is recommended. For use with Exodus, some local #define NC_MAX_VARS 524288 /* max variables per file */ ``` - * `cd netcdf-4.6.3` and enter the command: +* `cd netcdf-4.6.3` and enter the command: - * serial + * serial ```bash CFLAGS="-I${WHERE_TO_INSTALL}/include" \ CPPFLAGS="-DNDEBUG" LDFLAGS="-L${WHERE_TO_INSTALL}/lib" \ @@ -100,7 +94,7 @@ The most recent released version is recommended. For use with Exodus, some local --disable-dap --disable-v2 ``` - * parallel + * parallel ```bash CC='mpicc' CFLAGS="-I${WHERE_TO_INSTALL}/include" \ CPPFLAGS="-DNDEBUG" LDFLAGS="-L${WHERE_TO_INSTALL}/lib" \ @@ -109,7 +103,7 @@ The most recent released version is recommended. For use with Exodus, some local --disable-dap --disable-v2 ``` - * Check the results of the configure and make sure that the listings +* Check the results of the configure and make sure that the listings under features are similar to: ```bash @@ -126,16 +120,16 @@ The most recent released version is recommended. For use with Exodus, some local for seacas. For a serial build, `PNetCDF` and `NC-4 Parallel Support` should be `no` - * `make && make install` +* `make && make install` #### MatIO The MatIO library is used in the `exo2mat` and `mat2exo` programs which convert an exodus file to and from a MATLAB binary file. To use this do: - * Download matio via git: +* Download matio via git: - * `git clone https://github.com/tbeu/matio.git` +* `git clone https://github.com/tbeu/matio.git` - * `cd matio` and enter the command: +* `cd matio` and enter the command: ```bash ./autogen.sh # The -L is to find the hdf5 library... @@ -143,7 +137,7 @@ The MatIO library is used in the `exo2mat` and `mat2exo` programs which convert ./configure --with-hdf5=${WHERE_TO_INSTALL} --enable-mat73 --enable-shared --prefix=${WHERE_TO_INSTALL} ``` - * `make && make install` +* `make && make install` #### Faodel Faodel is a collection of data management tools that Sandia is developing to improve how datasets migrate between memory and storage resources in a distributed system. For SEACAS Faodel support means adding a new backend to IOSS. This enables additional data storage capabilities and the chance to communicate data between execution spaces. diff --git a/packages/seacas/applications/conjoin/CJ_Internals.C b/packages/seacas/applications/conjoin/CJ_Internals.C index e1ae0b188f09..1cd91e8639bf 100644 --- a/packages/seacas/applications/conjoin/CJ_Internals.C +++ b/packages/seacas/applications/conjoin/CJ_Internals.C @@ -344,6 +344,13 @@ int Excn::Internals::put_metadata(const Mesh &mesh, const CommunicationMeta ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } + + struct ex__file_item *file = ex__find_file_item(exodusFilePtr); + if (file) { + file->time_varid = varid; + } + + ex__compress_variable(exodusFilePtr, varid, -2); /* don't compress, but do set collective io */ } if (mesh.nodeCount > 0) { @@ -1179,61 +1186,43 @@ namespace { int varid; if (nodes > 0) { - if (ex_large_model(exodusFilePtr) == 1) { - // node coordinate arrays -- separate storage... - - dim[0] = node_dim; - if (dimension > 0) { - status = - nc_def_var(exodusFilePtr, VAR_COORD_X, nc_flt_code(exodusFilePtr), 1, dim, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define node x coordinate array in file id %d", - exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); - } - ex__compress_variable(exodusFilePtr, varid, 1); - } + // node coordinate arrays -- separate storage... - if (dimension > 1) { - status = - nc_def_var(exodusFilePtr, VAR_COORD_Y, nc_flt_code(exodusFilePtr), 1, dim, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define node y coordinate array in file id %d", - exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); - } - ex__compress_variable(exodusFilePtr, varid, 1); + dim[0] = node_dim; + if (dimension > 0) { + status = nc_def_var(exodusFilePtr, VAR_COORD_X, nc_flt_code(exodusFilePtr), 1, dim, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + errmsg = fmt::format("Error: failed to define node x coordinate array in file id %d", + exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 1); + } - if (dimension > 2) { - status = - nc_def_var(exodusFilePtr, VAR_COORD_Z, nc_flt_code(exodusFilePtr), 1, dim, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define node z coordinate array in file id %d", - exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); - } - ex__compress_variable(exodusFilePtr, varid, 1); + if (dimension > 1) { + status = nc_def_var(exodusFilePtr, VAR_COORD_Y, nc_flt_code(exodusFilePtr), 1, dim, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + errmsg = fmt::format("Error: failed to define node y coordinate array in file id %d", + exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 1); } - else { - // node coordinate arrays: -- all stored together (old method)2 - dim[0] = dim_dim; - dim[1] = node_dim; - status = nc_def_var(exodusFilePtr, VAR_COORD, nc_flt_code(exodusFilePtr), 2, dim, &varid); + + if (dimension > 2) { + status = nc_def_var(exodusFilePtr, VAR_COORD_Z, nc_flt_code(exodusFilePtr), 1, dim, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define node coordinate array in file id %d", + errmsg = fmt::format("Error: failed to define node z coordinate array in file id %d", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 1); } } diff --git a/packages/seacas/applications/ejoin/EJoin.C b/packages/seacas/applications/ejoin/EJoin.C index 996c83b717cf..94d6ed70e2b0 100644 --- a/packages/seacas/applications/ejoin/EJoin.C +++ b/packages/seacas/applications/ejoin/EJoin.C @@ -1,4 +1,4 @@ -// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// Copyright(C) 1999-2021 National Technology & Engineering Solutions // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with // NTESS, the U.S. Government retains certain rights in this software. // @@ -540,7 +540,7 @@ double ejoin(SystemInterface &interFace, std::vector &part_mesh, } output_region.output_summary(std::cout); fmt::print("******* END *******\n"); - fmt::print(stderr, "\nTotal Execution time = {:.5} seconds.\n", end - begin); + fmt::print(stderr, "\nTotal Execution Time = {:.5} seconds.\n", end - begin); if (steps > 0) { fmt::print(stderr, "\tMesh = {:.5} seconds; Timesteps = {:.5} seconds / step.\n\n", (ts_begin - begin), (end - ts_begin) / (double)(steps)); @@ -560,8 +560,7 @@ namespace { static int used_blocks = 0; const std::string &prefix = region.name(); - const Ioss::ElementBlockContainer &ebs = region.get_element_blocks(); - size_t total_elements = 0; + const Ioss::ElementBlockContainer &ebs = region.get_element_blocks(); for (auto eb : ebs) { if (!entity_is_omitted(eb)) { std::string name = eb->name(); @@ -577,7 +576,6 @@ namespace { } std::string type = eb->topology()->name(); size_t num_elem = eb->entity_count(); - total_elements += num_elem; if (num_elem > 0) { auto ebn = new Ioss::ElementBlock(output_region.get_database(), name, type, num_elem); @@ -602,8 +600,7 @@ namespace { { const std::string &prefix = region.name(); - const Ioss::SideSetContainer &fss = region.get_sidesets(); - size_t total_sides = 0; + const Ioss::SideSetContainer &fss = region.get_sidesets(); for (auto &fs : fss) { if (!entity_is_omitted(fs)) { std::string name = fs->name(); @@ -629,7 +626,6 @@ namespace { std::string fbtype = fb->topology()->name(); std::string partype = fb->parent_element_topology()->name(); size_t num_side = fb->entity_count(); - total_sides += num_side; auto block = new Ioss::SideBlock(output_region.get_database(), fbname, fbtype, partype, num_side); diff --git a/packages/seacas/applications/epu/EP_Internals.C b/packages/seacas/applications/epu/EP_Internals.C index 5cfb2dde2489..c01a4c9164b6 100644 --- a/packages/seacas/applications/epu/EP_Internals.C +++ b/packages/seacas/applications/epu/EP_Internals.C @@ -445,6 +445,13 @@ int Excn::Internals::put_metadata(const Mesh &mesh, const CommunicationMeta ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } + + struct ex__file_item *file = ex__find_file_item(exodusFilePtr); + if (file) { + file->time_varid = varid; + } + + ex__compress_variable(exodusFilePtr, varid, -2); /* don't compress, but do set collective io */ } if (mesh.nodeCount > 0) { @@ -1159,56 +1166,37 @@ namespace { int varid; if (nodes > 0) { - if (ex_large_model(exodusFilePtr) == 1) { - // node coordinate arrays -- separate storage... - dim[0] = node_dim; - if (dimension > 0) { - status = - nc_def_var(exodusFilePtr, VAR_COORD_X, nc_flt_code(exodusFilePtr), 1, dim, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define node x coordinate array in file id {}", - exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); - } - ex__compress_variable(exodusFilePtr, varid, 2); - } - - if (dimension > 1) { - status = - nc_def_var(exodusFilePtr, VAR_COORD_Y, nc_flt_code(exodusFilePtr), 1, dim, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define node y coordinate array in file id {}", - exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); - } - ex__compress_variable(exodusFilePtr, varid, 2); + // node coordinate arrays -- separate storage... + dim[0] = node_dim; + if (dimension > 0) { + status = nc_def_var(exodusFilePtr, VAR_COORD_X, nc_flt_code(exodusFilePtr), 1, dim, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + errmsg = fmt::format("Error: failed to define node x coordinate array in file id {}", + exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 2); + } - if (dimension > 2) { - status = - nc_def_var(exodusFilePtr, VAR_COORD_Z, nc_flt_code(exodusFilePtr), 1, dim, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define node z coordinate array in file id {}", - exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); - } - ex__compress_variable(exodusFilePtr, varid, 2); + if (dimension > 1) { + status = nc_def_var(exodusFilePtr, VAR_COORD_Y, nc_flt_code(exodusFilePtr), 1, dim, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + errmsg = fmt::format("Error: failed to define node y coordinate array in file id {}", + exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 2); } - else { - // node coordinate arrays: -- all stored together (old method)2 - dim[0] = dim_dim; - dim[1] = node_dim; - status = nc_def_var(exodusFilePtr, VAR_COORD, nc_flt_code(exodusFilePtr), 2, dim, &varid); + + if (dimension > 2) { + status = nc_def_var(exodusFilePtr, VAR_COORD_Z, nc_flt_code(exodusFilePtr), 1, dim, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define node coordinate array in file id {}", + errmsg = fmt::format("Error: failed to define node z coordinate array in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); diff --git a/packages/seacas/applications/epu/EP_Version.h b/packages/seacas/applications/epu/EP_Version.h index 4e9c20bb4fd0..edbf8c0d74ef 100644 --- a/packages/seacas/applications/epu/EP_Version.h +++ b/packages/seacas/applications/epu/EP_Version.h @@ -10,8 +10,8 @@ static char const *qainfo[] = { "epu -- E Pluribus Unum", - "2021/07/21", - "6.02", + "2021/08/18", + "6.03", }; #endif // SEACAS_Version_h diff --git a/packages/seacas/applications/epu/epu.C b/packages/seacas/applications/epu/epu.C index b2602cce453a..15135fd5385d 100644 --- a/packages/seacas/applications/epu/epu.C +++ b/packages/seacas/applications/epu/epu.C @@ -294,7 +294,7 @@ namespace { std::vector &global_sets, std::vector &local_mesh, std::vector> &local_sets, MasterValueVector &master_values, std::vector &values, - int part_count, int time_step, + int part_count, int time_step, int time_step_out, const std::vector> &local_element_to_global); void get_variable_params(int id, Excn::Variables &vars, @@ -1360,7 +1360,8 @@ int epu(SystemInterface &interFace, int start_part, int part_count, int cycle, T if (element_vars.count(InOut::IN) > 0) { read_write_master_values(element_vars, global, glob_blocks, local_mesh, blocks, master_values, - values, part_count, time_step, local_element_to_global); + values, part_count, time_step, time_step_out, + local_element_to_global); } // If adding the processor_id field, do it here... @@ -1383,7 +1384,7 @@ int epu(SystemInterface &interFace, int start_part, int part_count, int cycle, T if (sideset_vars.count(InOut::IN) > 0) { read_write_master_values(sideset_vars, global, glob_ssets, local_mesh, sidesets, - master_values, values, part_count, time_step, + master_values, values, part_count, time_step, time_step_out, local_element_to_global); } } @@ -1399,7 +1400,7 @@ int epu(SystemInterface &interFace, int start_part, int part_count, int cycle, T if (nodeset_vars.count(InOut::IN) > 0) { read_write_master_values(nodeset_vars, global, glob_nsets, local_mesh, nodesets, - master_values, values, part_count, time_step, + master_values, values, part_count, time_step, time_step_out, local_element_to_global); } } @@ -3598,7 +3599,7 @@ namespace { std::vector &global_sets, std::vector &local_mesh, std::vector> &local_sets, std::vector &master_values, std::vector &values, - int part_count, int time_step, + int part_count, int time_step, int time_step_out, const std::vector> &local_element_to_global) { bool is_sidenodeset = @@ -3664,7 +3665,7 @@ namespace { if (global.truthTable[static_cast(vars.objectType)][truth_table_loc]) { int error = - ex_put_var(id_out, time_step + 1, exodus_object_type(vars.objectType), ivar + 1, + ex_put_var(id_out, time_step_out, exodus_object_type(vars.objectType), ivar + 1, global_sets[b].id, global_sets[b].entity_count(), master_values.data()); if (error < 0) { exodus_error(__LINE__); diff --git a/packages/seacas/applications/exodiff/CMakeLists.txt b/packages/seacas/applications/exodiff/CMakeLists.txt index addb8423399b..d2bdecfa8a53 100644 --- a/packages/seacas/applications/exodiff/CMakeLists.txt +++ b/packages/seacas/applications/exodiff/CMakeLists.txt @@ -8,7 +8,6 @@ INCLUDE_DIRECTORIES( SET(HEADERS "") SET(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ED_SystemInterface.C ${CMAKE_CURRENT_SOURCE_DIR}/exo_entity.C - ${CMAKE_CURRENT_SOURCE_DIR}/FileInfo.C ${CMAKE_CURRENT_SOURCE_DIR}/exodiff.C ${CMAKE_CURRENT_SOURCE_DIR}/Tolerance.C ${CMAKE_CURRENT_SOURCE_DIR}/face_block.C diff --git a/packages/seacas/applications/exodiff/ED_SystemInterface.C b/packages/seacas/applications/exodiff/ED_SystemInterface.C index 71b2b17d05e0..e22eebc365a1 100644 --- a/packages/seacas/applications/exodiff/ED_SystemInterface.C +++ b/packages/seacas/applications/exodiff/ED_SystemInterface.C @@ -29,7 +29,6 @@ namespace { std::string sline = line; chop_whitespace(sline); Error(fmt::format("parsing input file, currently at \"{}\".\n", sline)); - exit(1); } std::string Parse_Variables(std::string xline, std::ifstream &cmd_file, bool &all_flag, @@ -58,12 +57,10 @@ namespace { Error(fmt::format(" Problem converting the string '{}'" " to a double value while parsing tolerance. Aborting...\n", str_val)); - exit(1); } if (val < 0.0) { Error(fmt::format(" Parsed a negative value \"{}\". Aborting...\n", val)); - exit(1); } return val; } @@ -158,7 +155,6 @@ namespace { Error(fmt::format("Parsing error: Cannot specify both " "variables to include and exclude without using the " "'(all)' specifier. Aborting...\n")); - exit(1); } if (num_include == 0 && num_exclude > 0) { all_flag = true; @@ -185,7 +181,6 @@ namespace { if (ival1 < 1) { Error(fmt::format("parsing exclusion times from command " "line .. value was less than 1\n")); - exit(1); } ++num_excluded_steps; @@ -199,7 +194,6 @@ namespace { if (ival2 < 1) { Error(fmt::format("parsing exclusion times from command " "line .. value was less than 1\n")); - exit(1); } if (ival1 < ival2) { @@ -211,7 +205,6 @@ namespace { Error(fmt::format("parsing exclusion times from command " "line .. first value in a range was greater than the " "second.\n")); - exit(1); } } @@ -570,7 +563,6 @@ bool SystemInterface::parse_options(int argc, char **argv) } else { Error("no files specified\n\n"); - return false; } // Get options from environment variable also... @@ -669,7 +661,6 @@ bool SystemInterface::parse_options(int argc, char **argv) Error(fmt::format("parse error for -explicit keyword. " "Expected ':', found '{}' Aborting...\n", temp)); - exit(1); } } } @@ -850,7 +841,6 @@ bool SystemInterface::parse_options(int argc, char **argv) command_file = temp; if (!summary_flag && (File_Exists(command_file) == 0)) { Error(fmt::format("Can't open file \"{}\".\n", command_file)); - exit(1); } // Command file exists, parse contents... @@ -862,7 +852,6 @@ bool SystemInterface::parse_options(int argc, char **argv) command_file = t2; if (!summary_flag && (File_Exists(command_file) == 0)) { Error(fmt::format("Can't open file \"{}\".\n", command_file)); - exit(1); } // Command file exists, parse contents... @@ -963,7 +952,6 @@ void SystemInterface::Parse_Command_File() Error(fmt::format(" expected \"TOLERANCE\" after the \"FINAL TIME\" keyword. " "Found \"{}\" instead. Aborting...\n", tok3)); - exit(1); } std::string tok = extract_token(xline, " \n\t=,"); if (tok == "") { @@ -1432,7 +1420,6 @@ namespace { !abbreviation(tok, "eigen_combine", 7) && !abbreviation(tok, "ignore", 3) && !abbreviation(tok, "floor", 3)) { Error(fmt::format("in parsing command file: unrecognized keyword \"{}\"\n", tok)); - exit(1); } if (tok == "(all)" || tok == "all") { @@ -1447,7 +1434,6 @@ namespace { if (tok == "floor" || tok == "") { Error(" Input file specifies a tolerance type " "but no tolerance\n"); - exit(1); } def_tol.value = To_Double(tok); tok = extract_token(xline, " \n\t=,"); @@ -1459,7 +1445,6 @@ namespace { if (tok == "floor" || tok == "") { Error("Input file specifies a tolerance type " "but no tolerance\n"); - exit(1); } def_tol.value = To_Double(tok); tok = extract_token(xline, " \n\t=,"); @@ -1471,7 +1456,6 @@ namespace { if (tok == "floor" || tok == "") { Error("Input file specifies a tolerance type " "but no tolerance\n"); - exit(1); } def_tol.value = To_Double(tok); tok = extract_token(xline, " \n\t=,"); @@ -1483,7 +1467,6 @@ namespace { if (tok == "floor" || tok == "") { Error("Input file specifies a tolerance type " "but no tolerance\n"); - exit(1); } def_tol.value = To_Double(tok); tok = extract_token(xline, " \n\t=,"); @@ -1495,7 +1478,6 @@ namespace { if (tok == "floor" || tok == "") { Error("Input file specifies a tolerance type " "but no tolerance\n"); - exit(1); } def_tol.value = To_Double(tok); tok = extract_token(xline, " \n\t=,"); @@ -1507,7 +1489,6 @@ namespace { if (tok == "floor" || tok == "") { Error("Input file specifies a tolerance type " "but no tolerance\n"); - exit(1); } def_tol.value = To_Double(tok); tok = extract_token(xline, " \n\t=,"); @@ -1519,7 +1500,6 @@ namespace { if (tok == "floor" || tok == "") { Error("Input file specifies a tolerance type " "but no tolerance\n"); - exit(1); } def_tol.value = To_Double(tok); tok = extract_token(xline, " \n\t=,"); @@ -1531,7 +1511,6 @@ namespace { if (tok == "floor" || tok == "") { Error("Input file specifies a tolerance type " "but no tolerance\n"); - exit(1); } def_tol.value = To_Double(tok); tok = extract_token(xline, " \n\t=,"); @@ -1548,7 +1527,6 @@ namespace { tok = extract_token(xline, " \n\t=,"); if (tok == "" || tok[0] == '#') { Error("Floor specified but couldn't find value\n"); - exit(1); } def_tol.floor = To_Double(tok); } diff --git a/packages/seacas/applications/exodiff/ED_Version.h b/packages/seacas/applications/exodiff/ED_Version.h index 334b962e866b..f0c6849ab87a 100644 --- a/packages/seacas/applications/exodiff/ED_Version.h +++ b/packages/seacas/applications/exodiff/ED_Version.h @@ -6,7 +6,7 @@ #ifndef ED_Version_h #define ED_Version_h -static std::string version("3.16"); -static std::string verdate("2021-06-28"); +static std::string version("3.18"); +static std::string verdate("2021-08-18"); #endif // ED_Version_h diff --git a/packages/seacas/applications/exodiff/check.C b/packages/seacas/applications/exodiff/check.C index 78a5a7cb868c..e97531ec606f 100644 --- a/packages/seacas/applications/exodiff/check.C +++ b/packages/seacas/applications/exodiff/check.C @@ -22,23 +22,24 @@ namespace { template - bool Check_Nodal(ExoII_Read &file1, ExoII_Read &file2, const INT *node_map, + bool Check_Nodal(ExoII_Read &file1, ExoII_Read &file2, const std::vector &node_map, const INT *id_map, bool check_only); template - bool Check_Elmt_Block(ExoII_Read &file1, ExoII_Read &file2, const INT *elmt_map, - const INT *node_map); + bool Check_Element_Block(ExoII_Read &file1, ExoII_Read &file2, + const std::vector &elmt_map, const std::vector &node_map); template - bool Check_Nodeset(ExoII_Read &file1, ExoII_Read &file2, const INT *node_map, - bool check_only); + bool Check_Nodeset(ExoII_Read &file1, ExoII_Read &file2, + const std::vector &node_map, bool check_only); template - bool Check_Sideset(ExoII_Read &file1, ExoII_Read &file2, const INT *elmt_map, - bool check_only); + bool Check_Sideset(ExoII_Read &file1, ExoII_Read &file2, + const std::vector &elmt_map, bool check_only); template - bool Check_Elmt_Block_Params(const Exo_Block *block1, const Exo_Block *block2); + bool Check_Element_Block_Params(const Exo_Block *block1, const Exo_Block *block2); template - bool Check_Elmt_Block_Connectivity(Exo_Block *block1, Exo_Block *block2, - const INT *elmt_map, const INT *node_map); + bool Check_Element_Block_Connectivity(Exo_Block *block1, Exo_Block *block2, + const std::vector &elmt_map, + const std::vector &node_map); bool close_compare(const std::string &st1, const std::string &st2); } // namespace @@ -46,69 +47,70 @@ template bool Check_Global(ExoII_Read &file1, ExoII_Read void Check_Compatible_Meshes(ExoII_Read &file1, ExoII_Read &file2, bool check_only, - const INT *node_map, const INT *elmt_map, const INT *node_id_map) + const std::vector &node_map, const std::vector &elmt_map, + const INT *node_id_map) { bool is_diff = false; // NOTE: Check_Global is called earlier. Don't repeat call here. if (!Check_Nodal(file1, file2, node_map, node_id_map, check_only)) { - Error(".. Differences found in mesh nodal coordinates.\n"); + Warning(".. Differences found in mesh nodal coordinates.\n"); is_diff = true; } - if (!Check_Elmt_Block(file1, file2, elmt_map, node_map)) { - Error(".. Differences found in element block metadata or connectivity.\n"); + if (!Check_Element_Block(file1, file2, elmt_map, node_map)) { + Warning(".. Differences found in element block metadata or connectivity.\n"); is_diff = true; } if (!Check_Nodeset(file1, file2, node_map, check_only)) { - Error(".. Differences found in node set metadata or node lists.\n"); + Warning(".. Differences found in node set metadata or node lists.\n"); is_diff = true; } if (!Check_Sideset(file1, file2, elmt_map, check_only)) { - Error(".. Differences found in side set metadata or side lists.\n"); + Warning(".. Differences found in side set metadata or side lists.\n"); is_diff = true; } if (is_diff) { - Error(".. Differences found in mesh (non-transient) data. Aborting...\n"); + Warning(".. Differences found in mesh (non-transient) data. Aborting...\n"); exit(1); } } namespace { template - bool Check_Nodal(ExoII_Read &file1, ExoII_Read &file2, const INT *node_map, + bool Check_Nodal(ExoII_Read &file1, ExoII_Read &file2, const std::vector &node_map, const INT *id_map, bool check_only) { bool is_same = true; @@ -144,8 +146,8 @@ namespace { double norm; for (size_t n = 0; n < file1.Num_Nodes() && (is_same || interFace.show_all_diffs); ++n) { // Should this node be processed... - if (node_map == nullptr || node_map[n] >= 0) { - INT n2 = node_map != nullptr ? node_map[n] : n; + if (node_map.empty() || node_map[n] >= 0) { + INT n2 = node_map.empty() ? n : node_map[n]; double dx = interFace.coord_tol.Delta(x1[n], x2[n2]); if (dx > interFace.coord_tol.value) { fmt::print(" x coord {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (node {})\n", @@ -188,36 +190,36 @@ namespace { } template - bool Check_Elmt_Block(ExoII_Read &file1, ExoII_Read &file2, const INT *elmt_map, - const INT *node_map) + bool Check_Element_Block(ExoII_Read &file1, ExoII_Read &file2, + const std::vector &elmt_map, const std::vector &node_map) { bool is_same = true; // Verify that element blocks match in the two files... - for (size_t b = 0; b < file1.Num_Elmt_Blocks(); ++b) { - Exo_Block *block1 = file1.Get_Elmt_Block_by_Index(b); + for (size_t b = 0; b < file1.Num_Element_Blocks(); ++b) { + Exo_Block *block1 = file1.Get_Element_Block_by_Index(b); Exo_Block *block2 = nullptr; if (interFace.map_flag != MapType::DISTANCE && interFace.map_flag != MapType::PARTIAL) { if (block1 != nullptr) { if (interFace.by_name) { - block2 = file2.Get_Elmt_Block_by_Name(block1->Name()); + block2 = file2.Get_Element_Block_by_Name(block1->Name()); } else { - block2 = file2.Get_Elmt_Block_by_Id(block1->Id()); + block2 = file2.Get_Element_Block_by_Id(block1->Id()); } if (block2 == nullptr) { - Error(fmt::format(".. Block id {} with name {} exists in first " - "file but not the second.\n", - block1->Id(), block1->Name())); + Warning(fmt::format(".. Block id {} with name {} exists in first " + "file but not the second.\n", + block1->Id(), block1->Name())); is_same = false; } else { - if (!Check_Elmt_Block_Params(block1, block2)) { + if (!Check_Element_Block_Params(block1, block2)) { is_same = false; } else { - // Only do this check if Check_Elmt_Block_Params does not fail. + // Only do this check if Check_Element_Block_Params does not fail. // TODO(gdsjaar): Pass in node_map and node_id_map... - if (!Check_Elmt_Block_Connectivity(block1, block2, elmt_map, node_map)) { + if (!Check_Element_Block_Connectivity(block1, block2, elmt_map, node_map)) { is_same = false; } } @@ -229,8 +231,9 @@ namespace { } template - bool Check_Elmt_Block_Connectivity(Exo_Block *block1, Exo_Block *block2, - const INT *elmt_map, const INT *node_map) + bool Check_Element_Block_Connectivity(Exo_Block *block1, Exo_Block *block2, + const std::vector &elmt_map, + const std::vector &node_map) { bool is_same = true; @@ -238,24 +241,25 @@ namespace { block1->Load_Connectivity(); block2->Load_Connectivity(); - const INT *conn1 = block1->Connectivity(); - const INT *conn2 = block2->Connectivity(); + const auto &conn1 = block1->Connectivity(); + const auto &conn2 = block2->Connectivity(); - SMART_ASSERT(block1->Size() == 0 || block1->Num_Nodes_per_Elmt() == 0 || conn1 != nullptr); - SMART_ASSERT(block2->Size() == 0 || block2->Num_Nodes_per_Elmt() == 0 || conn2 != nullptr); + SMART_ASSERT(block1->Size() == 0 || block1->Num_Nodes_per_Element() == 0 || !conn1.empty()); + SMART_ASSERT(block2->Size() == 0 || block2->Num_Nodes_per_Element() == 0 || !conn2.empty()); - if (interFace.map_flag == MapType::FILE_ORDER || elmt_map == nullptr) { - size_t node_count = block1->Size() * block1->Num_Nodes_per_Elmt(); - SMART_ASSERT(node_count == block2->Size() * block2->Num_Nodes_per_Elmt()); + if (interFace.map_flag == MapType::FILE_ORDER || elmt_map.empty()) { + size_t node_count = block1->Size() * block1->Num_Nodes_per_Element(); + SMART_ASSERT(node_count == block2->Size() * block2->Num_Nodes_per_Element()); - if (interFace.map_flag != MapType::FILE_ORDER && node_map != nullptr) { + if (interFace.map_flag != MapType::FILE_ORDER && !node_map.empty()) { for (size_t e = 0; e < node_count; ++e) { if (node_map[conn1[e] - 1] + 1 != conn2[e]) { - size_t elem = e / block2->Num_Nodes_per_Elmt(); - size_t node = e % block2->Num_Nodes_per_Elmt(); - Error(fmt::format(".. Connectivities in block id {} are not the same.\n" - " First difference is node {} of local element {}\n", - block1->Id(), node + 1, elem + 1)); + size_t elem = e / block2->Num_Nodes_per_Element(); + size_t node = e % block2->Num_Nodes_per_Element(); + Warning( + fmt::format(".. Connectivities in block id {} are not the same.\n" + " First difference is node {} of local element {}\n", + block1->Id(), node + 1, elem + 1)); is_same = false; break; } @@ -264,22 +268,23 @@ namespace { else { for (size_t e = 0; e < node_count; ++e) { if (conn1[e] != conn2[e]) { - size_t elem = e / block2->Num_Nodes_per_Elmt(); - size_t node = e % block2->Num_Nodes_per_Elmt(); - Error(fmt::format(".. Connectivities in block id {} are not the same.\n" - " First difference is node {} of local element {}\n", - block1->Id(), node + 1, elem + 1)); + size_t elem = e / block2->Num_Nodes_per_Element(); + size_t node = e % block2->Num_Nodes_per_Element(); + Warning( + fmt::format(".. Connectivities in block id {} are not the same.\n" + " First difference is node {} of local element {}\n", + block1->Id(), node + 1, elem + 1)); is_same = false; break; } } } } - else if (elmt_map != nullptr) { + else if (!elmt_map.empty()) { auto offset1 = block1->offset(); auto offset2 = block2->offset(); size_t num_element = block1->Size(); - size_t nnpe = block1->Num_Nodes_per_Elmt(); + size_t nnpe = block1->Num_Nodes_per_Element(); for (size_t e1 = 0; is_same && e1 < num_element; e1++) { for (size_t n = 0; is_same && n < nnpe; ++n) { size_t off1 = e1 * nnpe + n; @@ -288,12 +293,13 @@ namespace { e2 -= offset2; size_t off2 = e2 * nnpe + n; auto n1 = conn1[off1]; - auto map_n1 = node_map != nullptr ? node_map[n1 - 1] + 1 : n1; + auto map_n1 = node_map.empty() ? n1 : node_map[n1 - 1] + 1; if (map_n1 != conn2[off2]) { - Error(fmt::format(".. Connectivities in block id {} are not the same.\n" - " First difference is node {} of local element {} " - "(file1) {} (file2)\n", - block1->Id(), n + 1, e1 + 1, e2 + 1)); + Warning( + fmt::format(".. Connectivities in block id {} are not the same.\n" + " First difference is node {} of local element {} " + "(file1) {} (file2)\n", + block1->Id(), n + 1, e1 + 1, e2 + 1)); is_same = false; break; } @@ -308,41 +314,42 @@ namespace { } template - bool Check_Elmt_Block_Params(const Exo_Block *block1, const Exo_Block *block2) + bool Check_Element_Block_Params(const Exo_Block *block1, const Exo_Block *block2) { bool is_same = true; SMART_ASSERT(block1 && block2); if (!interFace.by_name && block1->Id() != block2->Id()) { - Error(fmt::format(".. Block ids don't agree ({} != {}\n", block1->Id(), block2->Id())); + Warning(fmt::format(".. Block ids don't agree ({} != {}\n", block1->Id(), block2->Id())); is_same = false; } if (interFace.by_name && block1->Name() != block2->Name()) { - Error( + Warning( fmt::format(".. Block names don't agree ({} != {}).\n", block1->Name(), block2->Name())); is_same = false; } - if (!(no_case_equals(block1->Elmt_Type(), block2->Elmt_Type()))) { + if (!(no_case_equals(block1->Element_Type(), block2->Element_Type()))) { if (!interFace.short_block_check || - !close_compare(block1->Elmt_Type(), block2->Elmt_Type())) { - Error(fmt::format(".. Block {}: element types don't agree ({} != {}).\n", block1->Id(), - block1->Elmt_Type(), block2->Elmt_Type())); + !close_compare(block1->Element_Type(), block2->Element_Type())) { + Warning(fmt::format(".. Block {}: element types don't agree ({} != {}).\n", block1->Id(), + block1->Element_Type(), block2->Element_Type())); is_same = false; } } if (block1->Size() != block2->Size()) { - Error(fmt::format(".. Block {}: number of elements doesn't agree ({} != {}).\n", block1->Id(), - block1->Size(), block2->Size())); + Warning(fmt::format(".. Block {}: number of elements doesn't agree ({} != {}).\n", + block1->Id(), block1->Size(), block2->Size())); is_same = false; } - if (block1->Num_Nodes_per_Elmt() != block2->Num_Nodes_per_Elmt()) { - Error(fmt::format(".. Block {}: number of nodes per element doesn't agree ({} != {}).\n", - block1->Id(), block1->Num_Nodes_per_Elmt(), block2->Num_Nodes_per_Elmt())); + if (block1->Num_Nodes_per_Element() != block2->Num_Nodes_per_Element()) { + Warning(fmt::format(".. Block {}: number of nodes per element doesn't agree ({} != {}).\n", + block1->Id(), block1->Num_Nodes_per_Element(), + block2->Num_Nodes_per_Element())); is_same = false; } #if 0 if (block1->Num_Attributes() != block2->Num_Attributes()) { - Error(fmt::format(".. Block {}: number of attributes doesn't agree ({} != {}).\n" + Warning(fmt::format(".. Block {}: number of attributes doesn't agree ({} != {}).\n" block1->Id(), block1->Num_Attributes(), block2->Num_Attributes()); is_same = false; } @@ -351,8 +358,8 @@ namespace { } template - bool Check_Nodeset(ExoII_Read &file1, ExoII_Read &file2, const INT *node_map, - bool /*unused*/) + bool Check_Nodeset(ExoII_Read &file1, ExoII_Read &file2, + const std::vector &node_map, bool /*unused*/) { // Currently don't set diff flag for most of these since we // can continue (somewhat) with these differences... @@ -361,7 +368,7 @@ namespace { bool is_same = true; if (file1.Num_Node_Sets() != file2.Num_Node_Sets()) { if (interFace.map_flag != MapType::PARTIAL) { - Error(".. Number of nodesets doesn't agree...\n"); + Warning(".. Number of nodesets doesn't agree...\n"); if (interFace.pedantic) { is_same = false; } @@ -379,7 +386,7 @@ namespace { } if (set2 == nullptr) { - Error( + Warning( fmt::format(".. Nodeset id {} exists in first file but not the second.\n", set1->Id())); if (interFace.pedantic) { is_same = false; @@ -387,7 +394,7 @@ namespace { } else { if (set1->Size() != set2->Size()) { - Error(fmt::format( + Warning(fmt::format( ".. The node count for nodeset id {} is not the same in the two files ({} != {}).\n", set1->Id(), set1->Size(), set2->Size())); if (interFace.pedantic) { @@ -402,7 +409,7 @@ namespace { for (size_t b = 0; b < file2.Num_Node_Sets(); ++b) { Node_Set *set2 = file2.Get_Node_Set_by_Index(b); if (set2 == nullptr) { - Error( + Warning( fmt::format(".. Could not access the Nodeset with index {} in the second file.\n", b)); if (interFace.pedantic) { is_same = false; @@ -428,7 +435,7 @@ namespace { continue; } - if (node_map != nullptr) { + if (!node_map.empty()) { set1->apply_map(node_map); } @@ -442,7 +449,7 @@ namespace { } } if (diff >= 0) { - Error(fmt::format( + Warning(fmt::format( ".. The nodelists for nodeset id {} are not the same in the two files.\n" "\t\tThe first difference is at position {}: Node {} vs. Node {}.\n", set1->Id(), set1->Node_Index(diff) + 1, set1->Node_Id(diff), set2->Node_Id(diff))); @@ -450,8 +457,8 @@ namespace { is_same = false; } else { - Error(".. The nodelist differences are ignored for the " - "partial_map case.\n"); + Warning(".. The nodelist differences are ignored for the " + "partial_map case.\n"); } } } @@ -461,8 +468,8 @@ namespace { } template - bool Check_Sideset(ExoII_Read &file1, ExoII_Read &file2, const INT *elmt_map, - bool /*unused*/) + bool Check_Sideset(ExoII_Read &file1, ExoII_Read &file2, + const std::vector &elmt_map, bool /*unused*/) { // Currently don't set diff flag for most of these since we // can continue (somewhat) with these differences... @@ -471,7 +478,7 @@ namespace { bool is_same = true; if (file1.Num_Side_Sets() != file2.Num_Side_Sets()) { if (interFace.map_flag != MapType::PARTIAL) { - Error(".. Number of sidesets doesn't agree...\n"); + Warning(".. Number of sidesets doesn't agree...\n"); if (interFace.pedantic) { is_same = false; } @@ -489,7 +496,7 @@ namespace { } if (set2 == nullptr) { - Error( + Warning( fmt::format(".. Sideset id {} exists in first file but not the second.\n", set1->Id())); if (interFace.pedantic) { is_same = false; @@ -497,7 +504,7 @@ namespace { } else { if (set1->Size() != set2->Size()) { - Error(fmt::format( + Warning(fmt::format( ".. The side count for sideset id {} is not the same in the two files ({} != {}).\n", set1->Id(), set1->Size(), set2->Size())); if (interFace.pedantic) { @@ -510,7 +517,7 @@ namespace { for (size_t b = 0; b < file2.Num_Side_Sets(); ++b) { Side_Set *set2 = file2.Get_Side_Set_by_Index(b); if (set2 == nullptr) { - Error( + Warning( fmt::format(".. Could not access the Sideset with index {} in the second file.\n", b)); if (interFace.pedantic) { is_same = false; @@ -536,7 +543,7 @@ namespace { continue; } - if (elmt_map != nullptr) { + if (!elmt_map.empty()) { set1->apply_map(elmt_map); } @@ -556,10 +563,10 @@ namespace { // appears in the file. If don't do this, error message is very confusing for the // user... auto set1_id = set1->Side_Id(diff).first; - if (elmt_map != nullptr) { + if (!elmt_map.empty()) { // Iterate map to find an entry equal to `set1_id`. Its position is then the file1 id // of the element. - for (size_t i = 0; i < file1.Num_Elmts(); i++) { + for (size_t i = 0; i < file1.Num_Elements(); i++) { if (elmt_map[i] == set1_id - 1) { set1_id = i + 1; break; @@ -567,7 +574,7 @@ namespace { } } - Error(fmt::format( + Warning(fmt::format( ".. The sidelists for sideset id {} are not the same in the two files.\n" "\t\tThe first difference is at position {}: Side {}.{} .vs. Side {}.{}.\n", set1->Id(), set1->Side_Index(diff) + 1, set1_id, set1->Side_Id(diff).second, @@ -576,8 +583,8 @@ namespace { is_same = false; } else { - Error(".. The sidelist differences are ignored for the " - "partial_map case.\n"); + Warning(".. The sidelist differences are ignored for the " + "partial_map case.\n"); } } } @@ -623,10 +630,11 @@ namespace { template bool Check_Global(ExoII_Read &file1, ExoII_Read &file2); template void Check_Compatible_Meshes(ExoII_Read &file1, ExoII_Read &file2, - bool check_only, const int *node_map, const int *elmt_map, - const int *node_id_map); + bool check_only, const std::vector &node_map, + const std::vector &elmt_map, const int *node_id_map); template bool Check_Global(ExoII_Read &file1, ExoII_Read &file2); template void Check_Compatible_Meshes(ExoII_Read &file1, ExoII_Read &file2, - bool check_only, const int64_t *node_map, - const int64_t *elmt_map, const int64_t *node_id_map); + bool check_only, const std::vector &node_map, + const std::vector &elmt_map, + const int64_t * node_id_map); diff --git a/packages/seacas/applications/exodiff/create_file.C b/packages/seacas/applications/exodiff/create_file.C index 8f61bc93782f..064a4f68d664 100644 --- a/packages/seacas/applications/exodiff/create_file.C +++ b/packages/seacas/applications/exodiff/create_file.C @@ -54,13 +54,13 @@ void Build_Variable_Names(ExoII_Read &file1, ExoII_Read &file2, bool * // Build (and compare) element variable names. build_variable_names("element", interFace.elmt_var_names, interFace.elmt_var, interFace.elmt_var_default, interFace.elmt_var_do_all_flag, - file1.Elmt_Var_Names(), file2.Elmt_Var_Names(), diff_found); + file1.Element_Var_Names(), file2.Element_Var_Names(), diff_found); // Build (and compare) element variable names. if (!interFace.ignore_attributes) { build_variable_names("element attribute", interFace.elmt_att_names, interFace.elmt_att, interFace.elmt_att_default, interFace.elmt_att_do_all_flag, - file1.Elmt_Att_Names(), file2.Elmt_Att_Names(), diff_found); + file1.Element_Att_Names(), file2.Element_Att_Names(), diff_found); } // Build (and compare) nodeset variable names. @@ -117,7 +117,6 @@ int Create_File(ExoII_Read &file1, ExoII_Read &file2, const std::strin out_file_id = ex_create(diffile_name.c_str(), mode, &compws, &iows); if (out_file_id < 0) { Error(fmt::format("Couldn't create output file \"{}\".\n", diffile_name)); - exit(1); } ex_copy(file1.File_ID(), out_file_id); } @@ -154,9 +153,10 @@ int Create_File(ExoII_Read &file1, ExoII_Read &file2, const std::strin DIFF_OUT(info, fmt::color::yellow); } if (interFace.coord_tol.type != ToleranceMode::IGNORE_) { - fmt::print("\nNodal coordinates will be compared .. tol: {:8g} ({}), floor: {:8g}\n", - interFace.coord_tol.value, interFace.coord_tol.typestr(), - interFace.coord_tol.floor); + fmt::print( + "\nNodal coordinates will be compared:\n {:<{}} tol: {:8g} ({}), floor: {:8g}\n", + "...", name_length(), interFace.coord_tol.value, interFace.coord_tol.typestr(), + interFace.coord_tol.floor); } else { std::ostringstream info; @@ -165,8 +165,8 @@ int Create_File(ExoII_Read &file1, ExoII_Read &file2, const std::strin } if (interFace.time_tol.type != ToleranceMode::IGNORE_) { - fmt::print("Time step values will be compared .. tol: {:8g} ({}), floor: {:8g}\n", - interFace.time_tol.value, interFace.time_tol.typestr(), + fmt::print("Time step values will be compared:\n {:<{}} tol: {:8g} ({}), floor: {:8g}\n", + "...", name_length(), interFace.time_tol.value, interFace.time_tol.typestr(), interFace.time_tol.floor); } else { @@ -182,10 +182,10 @@ int Create_File(ExoII_Read &file1, ExoII_Read &file2, const std::strin file1.Num_Nodal_Vars(), file2.Num_Nodal_Vars()); output_compare_names("Element", interFace.elmt_var_names, interFace.elmt_var, - file1.Num_Elmt_Vars(), file2.Num_Elmt_Vars()); + file1.Num_Element_Vars(), file2.Num_Element_Vars()); output_compare_names("Element Attribute", interFace.elmt_att_names, interFace.elmt_att, - file1.Num_Elmt_Atts(), file2.Num_Elmt_Atts()); + file1.Num_Element_Atts(), file2.Num_Element_Atts()); output_compare_names("Nodeset", interFace.ns_var_names, interFace.ns_var, file1.Num_NS_Vars(), file2.Num_NS_Vars()); @@ -194,9 +194,10 @@ int Create_File(ExoII_Read &file1, ExoII_Read &file2, const std::strin file2.Num_SS_Vars()); if (!interFace.ignore_sideset_df && interFace.ss_df_tol.type != ToleranceMode::IGNORE_ && file1.Num_Side_Sets() > 0 && file2.Num_Side_Sets() > 0) { - fmt::print( - "Sideset Distribution Factors will be compared .. tol: {:8g} ({}), floor: {:8g}\n", - interFace.ss_df_tol.value, interFace.ss_df_tol.typestr(), interFace.ss_df_tol.floor); + fmt::print("Sideset Distribution Factors will be compared:\n {:<{}} tol: {:8g} ({}), " + "floor: {:8g}\n", + "...", name_length(), interFace.ss_df_tol.value, interFace.ss_df_tol.typestr(), + interFace.ss_df_tol.floor); } else { if (interFace.ignore_sideset_df || interFace.ss_df_tol.type == ToleranceMode::IGNORE_) { @@ -217,8 +218,8 @@ int Create_File(ExoII_Read &file1, ExoII_Read &file2, const std::strin std::vector truth_tab; build_truth_table(EX_ELEM_BLOCK, "Element Block", interFace.elmt_var_names, - file1.Num_Elmt_Blocks(), file1, file2, file1.Elmt_Var_Names(), - file2.Elmt_Var_Names(), truth_tab, interFace.quiet_flag, diff_found); + file1.Num_Element_Blocks(), file1, file2, file1.Element_Var_Names(), + file2.Element_Var_Names(), truth_tab, interFace.quiet_flag, diff_found); std::vector ns_truth_tab; build_truth_table(EX_NODE_SET, "Nodeset", interFace.ns_var_names, file1.Num_Node_Sets(), file1, @@ -279,12 +280,12 @@ namespace { fmt::print("{} variables to be compared:\n", type); for (unsigned v = 0; v < names.size(); ++v) { if (v == 0) { - fmt::print("{:<32} tol: {:8g} ({}), floor: {:8g}\n", names[v], tol[v].value, - tol[v].typestr(), tol[v].floor); + fmt::print(" {:<{}} tol: {:8g} ({}), floor: {:8g}\n", names[v], name_length(), + tol[v].value, tol[v].typestr(), tol[v].floor); } else { - fmt::print("{:<32} {:8g} ({}), {:8g}\n", names[v], tol[v].value, - tol[v].typestr(), tol[v].floor); + fmt::print(" {:<{}} {:8g} ({}), {:8g}\n", names[v], name_length(), + tol[v].value, tol[v].typestr(), tol[v].floor); } } } @@ -467,7 +468,6 @@ namespace { int idx2 = find_string(var_names2, name, interFace.nocase_var_names); if (idx1 < 0 || idx2 < 0) { Error(fmt::format("Unable to find variable named '{}' on database.\n", name)); - exit(1); } if (set1->is_valid_var(idx1)) { diff --git a/packages/seacas/applications/exodiff/edge_block.C b/packages/seacas/applications/exodiff/edge_block.C index 7eddb525e3cb..ca0c39bdbd5c 100644 --- a/packages/seacas/applications/exodiff/edge_block.C +++ b/packages/seacas/applications/exodiff/edge_block.C @@ -26,12 +26,7 @@ Edge_Block::Edge_Block(int file_id, size_t id, size_t ne) : Exo_Entity(file SMART_ASSERT(id > 0); } -template Edge_Block::~Edge_Block() -{ - SMART_ASSERT(Check_State()); - - delete[] edgeIndex; -} +template Edge_Block::~Edge_Block() { SMART_ASSERT(Check_State()); } template EXOTYPE Edge_Block::exodus_type() const { return EX_EDGE_BLOCK; } @@ -46,7 +41,6 @@ template void Edge_Block::entity_load_params() if (err < 0) { Error("Edge_Block::entity_load_params(): Failed to get edge" " block parameters! Aborting...\n"); - exit(1); } numEntity = block.num_entry; @@ -62,35 +56,19 @@ template void Edge_Block::entity_load_params() "\tnum attributes = {}\n" " ... Aborting...\n", numEntity, num_edges_per_elmt, num_attr)); - exit(1); - } -} - -template void Edge_Block::load_edges(const INT *elmt_map) const -{ - if ((edgeIndex == nullptr) && numEntity > 0) { - edgeIndex = new INT[numEntity]; - SMART_ASSERT(edgeIndex != nullptr); - - for (size_t i = 0; i < numEntity; i++) { - edgeIndex[i] = i; - } - SMART_ASSERT(Check_State()); } } template size_t Edge_Block::Edge_Index(size_t position) const { - load_edges(); SMART_ASSERT(position < numEntity); - return edgeIndex[position]; + return position; } template int Edge_Block::Check_State() const { SMART_ASSERT(id_ >= EX_INVALID_ID); SMART_ASSERT(!(id_ == EX_INVALID_ID && numEntity > 0)); - SMART_ASSERT(!(id_ == EX_INVALID_ID && edgeIndex)); return 1; } diff --git a/packages/seacas/applications/exodiff/edge_block.h b/packages/seacas/applications/exodiff/edge_block.h index 08628d188edf..3ce1eeeaf2ab 100644 --- a/packages/seacas/applications/exodiff/edge_block.h +++ b/packages/seacas/applications/exodiff/edge_block.h @@ -28,15 +28,12 @@ template class Edge_Block : public Exo_Entity Edge_Block(const Edge_Block &) = delete; // Not written. const Edge_Block &operator=(const Edge_Block &) = delete; // Not written. - void load_edges(const INT *elmt_map = nullptr) const; void entity_load_params() override; EXOTYPE exodus_type() const override; const char *label() const override { return "Edgeblock"; } const char *short_label() const override { return "edgeblock"; } - mutable INT *edgeIndex{nullptr}; - std::string elmt_type; int num_edges_per_elmt{-1}; diff --git a/packages/seacas/applications/exodiff/exoII_read.C b/packages/seacas/applications/exodiff/exoII_read.C index a164ab9c1241..64039a0ef11d 100644 --- a/packages/seacas/applications/exodiff/exoII_read.C +++ b/packages/seacas/applications/exodiff/exoII_read.C @@ -82,7 +82,6 @@ template std::string ExoII_Read::Close_File() if (err < 0) { Error(fmt::format("ExoII_Read::Close_File(): {}: Unable to close file! Aborting...\n", err)); - exit(1); } if (err > 0) { return fmt::format("WARNING: {} issued upon close", err); @@ -114,14 +113,14 @@ template const std::string &ExoII_Read::Nodal_Var_Name(int i return nodal_vars[index]; } -template const std::string &ExoII_Read::Elmt_Var_Name(int index) const +template const std::string &ExoII_Read::Element_Var_Name(int index) const { SMART_ASSERT(Check_State()); SMART_ASSERT(index >= 0 && (unsigned)index < elmt_vars.size()); return elmt_vars[index]; } -template const std::string &ExoII_Read::Elmt_Att_Name(int index) const +template const std::string &ExoII_Read::Element_Att_Name(int index) const { SMART_ASSERT(Check_State()); SMART_ASSERT(index >= 0 && (unsigned)index < elmt_atts.size()); @@ -157,14 +156,14 @@ template const std::string &ExoII_Read::FB_Var_Name(int inde } template -Exo_Block *ExoII_Read::Get_Elmt_Block_by_Index(size_t block_index) const +Exo_Block *ExoII_Read::Get_Element_Block_by_Index(size_t block_index) const { SMART_ASSERT(Check_State()); SMART_ASSERT(block_index < num_elmt_blocks); return &eblocks[block_index]; } -template Exo_Block *ExoII_Read::Get_Elmt_Block_by_Id(size_t id) const +template Exo_Block *ExoII_Read::Get_Element_Block_by_Id(size_t id) const { SMART_ASSERT(Check_State()); for (size_t i = 0; i < num_elmt_blocks; i++) { @@ -176,7 +175,7 @@ template Exo_Block *ExoII_Read::Get_Elmt_Block_by_Id(si } template -Exo_Block *ExoII_Read::Get_Elmt_Block_by_Name(const std::string &name) const +Exo_Block *ExoII_Read::Get_Element_Block_by_Name(const std::string &name) const { SMART_ASSERT(Check_State()); for (size_t i = 0; i < num_elmt_blocks; i++) { @@ -386,7 +385,7 @@ Face_Block *ExoII_Read::Get_Face_Block_by_Name(const std::string &name } template -std::string ExoII_Read::Load_Elmt_Block_Description(size_t block_index) const +std::string ExoII_Read::Load_Element_Block_Description(size_t block_index) const { SMART_ASSERT(Check_State()); if (!Open()) { @@ -401,7 +400,7 @@ std::string ExoII_Read::Load_Elmt_Block_Description(size_t block_index) con return ""; } -template std::string ExoII_Read::Load_Elmt_Block_Descriptions() const +template std::string ExoII_Read::Load_Element_Block_Descriptions() const { SMART_ASSERT(Check_State()); if (!Open()) { @@ -409,13 +408,12 @@ template std::string ExoII_Read::Load_Elmt_Block_Description } for (size_t b = 0; b < num_elmt_blocks; ++b) { eblocks[b].Load_Connectivity(); - // eblocks[b].Load_Attributes(); } return ""; } -template std::string ExoII_Read::Free_Elmt_Block(size_t block_index) const +template std::string ExoII_Read::Free_Element_Block(size_t block_index) const { SMART_ASSERT(Check_State()); @@ -423,13 +421,11 @@ template std::string ExoII_Read::Free_Elmt_Block(size_t bloc eblocks[block_index].Free_Connectivity(); eblocks[block_index].Free_Attributes(); - // eblocks[idx].Free_Connectivity(); - // eblocks[idx].Free_Attributes(); return ""; } -template std::string ExoII_Read::Free_Elmt_Blocks() const +template std::string ExoII_Read::Free_Element_Blocks() const { SMART_ASSERT(Check_State()); @@ -441,15 +437,6 @@ template std::string ExoII_Read::Free_Elmt_Blocks() const return ""; } -template -std::string ExoII_Read::Give_Connectivity(size_t block_index, size_t &num_e, size_t &npe, - INT *&new_conn) -{ - SMART_ASSERT(block_index < num_elmt_blocks); - - return eblocks[block_index].Give_Connectivity(num_e, npe, new_conn); -} - template size_t ExoII_Read::Block_Id(size_t block_index) const { SMART_ASSERT(Check_State()); @@ -479,7 +466,6 @@ template std::string ExoII_Read::Load_Node_Map() if (err < 0) { Error(fmt::format("Unable to load node map; Exodus error = {}. Aborting...\n", err)); - exit(1); } else if (err > 0) { return "WARNING: Default node map being used."; @@ -497,7 +483,7 @@ template std::string ExoII_Read::Free_Node_Map() return ""; } -template std::string ExoII_Read::Load_Elmt_Map() +template std::string ExoII_Read::Load_Element_Map() { SMART_ASSERT(Check_State()); @@ -519,7 +505,6 @@ template std::string ExoII_Read::Load_Elmt_Map() if (err < 0) { Error(fmt::format("Unable to load element map; Exodus error = {}. Aborting...\n", err)); - exit(1); } else if (err > 0) { return "WARNING: Default element map being used."; @@ -527,7 +512,7 @@ template std::string ExoII_Read::Load_Elmt_Map() return ""; } -template std::string ExoII_Read::Free_Elmt_Map() +template std::string ExoII_Read::Free_Element_Map() { SMART_ASSERT(Check_State()); @@ -559,7 +544,6 @@ template std::string ExoII_Read::Load_Nodal_Coordinates() int err = ex_get_coord(file_id, x, y, z); if (err < 0) { Error("Failed to get nodal coordinates! Aborting...\n"); - exit(1); } else if (err > 0) { delete[] nodes; @@ -609,7 +593,6 @@ std::string ExoII_Read::Load_Nodal_Results(int time_step_num, int var_index if (err < 0) { Error("ExoII_Read::Load_Nodal_Results(): Failed to get " "nodal variable values! Aborting...\n"); - exit(1); } else if (err > 0) { delete[] results[var_index]; @@ -650,7 +633,6 @@ const double *ExoII_Read::Get_Nodal_Results(int t1, int t2, double proporti if (err < 0) { Error("ExoII_Read::Get_Nodal_Results(): Failed to get " "nodal variable values! Aborting...\n"); - exit(1); } if (t1 != t2) { @@ -662,7 +644,6 @@ const double *ExoII_Read::Get_Nodal_Results(int t1, int t2, double proporti if (err < 0) { Error("ExoII_Read::Load_Nodal_Results(): Failed to get " "nodal variable values! Aborting...\n"); - exit(1); } // Interpolate the values... @@ -732,7 +713,6 @@ template std::string ExoII_Read::Load_Global_Results(int tim if (err < 0) { Error("ExoII_Read::Load_Global_Results(): Failed to get " "global variable values! Aborting...\n"); - exit(1); } else if (err > 0) { return fmt::format("ExoII_Read::Load_Global_Results(): WARNING: " @@ -774,7 +754,6 @@ std::string ExoII_Read::Load_Global_Results(int t1, int t2, double proporti if (err < 0) { Error("ExoII_Read::Load_Global_Results(): Failed to get " "global variable values! Aborting...\n"); - exit(1); } if (t2 != t1) { @@ -782,7 +761,6 @@ std::string ExoII_Read::Load_Global_Results(int t1, int t2, double proporti if (err < 0) { Error("ExoII_Read::Load_Global_Results(): Failed to get " "global variable values! Aborting...\n"); - exit(1); } // Do the interpolation... @@ -846,29 +824,26 @@ Face_Block *ExoII_Read::Get_Face_Block_by_Index(size_t face_block_inde // This function converts an Exodus global element number (1-offset) into // its block index (0-offset) and block element index (0-offset). template -std::string ExoII_Read::Global_to_Block_Local(size_t global_elmt_num, int &block_index, - size_t &local_elmt_index) const +std::pair ExoII_Read::Global_to_Block_Local(size_t global_elmt_num) const { SMART_ASSERT(Check_State()); if (!Open()) { - return "exodiff: ERROR: File not open!"; + Error("exodiff: ERROR: File not open!"); } if (global_elmt_num < 1 || global_elmt_num > num_elmts) { - return fmt::format("exodiff: ERROR: global_elmt_num = {:L} is out of bounds [1, {:L}]!", - global_elmt_num, num_elmts); + Error(fmt::format("exodiff: ERROR: global_elmt_num = {:L} is out of bounds [1, {:L}]!", + global_elmt_num, num_elmts)); } - block_index = 0; + int block_index = 0; size_t total = 0; while (total + eblocks[block_index].Size() < global_elmt_num) { total += eblocks[block_index++].Size(); } - local_elmt_index = global_elmt_num - total - 1; - - return ""; + return std::make_pair(block_index, global_elmt_num - total - 1); } template int ExoII_Read::Check_State() const @@ -976,12 +951,13 @@ template void ExoII_Read::Get_Init_Data() Error(fmt::format("Failed to get init data!" " Error number = {}. Aborting...\n", err)); - exit(1); } dimension = info.num_dim; num_nodes = info.num_nodes; num_elmts = info.num_elem; + num_faces = info.num_face; + num_edges = info.num_edge; num_elmt_blocks = info.num_elem_blk; num_node_sets = info.num_node_sets; num_side_sets = info.num_side_sets; @@ -1005,7 +981,6 @@ template void ExoII_Read::Get_Init_Data() " ... Aborting...\n", dimension, num_nodes, num_elmts, num_elmt_blocks, num_node_sets, num_edge_blocks, num_face_blocks, num_side_sets)); - exit(1); } int num_qa = ex_inquire_int(file_id, EX_INQ_QA); @@ -1017,7 +992,6 @@ template void ExoII_Read::Get_Init_Data() " num_info = {}\n" " ... Aborting...\n", num_qa, num_info)); - exit(1); } // Coordinate Names... @@ -1026,7 +1000,6 @@ template void ExoII_Read::Get_Init_Data() err = ex_get_coord_names(file_id, coords); if (err < 0) { Error("Failed to get coordinate names! Aborting...\n"); - exit(1); } coord_names.clear(); @@ -1050,7 +1023,6 @@ template void ExoII_Read::Get_Init_Data() if (err < 0) { Error("Failed to get element block ids! Aborting...\n"); - exit(1); } size_t e_count = 0; @@ -1102,7 +1074,6 @@ template void ExoII_Read::Get_Init_Data() if (err < 0) { Error("Failed to get nodeset ids! Aborting...\n"); - exit(1); } for (size_t nset = 0; nset < num_node_sets; ++nset) { @@ -1131,7 +1102,6 @@ template void ExoII_Read::Get_Init_Data() if (err < 0) { Error("Failed to get sideset ids! Aborting...\n"); - exit(1); } for (size_t sset = 0; sset < num_side_sets; ++sset) { @@ -1160,7 +1130,6 @@ template void ExoII_Read::Get_Init_Data() if (err < 0) { Error("Failed to get edgeblock ids! Aborting...\n"); - exit(1); } for (size_t edge_block = 0; edge_block < num_edge_blocks; ++edge_block) { @@ -1189,7 +1158,6 @@ template void ExoII_Read::Get_Init_Data() if (err < 0) { Error("Failed to get faceblock ids! Aborting...\n"); - exit(1); } for (size_t face_block = 0; face_block < num_face_blocks; ++face_block) { @@ -1213,43 +1181,36 @@ template void ExoII_Read::Get_Init_Data() err = ex_get_variable_param(file_id, EX_GLOBAL, &num_global_vars); if (err < 0) { Error("Failed to get number of global variables! Aborting...\n"); - exit(1); } err = ex_get_variable_param(file_id, EX_NODAL, &num_nodal_vars); if (err < 0) { Error("Failed to get number of nodal variables! Aborting...\n"); - exit(1); } err = ex_get_variable_param(file_id, EX_ELEM_BLOCK, &num_elmt_vars); if (err < 0) { Error("Failed to get number of element variables! Aborting...\n"); - exit(1); } err = ex_get_variable_param(file_id, EX_NODE_SET, &num_ns_vars); if (err < 0) { Error("Failed to get number of nodeset variables! Aborting...\n"); - exit(1); } err = ex_get_variable_param(file_id, EX_SIDE_SET, &num_ss_vars); if (err < 0) { Error("Failed to get number of sideset variables! Aborting...\n"); - exit(1); } err = ex_get_variable_param(file_id, EX_EDGE_BLOCK, &num_edge_vars); if (err < 0) { Error("Failed to get number of edgeblock variables! Aborting...\n"); - exit(1); } err = ex_get_variable_param(file_id, EX_FACE_BLOCK, &num_face_vars); if (err < 0) { Error("Failed to get number of faceblock variables! Aborting...\n"); - exit(1); } if (num_global_vars < 0 || num_nodal_vars < 0 || num_elmt_vars < 0 || num_ns_vars < 0 || @@ -1266,7 +1227,6 @@ template void ExoII_Read::Get_Init_Data() " ... Aborting...\n", num_global_vars, num_nodal_vars, num_elmt_vars, num_ns_vars, num_ss_vars, num_edge_vars, num_face_vars)); - exit(1); } read_vars(file_id, EX_GLOBAL, "Global", num_global_vars, global_vars); @@ -1281,15 +1241,13 @@ template void ExoII_Read::Get_Init_Data() num_times = ex_inquire_int(file_id, EX_INQ_TIME); if (num_times < 0) { Error(fmt::format("Number of time steps came back negative ({})! Aborting...\n", num_times)); - exit(1); } if ((num_global_vars > 0 || num_nodal_vars > 0 || num_elmt_vars > 0 || num_ns_vars > 0 || num_ss_vars > 0 || num_edge_vars > 0 || num_face_vars > 0) && num_times == 0) { - Error("Consistency error -- The database contains transient variables, but no " - "timesteps!\n"); - exit(1); + Warning("Consistency error -- The database contains transient variables, but no " + "timesteps!\n"); } if (num_times) { @@ -1300,9 +1258,9 @@ template void ExoII_Read::Get_Init_Data() if (num_nodal_vars != 0) { if (num_times == 0) { - Error(fmt::format("Consistency error--The database contains {}" - " nodal variables, but there are no time steps defined.\n", - num_nodal_vars)); + Warning(fmt::format("Consistency error--The database contains {}" + " nodal variables, but there are no time steps defined.\n", + num_nodal_vars)); } if (num_times) { results = new double *[num_nodal_vars]; @@ -1325,7 +1283,6 @@ namespace { if (err < 0) { Error(fmt::format("Failed to get {} variable names! Aborting...\n", type)); - exit(1); } else if (err > 0 && !interFace.quiet_flag) { fmt::print( @@ -1347,8 +1304,7 @@ namespace { fmt::print(out, "\t\t{}) \"{}\"\n", k, varnames[k - 1]); } fmt::print(out, " Aborting...\n"); - DIFF_OUT(out); - exit(1); + Error(out); } std::string n(varnames[vg]); diff --git a/packages/seacas/applications/exodiff/exoII_read.h b/packages/seacas/applications/exodiff/exoII_read.h index a30d1e3819d5..c184c0d97419 100644 --- a/packages/seacas/applications/exodiff/exoII_read.h +++ b/packages/seacas/applications/exodiff/exoII_read.h @@ -1,4 +1,4 @@ -// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// Copyright(C) 1999-2021 National Technology & Engineering Solutions // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with // NTESS, the U.S. Government retains certain rights in this software. // @@ -56,7 +56,9 @@ template class ExoII_Read const std::string &Title() const { return title; } int Dimension() const { return dimension; } size_t Num_Nodes() const { return num_nodes; } - size_t Num_Elmts() const { return num_elmts; } + size_t Num_Elements() const { return num_elmts; } + size_t Num_Faces() const { return num_faces; } + size_t Num_Edges() const { return num_edges; } size_t Num_Node_Sets() const { return num_node_sets; } size_t Num_Side_Sets() const { return num_side_sets; } size_t Num_Edge_Blocks() const { return num_edge_blocks; } @@ -71,16 +73,16 @@ template class ExoII_Read size_t Num_Global_Vars() const { return global_vars.size(); } size_t Num_Nodal_Vars() const { return nodal_vars.size(); } - size_t Num_Elmt_Vars() const { return elmt_vars.size(); } - size_t Num_Elmt_Atts() const { return elmt_atts.size(); } + size_t Num_Element_Vars() const { return elmt_vars.size(); } + size_t Num_Element_Atts() const { return elmt_atts.size(); } size_t Num_NS_Vars() const { return ns_vars.size(); } size_t Num_SS_Vars() const { return ss_vars.size(); } size_t Num_EB_Vars() const { return eb_vars.size(); } size_t Num_FB_Vars() const { return fb_vars.size(); } const std::vector &Global_Var_Names() const { return global_vars; } const std::vector &Nodal_Var_Names() const { return nodal_vars; } - const std::vector &Elmt_Var_Names() const { return elmt_vars; } - const std::vector &Elmt_Att_Names() const { return elmt_atts; } + const std::vector &Element_Var_Names() const { return elmt_vars; } + const std::vector &Element_Att_Names() const { return elmt_atts; } const std::vector &NS_Var_Names() const { return ns_vars; } const std::vector &SS_Var_Names() const { return ss_vars; } const std::vector &EB_Var_Names() const { return eb_vars; } @@ -88,34 +90,31 @@ template class ExoII_Read const std::string &Global_Var_Name(int index) const; const std::string &Nodal_Var_Name(int index) const; - const std::string &Elmt_Var_Name(int index) const; - const std::string &Elmt_Att_Name(int index) const; + const std::string &Element_Var_Name(int index) const; + const std::string &Element_Att_Name(int index) const; const std::string &NS_Var_Name(int index) const; const std::string &SS_Var_Name(int index) const; const std::string &EB_Var_Name(int index) const; const std::string &FB_Var_Name(int index) const; // Element blocks: - size_t Num_Elmt_Blocks() const { return num_elmt_blocks; } + size_t Num_Element_Blocks() const { return num_elmt_blocks; } - std::string Load_Elmt_Block_Description(size_t block_index) const; - std::string Load_Elmt_Block_Descriptions() const; // Loads all blocks. - std::string Free_Elmt_Block(size_t block_index) const; // Frees all dynamic memory. - std::string Free_Elmt_Blocks() const; // Frees all blocks. - - // Moves array of connectivities from the block to the conn array. - std::string Give_Connectivity(size_t block_index, size_t &num_e, size_t &npe, INT *&new_conn); + std::string Load_Element_Block_Description(size_t block_index) const; + std::string Load_Element_Block_Descriptions() const; // Loads all blocks. + std::string Free_Element_Block(size_t block_index) const; // Frees all dynamic memory. + std::string Free_Element_Blocks() const; // Frees all blocks. // Number maps: std::string Load_Node_Map(); std::string Free_Node_Map(); const INT * Get_Node_Map() { return node_map; } - std::string Load_Elmt_Map(); - std::string Free_Elmt_Map(); - const INT * Get_Elmt_Map() { return elmt_map; } - inline INT Node_Map(size_t node_num) const; // numbers are global, 1-offset - inline INT Elmt_Map(size_t elmt_num) const; // numbers are global, 1-offset - inline INT Elmt_Order(size_t elmt_num) const; // numbers are global, 1-offset + std::string Load_Element_Map(); + std::string Free_Element_Map(); + const INT * Get_Element_Map() { return elmt_map; } + inline INT Node_Map(size_t node_num) const; // numbers are global, 1-offset + inline INT Element_Map(size_t elmt_num) const; // numbers are global, 1-offset + inline INT Element_Order(size_t elmt_num) const; // numbers are global, 1-offset // Nodal data: @@ -160,9 +159,9 @@ template class ExoII_Read Exo_Entity *Get_Entity_by_Name(EXOTYPE type, const std::string &name) const; size_t Block_Id(size_t block_index) const; // Returns associated block id. - Exo_Block *Get_Elmt_Block_by_Id(size_t id) const; - Exo_Block *Get_Elmt_Block_by_Index(size_t block_index) const; - Exo_Block *Get_Elmt_Block_by_Name(const std::string &name) const; + Exo_Block *Get_Element_Block_by_Id(size_t id) const; + Exo_Block *Get_Element_Block_by_Index(size_t block_index) const; + Exo_Block *Get_Element_Block_by_Name(const std::string &name) const; Side_Set *Get_Side_Set_by_Id(size_t set_id) const; Side_Set *Get_Side_Set_by_Index(size_t side_set_index) const; @@ -182,11 +181,9 @@ template class ExoII_Read // Misc functions: - virtual int Check_State() const; // Checks state of obj (not the file). - int File_ID() const { return file_id; } // This is temporary. - std::string Global_to_Block_Local(size_t global_elmt_num, // 1-offset - int & block_index, // 0-offset - size_t &local_elmt_index) const; // 0-offset + virtual int Check_State() const; // Checks state of obj (not the file). + int File_ID() const { return file_id; } // This is temporary. + std::pair Global_to_Block_Local(size_t global_elmt_num) const; protected: std::string file_name; @@ -199,6 +196,8 @@ template class ExoII_Read size_t num_nodes{0}; int dimension{0}; size_t num_elmts{0}; + size_t num_faces{0}; + size_t num_edges{0}; size_t num_elmt_blocks{0}; size_t num_node_sets{0}; size_t num_side_sets{0}; @@ -257,7 +256,7 @@ template inline INT ExoII_Read::Node_Map(size_t node_num) co return 0; } -template inline INT ExoII_Read::Elmt_Map(size_t elmt_num) const +template inline INT ExoII_Read::Element_Map(size_t elmt_num) const { SMART_ASSERT(Check_State()); SMART_ASSERT(elmt_num <= num_elmts); @@ -268,7 +267,7 @@ template inline INT ExoII_Read::Elmt_Map(size_t elmt_num) co return 0; } -template inline INT ExoII_Read::Elmt_Order(size_t elmt_num) const +template inline INT ExoII_Read::Element_Order(size_t elmt_num) const { SMART_ASSERT(Check_State()); SMART_ASSERT(elmt_num <= num_elmts); diff --git a/packages/seacas/applications/exodiff/exo_block.C b/packages/seacas/applications/exodiff/exo_block.C index 8522fe3070a2..1f86b2185daf 100644 --- a/packages/seacas/applications/exodiff/exo_block.C +++ b/packages/seacas/applications/exodiff/exo_block.C @@ -32,13 +32,6 @@ Exo_Block::Exo_Block(int file_id, size_t id, const char *type, size_t num_e SMART_ASSERT(num_npe > 0); } -template Exo_Block::~Exo_Block() -{ - if (conn) { - delete[] conn; - } -} - template EXOTYPE Exo_Block::exodus_type() const { return EX_ELEM_BLOCK; } template void Exo_Block::entity_load_params() @@ -52,7 +45,6 @@ template void Exo_Block::entity_load_params() if (err < 0) { Error("Exo_Block::entity_load_params(): Failed to get element" " block parameters! Aborting...\n"); - exit(1); } numEntity = block.num_entry; @@ -67,7 +59,6 @@ template void Exo_Block::entity_load_params() "\tnum attributes = {}\n" " ... Aborting...\n", numEntity, num_nodes_per_elmt, num_attr)); - exit(1); } } @@ -81,22 +72,19 @@ template std::string Exo_Block::Load_Connectivity() if (id_ == EX_INVALID_ID) { return "ERROR: Must initialize block parameters first!"; } - if (conn) { - delete[] conn; + if (!conn.empty()) { + conn.clear(); } - conn = nullptr; if (numEntity && num_nodes_per_elmt) { - conn = new INT[numEntity * num_nodes_per_elmt]; - SMART_ASSERT(conn != nullptr); + conn.resize(numEntity * num_nodes_per_elmt); - int err = ex_get_conn(fileId, EX_ELEM_BLOCK, id_, conn, nullptr, nullptr); + int err = ex_get_conn(fileId, EX_ELEM_BLOCK, id_, conn.data(), nullptr, nullptr); if (err < 0) { Error(fmt::format("Exo_Block::Load_Connectivity(): Call to ex_get_conn returned error " "value! Block id = {}\n" "Aborting...\n", id_)); - exit(1); } else if (err > 0) { return fmt::format("WARNING: Number {} returned from call to ex_get_conn()", err); @@ -108,11 +96,8 @@ template std::string Exo_Block::Load_Connectivity() template std::string Exo_Block::Free_Connectivity() { - SMART_ASSERT(Check_State()); - if (conn) { - delete[] conn; - } - conn = nullptr; + conn.clear(); + SMART_ASSERT(conn.empty()); return ""; } @@ -120,36 +105,21 @@ template const INT *Exo_Block::Connectivity(size_t elmt_inde { SMART_ASSERT(Check_State()); - if (!conn || elmt_index >= numEntity) { + if (conn.empty() || elmt_index >= numEntity) { return nullptr; } return &conn[elmt_index * num_nodes_per_elmt]; } -template -std::string Exo_Block::Give_Connectivity(size_t &num_e, size_t &npe, INT *&recv_conn) -{ - if (num_nodes_per_elmt < 0) { - return "ERROR: Connectivity parameters have not been determined!"; - } - num_e = numEntity; - npe = num_nodes_per_elmt; - recv_conn = conn; - - conn = nullptr; // Transfers responsibility of deleting to the receiving pointer. - - return ""; -} - template int Exo_Block::Check_State() const { SMART_ASSERT(id_ >= EX_INVALID_ID); SMART_ASSERT(!(id_ == EX_INVALID_ID && elmt_type != "")); SMART_ASSERT(!(id_ == EX_INVALID_ID && num_nodes_per_elmt >= 0)); - SMART_ASSERT(!(id_ == EX_INVALID_ID && conn)); + SMART_ASSERT(!(id_ == EX_INVALID_ID && !conn.empty())); - SMART_ASSERT(!(conn && (numEntity == 0 || num_nodes_per_elmt <= 0))); + SMART_ASSERT(!(!conn.empty() && (numEntity == 0 || num_nodes_per_elmt <= 0))); return 1; } diff --git a/packages/seacas/applications/exodiff/exo_block.h b/packages/seacas/applications/exodiff/exo_block.h index e198de09c8ed..e38fa60f897a 100644 --- a/packages/seacas/applications/exodiff/exo_block.h +++ b/packages/seacas/applications/exodiff/exo_block.h @@ -20,7 +20,7 @@ template class Exo_Block : public Exo_Entity Exo_Block(); Exo_Block(int file_id, size_t exo_block_id); Exo_Block(int file_id, size_t id, const char *type, size_t num_e, size_t num_npe); - ~Exo_Block() override; + ~Exo_Block() override = default; Exo_Block(const Exo_Block &) = delete; const Exo_Block &operator=(const Exo_Block &) = delete; @@ -28,16 +28,12 @@ template class Exo_Block : public Exo_Entity std::string Free_Connectivity(); // Access functions: - const std::string &Elmt_Type() const { return elmt_type; } - size_t Num_Nodes_per_Elmt() const { return num_nodes_per_elmt; } + const std::string &Element_Type() const { return elmt_type; } + size_t Num_Nodes_per_Element() const { return num_nodes_per_elmt; } // Block description access functions: - const INT *Connectivity() const { return conn; } // 1-offset connectivity - const INT *Connectivity(size_t elmt_index) const; // 1-offset connectivity - - std::string Give_Connectivity(size_t &num_e, // Moves connectivity matrix - size_t &npe, // to conn pointer and sets - INT *& recv_conn); // its own to null. + const std::vector &Connectivity() const { return conn; } // 1-offset connectivity + const INT * Connectivity(size_t elmt_index) const; // 1-offset connectivity // Misc: int Check_State() const; @@ -52,10 +48,10 @@ template class Exo_Block : public Exo_Entity const char *label() const override { return "Element Block"; } const char *short_label() const override { return "block"; } - std::string elmt_type; - int num_nodes_per_elmt{-1}; - int64_t offset_{0}; - INT * conn{nullptr}; // Array; holds a matrix, num_elmts by num_nodes_per_elmt. + std::string elmt_type; + int num_nodes_per_elmt{-1}; + int64_t offset_{0}; + std::vector conn; // Array; holds a matrix, num_elmts by num_nodes_per_elmt. friend class ExoII_Read; }; diff --git a/packages/seacas/applications/exodiff/exo_entity.C b/packages/seacas/applications/exodiff/exo_entity.C index 12a4a1ab5721..f887903df7a6 100644 --- a/packages/seacas/applications/exodiff/exo_entity.C +++ b/packages/seacas/applications/exodiff/exo_entity.C @@ -99,7 +99,6 @@ std::string Exo_Entity::Load_Results(int time_step, int var_index) } if (var_index < 0 || var_index >= numVars) { Error("Exo_Entity::Load_Results(): var_index is invalid. Aborting...\n"); - exit(1); } SMART_ASSERT(time_step >= 1 && time_step <= (int)get_num_timesteps(fileId)); @@ -128,7 +127,6 @@ std::string Exo_Entity::Load_Results(int time_step, int var_index) " returned error value! {} id = {}\n" "Aborting...\n", label(), id_)); - exit(1); } else if (err > 0) { return fmt::format("WARNING: Number {} returned from call to exodus get variable routine.", @@ -185,7 +183,6 @@ std::string Exo_Entity::Load_Results(int t1, int t2, double proportion, int var_ "Exo_Entity::Load_Results(): Call to exodus routine returned error value! {} id = {}\n" "Aborting...\n", label(), id_)); - exit(1); } else if (err > 0) { return fmt::format("WARNING: Number {} returned from call to exodus get variable routine.", @@ -201,7 +198,6 @@ std::string Exo_Entity::Load_Results(int t1, int t2, double proportion, int var_ " returned error value! {} id = {}\n" "Aborting...\n", label(), id_)); - exit(1); } double *results1 = results_[var_index]; @@ -288,7 +284,6 @@ std::string Exo_Entity::Load_Attributes(int attr_index) " returned error value! {} id = {}\n" "Aborting...\n", label(), id_)); - exit(1); } else if (err > 0) { return fmt::format("WARNING: Number {} returned from call to exodus get variable routine.", @@ -374,7 +369,6 @@ void Exo_Entity::internal_load_params() Error(fmt::format( "ExoII_Read::Get_Init_Data(): Failed to get {} attribute names! Aborting...\n", label())); - exit(1); } for (int vg = 0; vg < numAttr; ++vg) { @@ -445,7 +439,7 @@ namespace { case EX_SIDE_SET: inquiry = EX_INQ_SIDE_SETS; break; case EX_EDGE_BLOCK: inquiry = EX_INQ_EDGE_BLK; break; case EX_FACE_BLOCK: inquiry = EX_INQ_FACE_BLK; break; - default: Error("Invalid entity type in get_num_entities\n"); exit(1); + default: Error("Invalid entity type in get_num_entities\n"); } SMART_ASSERT(inquiry != EX_INQ_INVALID); return ex_inquire_int(file_id, inquiry); @@ -457,7 +451,6 @@ namespace { int err = ex_get_variable_param(file_id, type, &num_vars); if (err < 0) { Error(fmt::format("Failed to get number of '{}' variables! Aborting...\n", label)); - exit(1); } return num_vars; } @@ -468,7 +461,6 @@ namespace { int err = ex_get_attr_param(file_id, type, id, &num_attr); if (err < 0) { Error(fmt::format("Failed to get number of '{}' attributes! Aborting...\n", label)); - exit(1); } return num_attr; } diff --git a/packages/seacas/applications/exodiff/exo_entity.h b/packages/seacas/applications/exodiff/exo_entity.h index 3423ec028dec..281838e70212 100644 --- a/packages/seacas/applications/exodiff/exo_entity.h +++ b/packages/seacas/applications/exodiff/exo_entity.h @@ -61,6 +61,15 @@ class Exo_Entity const std::vector &Attribute_Names() const { return attributeNames; } int Find_Attribute_Index(const std::string &name) const; + // Return "Element Block", "Nodeset", "Sideset, depending on underlying type. + virtual const char *label() const = 0; + + // Return "block", "nodelist", "surface", depending on underlying type. + virtual const char *short_label() const = 0; + + // Return EX_ELEM_BLOCK, EX_NODE_SET, ... of underlying type + virtual EXOTYPE exodus_type() const = 0; + protected: std::string name_; int fileId{-1}; @@ -72,15 +81,6 @@ class Exo_Entity virtual void entity_load_params() = 0; void internal_load_params(); - // Return "Element Block", "Nodeset", "Sideset, depending on underlying type. - virtual const char *label() const = 0; - - // Return "block", "nodelist", "surface", depending on underlying type. - virtual const char *short_label() const = 0; - - // Return EX_ELEM_BLOCK, EX_NODE_SET, ... of underlying type - virtual EXOTYPE exodus_type() const = 0; - void get_truth_table() const; mutable int *truth_{nullptr}; // Array; holds local truth table for this entity diff --git a/packages/seacas/applications/exodiff/exodiff.C b/packages/seacas/applications/exodiff/exodiff.C index 6a96836d4e7d..0f69329954a9 100644 --- a/packages/seacas/applications/exodiff/exodiff.C +++ b/packages/seacas/applications/exodiff/exodiff.C @@ -80,12 +80,6 @@ void Print_Banner(const char *prefix) prefix, Date()); } -// TODO(gdsjaar): - copy node & side sets - -// - copy node & element maps -// - copy coordinate variable names (instead of always using X,Y,Z) -// - more checks on success of Exodus calls - // Issues: - When mapping element numbers, blocks are irrelevant. Problem is // the variables that are determined to be stored in each file are // NOT independent of blocks .. in fact, that is how it determines @@ -101,8 +95,8 @@ template extern bool Check_Global(ExoII_Read &file1, ExoII_R template extern void Check_Compatible_Meshes(ExoII_Read &file1, ExoII_Read &file2, bool check_only, - const INT *node_map, const INT *elmt_map, - const INT *node_id_map); + const std::vector &node_map, + const std::vector &elmt_map, const INT *node_id_map); template int Create_File(ExoII_Read &file1, ExoII_Read &file2, const std::string &diffile_name, @@ -127,37 +121,59 @@ template const double *get_nodal_values(ExoII_Read &filen, const TimeInterp &t, size_t idx, size_t fno, const std::string &name, bool *diff_flag); +template +void do_summaries(ExoII_Read &file, int time_step, std::vector &mm_glob, + std::vector &mm_node, std::vector &mm_elmt, + std::vector &mm_ns, std::vector &mm_ss, + std::vector &mm_eb, std::vector &mm_fb, + const std::vector &elmt_map, bool *diff_flag); + template void do_diffs(ExoII_Read &file1, ExoII_Read &file2, int time_step1, const TimeInterp &t2, - int out_file_id, std::vector &mm_glob, std::vector &mm_node, - std::vector &mm_elmt, std::vector &mm_ns, - std::vector &mm_ss, std::vector &mm_eb, - std::vector &mm_fb, INT *node_map, const INT *node_id_map, INT *elmt_map, - const INT *elem_id_map, Exo_Block **blocks2, double *var_vals, bool *diff_flag); + int out_file_id, const std::vector &node_map, const INT *node_id_map, + const std::vector &elmt_map, const INT *elem_id_map, Exo_Block **blocks2, + std::vector &var_vals, bool *diff_flag); + +template +bool summarize_globals(ExoII_Read &file, int step, std::vector &mm_glob); +template +bool summarize_nodals(ExoII_Read &file, int step, std::vector &mm_node); +template +bool summarize_element(ExoII_Read &file, int step, const std::vector &elmt_map, + std::vector &mm_elmt); +template +bool summarize_nodeset(ExoII_Read &file, int step, std::vector &mm_ns); +template +bool summarize_sideset(ExoII_Read &file, int step, std::vector &mm_ss); +template +bool summarize_edgeblock(ExoII_Read &file, int step, std::vector &mm_eb); +template +bool summarize_faceblock(ExoII_Read &file, int step, std::vector &mm_fb); template bool diff_globals(ExoII_Read &file1, ExoII_Read &file2, int step1, const TimeInterp &t2, - int out_file_id, std::vector &mm_glob, double *gvals); + int out_file_id, std::vector &gvals); template bool diff_nodals(ExoII_Read &file1, ExoII_Read &file2, int step1, const TimeInterp &t2, - int out_file_id, INT *node_map, const INT *id_map, - std::vector &mm_node, double *nvals); + int out_file_id, const std::vector &node_map, const INT *id_map, + std::vector &nvals); template bool diff_element(ExoII_Read &file1, ExoII_Read &file2, int step1, const TimeInterp &t2, - int out_file_id, INT *elmt_map, const INT *id_map, Exo_Block **blocks2, - std::vector &mm_elmt, double *evals); + int out_file_id, const std::vector &elmt_map, const INT *id_map, + Exo_Block **blocks2, std::vector &evals); template -bool diff_element_attributes(ExoII_Read &file1, ExoII_Read &file2, INT *elmt_map, - const INT *id_map, Exo_Block **blocks2); +bool diff_element_attributes(ExoII_Read &file1, ExoII_Read &file2, + const std::vector &elmt_map, const INT *id_map, + Exo_Block **blocks2); template bool diff_nodeset(ExoII_Read &file1, ExoII_Read &file2, int step1, const TimeInterp &t2, - int out_file_id, const INT *id_map, std::vector &mm_ns, double *vals); + int out_file_id, const INT *id_map, std::vector &vals); template bool diff_sideset(ExoII_Read &file1, ExoII_Read &file2, int step1, const TimeInterp &t2, - int out_file_id, const INT *id_map, std::vector &mm_ss, double *vals); + int out_file_id, const INT *id_map, std::vector &vals); template bool diff_sideset_df(ExoII_Read &file1, ExoII_Read &file2, const INT *id_map); @@ -191,7 +207,6 @@ void floating_point_exception_handler(int signo) { if (!checking_invalid) { Error(fmt::format("caught floating point exception ({}) bad data?\n", signo)); - exit(1); } else { invalid_data = true; @@ -212,7 +227,6 @@ namespace { int exoid = ex_open(file_name.c_str(), EX_READ, &comp_ws, &ws, &dumb); if (exoid < 0) { Error(fmt::format("Couldn't open file \"{}\".\n", file_name)); - return 0; } int size = (ex_int64_status(exoid) & EX_ALL_INT64_DB) != 0 ? 8 : 4; ex_close(exoid); @@ -271,20 +285,26 @@ namespace { fmt::print( "{0} FILE {19}: {1}\n" "{0} Title: {2}\n" - "{0} Dim = {3}, Nodes = {5}, Elements = {6},\n" + "{0} Dim = {3}, Nodes = {5}, Elements = {6}, Faces = {20}, Edges = {21}\n" "{0} Element Blocks = {4}, Face Blocks = {10}, Edge Blocks = {9}, Nodesets = {7}, " "Sidesets = {8}\n" "{0} Vars: Global = {11}, Nodal = {12}, Element = {13}, Face = {17}, Edge = {18}, " - "Nodeset = {14}, " - "Sideset = {15}, Times = {16}\n\n", - prefix, fi.realpath(), file.Title(), file.Dimension(), file.Num_Elmt_Blocks(), - file.Num_Nodes(), file.Num_Elmts(), file.Num_Node_Sets(), file.Num_Side_Sets(), + "Nodeset = {14}, Sideset = {15}, Times = {16}\n\n", + prefix, fi.realpath(), file.Title(), file.Dimension(), file.Num_Element_Blocks(), + file.Num_Nodes(), file.Num_Elements(), file.Num_Node_Sets(), file.Num_Side_Sets(), file.Num_Edge_Blocks(), file.Num_Face_Blocks(), file.Num_Global_Vars(), - file.Num_Nodal_Vars(), file.Num_Elmt_Vars(), file.Num_NS_Vars(), file.Num_SS_Vars(), - file.Num_Times(), file.Num_FB_Vars(), file.Num_EB_Vars(), count); + file.Num_Nodal_Vars(), file.Num_Element_Vars(), file.Num_NS_Vars(), file.Num_SS_Vars(), + file.Num_Times(), file.Num_FB_Vars(), file.Num_EB_Vars(), count, file.Num_Faces(), + file.Num_Edges()); } - std::string buf; + void initialize(std::vector &mm_entity, size_t size, const ToleranceType &ttype) + { + mm_entity.resize(size); + for (auto &mm : mm_entity) { + mm.type = ttype; + } + } template bool exodiff(ExoII_Read &file1, ExoII_Read &file2); } // namespace @@ -320,7 +340,6 @@ int main(int argc, char *argv[]) if (interFace.summary_flag && file1_name == "") { Error(fmt::format("Summary option specified but an exodus " "file was not specified.\n")); - exit(1); } if (interFace.summary_flag) { @@ -336,14 +355,12 @@ int main(int argc, char *argv[]) interFace.fb_var_do_all_flag = true; interFace.map_flag = MapType::FILE_ORDER; interFace.quiet_flag = false; + Print_Banner("#"); } if (!interFace.quiet_flag && !interFace.summary_flag) { Print_Banner(" "); } - if (interFace.summary_flag) { - Print_Banner("#"); - } // Check integer sizes in input file(s)... int int_size = 4; @@ -394,7 +411,6 @@ namespace { std::string serr = file1.Open_File(); if (!serr.empty()) { Error(fmt::format("{}\n", serr)); - exit(1); } if (!interFace.summary_flag) { if (!interFace.quiet_flag) { @@ -403,7 +419,6 @@ namespace { serr = file2.Open_File(); if (!serr.empty()) { Error(fmt::format("{}\n", serr)); - exit(1); } } @@ -433,8 +448,8 @@ namespace { // When mapping is on ("-m"), node_map maps indexes from file1 to indexes // into file2. Similarly with elmt_map. - INT *node_map = nullptr; - INT *elmt_map = nullptr; + std::vector node_map; + std::vector elmt_map; if (interFace.map_flag == MapType::DISTANCE) { Compute_Maps(node_map, elmt_map, file1, file2); } @@ -450,23 +465,23 @@ namespace { } else { size_t num_nodes = file1.Num_Nodes(); - node_map = new INT[num_nodes]; - std::iota(node_map, node_map + num_nodes, 0); + node_map.resize(num_nodes); + std::iota(node_map.begin(), node_map.end(), 0); - size_t num_elem = file1.Num_Elmts(); - elmt_map = new INT[num_elem]; - std::iota(elmt_map, elmt_map + num_elem, 0); + size_t num_elem = file1.Num_Elements(); + elmt_map.resize(num_elem); + std::iota(elmt_map.begin(), elmt_map.end(), 0); } } else if (interFace.map_flag == MapType::FILE_ORDER) { // Match by implicit ordering... IDs in that ordering must match (checked later) size_t num_nodes = file1.Num_Nodes(); - node_map = new INT[num_nodes]; - std::iota(node_map, node_map + num_nodes, 0); + node_map.resize(num_nodes); + std::iota(node_map.begin(), node_map.end(), 0); - size_t num_elem = file1.Num_Elmts(); - elmt_map = new INT[num_elem]; - std::iota(elmt_map, elmt_map + num_elem, 0); + size_t num_elem = file1.Num_Elements(); + elmt_map.resize(num_elem); + std::iota(elmt_map.begin(), elmt_map.end(), 0); } else { Error("Invalid map option.\n"); @@ -488,9 +503,9 @@ namespace { const INT *elem_id_map = nullptr; if (!interFace.ignore_maps) { file1.Load_Node_Map(); - file1.Load_Elmt_Map(); + file1.Load_Element_Map(); node_id_map = file1.Get_Node_Map(); - elem_id_map = file1.Get_Elmt_Map(); + elem_id_map = file1.Get_Element_Map(); if (!interFace.summary_flag) { bool diff = Compare_Maps(file1, file2, node_map, elmt_map, interFace.map_flag == MapType::PARTIAL); @@ -514,9 +529,9 @@ namespace { INT *tmp_map = const_cast(node_id_map); std::iota(tmp_map, tmp_map + file1.Num_Nodes(), 1); - elem_id_map = new INT[file1.Num_Elmts()]; + elem_id_map = new INT[file1.Num_Elements()]; tmp_map = const_cast(elem_id_map); - std::iota(tmp_map, tmp_map + file1.Num_Elmts(), 1); + std::iota(tmp_map, tmp_map + file1.Num_Elements(), 1); } int out_file_id = -1; @@ -540,30 +555,36 @@ namespace { } } - double *var_vals = nullptr; + std::vector var_vals; if (out_file_id >= 0) { size_t max_ent = interFace.glob_var_names.size(); if (file1.Num_Nodes() > max_ent) { max_ent = file1.Num_Nodes(); } - if (file1.Num_Elmts() > max_ent) { - max_ent = file1.Num_Elmts(); + if (file1.Num_Elements() > max_ent) { + max_ent = file1.Num_Elements(); + } + if (file1.Num_Faces() > max_ent) { + max_ent = file1.Num_Faces(); + } + if (file1.Num_Edges() > max_ent) { + max_ent = file1.Num_Edges(); } - var_vals = new double[max_ent]; + var_vals.resize(max_ent); } // When mapping is in effect, it is efficient to grab pointers to all blocks. Exo_Block **blocks2 = nullptr; - if (elmt_map != nullptr) { - blocks2 = new Exo_Block *[file2.Num_Elmt_Blocks()]; - for (size_t b = 0; b < file2.Num_Elmt_Blocks(); ++b) { - blocks2[b] = file2.Get_Elmt_Block_by_Index(b); + if (!elmt_map.empty()) { + blocks2 = new Exo_Block *[file2.Num_Element_Blocks()]; + for (size_t b = 0; b < file2.Num_Element_Blocks(); ++b) { + blocks2[b] = file2.Get_Element_Block_by_Index(b); } } // Diff attributes... - if (!interFace.ignore_attributes && elmt_map == nullptr && !interFace.summary_flag) { + if (!interFace.ignore_attributes && elmt_map.empty() && !interFace.summary_flag) { if (diff_element_attributes(file1, file2, elmt_map, elem_id_map, blocks2)) { diff_flag = true; } @@ -590,55 +611,14 @@ namespace { std::vector mm_fb; if (interFace.summary_flag) { - int n; - if ((n = interFace.glob_var_names.size()) > 0) { - mm_glob.resize(n); - for (int i = 0; i < n; i++) { - mm_glob[i].type = ToleranceType::mm_global; - } - } - if ((n = interFace.node_var_names.size()) > 0) { - mm_node.resize(n); - for (int i = 0; i < n; i++) { - mm_node[i].type = ToleranceType::mm_nodal; - } - } - if ((n = interFace.elmt_var_names.size()) > 0) { - mm_elmt.resize(n); - for (int i = 0; i < n; i++) { - mm_elmt[i].type = ToleranceType::mm_element; - } - } - if ((n = interFace.elmt_att_names.size()) > 0) { - mm_eatt.resize(n); - for (int i = 0; i < n; i++) { - mm_eatt[i].type = ToleranceType::mm_elematt; - } - } - if ((n = interFace.ns_var_names.size()) > 0) { - mm_ns.resize(n); - for (int i = 0; i < n; i++) { - mm_ns[i].type = ToleranceType::mm_nodeset; - } - } - if ((n = interFace.ss_var_names.size()) > 0) { - mm_ss.resize(n); - for (int i = 0; i < n; i++) { - mm_ss[i].type = ToleranceType::mm_sideset; - } - } - if ((n = interFace.eb_var_names.size()) > 0) { - mm_eb.resize(n); - for (int i = 0; i < n; i++) { - mm_eb[i].type = ToleranceType::mm_edgeblock; - } - } - if ((n = interFace.fb_var_names.size()) > 0) { - mm_fb.resize(n); - for (int i = 0; i < n; i++) { - mm_fb[i].type = ToleranceType::mm_faceblock; - } - } + initialize(mm_glob, interFace.glob_var_names.size(), ToleranceType::mm_global); + initialize(mm_node, interFace.node_var_names.size(), ToleranceType::mm_nodal); + initialize(mm_elmt, interFace.elmt_var_names.size(), ToleranceType::mm_element); + initialize(mm_eatt, interFace.elmt_att_names.size(), ToleranceType::mm_elematt); + initialize(mm_ns, interFace.ns_var_names.size(), ToleranceType::mm_nodeset); + initialize(mm_ss, interFace.ss_var_names.size(), ToleranceType::mm_sideset); + initialize(mm_eb, interFace.eb_var_names.size(), ToleranceType::mm_edgeblock); + initialize(mm_fb, interFace.fb_var_names.size(), ToleranceType::mm_faceblock); } else { min_num_times = @@ -676,15 +656,22 @@ namespace { ts2); } else { - buf = fmt::format(" --------- Explicit Time step File 1: {}, {:13.7e} ~ File 2: {}, " - "{:13.7e} ---------", - ts1, file1.Time(ts1), ts2, t2.time); + std::string buf = + fmt::format(" --------- Explicit Time step File 1: {}, {:13.7e} ~ File 2: {}, " + "{:13.7e} ---------", + ts1, file1.Time(ts1), ts2, t2.time); DIFF_OUT(buf, fmt::color::green); } } - do_diffs(file1, file2, ts1, t2, out_file_id, mm_glob, mm_node, mm_elmt, mm_ns, mm_ss, mm_eb, - mm_fb, node_map, node_id_map, elmt_map, elem_id_map, blocks2, var_vals, &diff_flag); + if (interFace.summary_flag) { + do_summaries(file1, ts1, mm_glob, mm_node, mm_elmt, mm_ns, mm_ss, mm_eb, mm_fb, elmt_map, + &diff_flag); + } + else { + do_diffs(file1, file2, ts1, t2, out_file_id, node_map, node_id_map, elmt_map, elem_id_map, + blocks2, var_vals, &diff_flag); + } } else { @@ -696,7 +683,6 @@ namespace { if (interFace.time_step_offset < 0) { Error("Second database must have less timesteps than " "first database.\n"); - exit(1); } } @@ -707,7 +693,6 @@ namespace { if (file1.Num_Times() < file2.Num_Times()) { Error("Second database must have less timesteps than " "first database.\n"); - exit(1); } double t2 = file2.Time(1); @@ -758,7 +743,7 @@ namespace { } if (interFace.time_step_start > min_num_times && min_num_times > 0) { - Error("Time step options resulted in no timesteps being compared.\n"); + Warning("Time step options resulted in no timesteps being compared.\n"); diff_flag = true; } @@ -795,6 +780,7 @@ namespace { } else if (out_file_id < 0) { if (!interFace.quiet_flag) { + std::string buf; if (interFace.interpolating) { if (t2.step1 == -1) { buf = fmt::format( @@ -874,9 +860,14 @@ namespace { continue; } - do_diffs(file1, file2, time_step1, t2, out_file_id, mm_glob, mm_node, mm_elmt, mm_ns, mm_ss, - mm_eb, mm_fb, node_map, node_id_map, elmt_map, elem_id_map, blocks2, var_vals, - &diff_flag); + if (interFace.summary_flag) { + do_summaries(file1, time_step1, mm_glob, mm_node, mm_elmt, mm_ns, mm_ss, mm_eb, mm_fb, + elmt_map, &diff_flag); + } + else { + do_diffs(file1, file2, time_step1, t2, out_file_id, node_map, node_id_map, elmt_map, + elem_id_map, blocks2, var_vals, &diff_flag); + } } // End of time step loop. @@ -934,17 +925,14 @@ namespace { if (!interFace.ignore_maps) { file1.Free_Node_Map(); - file1.Free_Elmt_Map(); + file1.Free_Element_Map(); } else { delete[] node_id_map; delete[] elem_id_map; } - delete[] var_vals; delete[] blocks2; - delete[] node_map; - delete[] elmt_map; file1.Close_File(); if (!interFace.summary_flag) { @@ -1019,16 +1007,16 @@ void Die_TS(double ts) template size_t global_elmt_num(ExoII_Read &file, size_t b_idx, size_t e_idx) { - SMART_ASSERT(b_idx < file.Num_Elmt_Blocks()); + SMART_ASSERT(b_idx < file.Num_Element_Blocks()); size_t g = 0; - for (size_t b = 0; b < file.Num_Elmt_Blocks(); ++b) { + for (size_t b = 0; b < file.Num_Element_Blocks(); ++b) { if (b_idx == b) { return g + e_idx + 1; } - SMART_ASSERT(file.Get_Elmt_Block_by_Index(b) != 0); - g += file.Get_Elmt_Block_by_Index(b)->Size(); + SMART_ASSERT(file.Get_Element_Block_by_Index(b) != 0); + g += file.Get_Element_Block_by_Index(b)->Size(); } SMART_ASSERT(0); return 0; @@ -1084,7 +1072,7 @@ const double *get_nodal_values(ExoII_Read &filen, int time_step, size_t idx if (vals != nullptr) { if (Invalid_Values(vals, filen.Num_Nodes())) { - Error(fmt::format("NaN found for nodal variable '{}' in file {}\n", name, fno)); + Warning(fmt::format("NaN found for nodal variable '{}' in file {}\n", name, fno)); *diff_flag = true; } } @@ -1102,7 +1090,7 @@ const double *get_nodal_values(ExoII_Read &filen, const TimeInterp &t, size if (vals != nullptr) { if (Invalid_Values(vals, filen.Num_Nodes())) { - Error(fmt::format("NaN found for nodal variable '{}' in file {}\n", name, fno)); + Warning(fmt::format("NaN found for nodal variable '{}' in file {}\n", name, fno)); *diff_flag = true; } } @@ -1110,48 +1098,318 @@ const double *get_nodal_values(ExoII_Read &filen, const TimeInterp &t, size return vals; } +template +bool summarize_globals(ExoII_Read &file, int step, std::vector &mm_glob) +{ + bool diff_flag = false; + if (interFace.glob_var_names.empty()) { + return diff_flag; + } + + // Global variables. + file.Load_Global_Results(step); + const double *vals = file.Get_Global_Results(); + if (vals == nullptr) { + Error("Could not find global variables on file 1.\n"); + } + + for (unsigned out_idx = 0; out_idx < interFace.glob_var_names.size(); ++out_idx) { + const std::string &name = (interFace.glob_var_names)[out_idx]; + int idx = find_string(file.Global_Var_Names(), name, interFace.nocase_var_names); + if (idx < 0) { + Error(fmt::format("Unable to find global variable named '{}' on database.\n", name)); + } + mm_glob[out_idx].spec_min_max(vals[idx], step); + } + return diff_flag; +} + +template +bool summarize_nodals(ExoII_Read &file, int step, std::vector &mm_node) +{ + bool diff_flag = false; + for (unsigned n_idx = 0; n_idx < interFace.node_var_names.size(); ++n_idx) { + const std::string &name = (interFace.node_var_names)[n_idx]; + int idx = find_string(file.Nodal_Var_Names(), name, interFace.nocase_var_names); + if (idx < 0) { + Error(fmt::format("Unable to find nodal variable named '{}' on database.\n", name)); + } + const double *vals = get_nodal_values(file, step, idx, 1, name, &diff_flag); + + if (vals == nullptr) { + Error("Could not find nodal variables on file 1\n"); + } + + size_t ncount = file.Num_Nodes(); + for (size_t n = 0; n < ncount; ++n) { + mm_node[n_idx].spec_min_max(vals[n], step, n); + } + file.Free_Nodal_Results(idx); + } + file.Free_Nodal_Results(); + return diff_flag; +} + +const double *get_validated_variable(Exo_Entity *entity, int step, int vidx, + const std::string &name, bool *diff_flag) +{ + if (entity->Size() == 0) { + return nullptr; + } + if (!entity->is_valid_var(vidx)) { + return nullptr; + } + + entity->Load_Results(step, vidx); + const double *vals = entity->Get_Results(vidx); + if (vals == nullptr) { + Warning(fmt::format("Could not find variable '{}' in {} {}, file 1.\n", name, + entity->short_label(), entity->Id())); + *diff_flag = true; + return vals; + } + + if (Invalid_Values(vals, entity->Size())) { + Warning(fmt::format("NaN found for variable '{}' in {} {}, file 1\n", name, + entity->short_label(), entity->Id())); + *diff_flag = true; + } + return vals; +} + +const double *get_validated_variable(Exo_Entity *entity, const TimeInterp &t2, int vidx, + const std::string &name, bool *diff_flag) +{ + if (entity == nullptr) { + return nullptr; + } + if (entity->Size() == 0) { + return nullptr; + } + if (!entity->is_valid_var(vidx)) { + return nullptr; + } + + entity->Load_Results(t2.step1, t2.step2, t2.proportion, vidx); + const double *vals = entity->Get_Results(vidx); + if (vals == nullptr) { + Warning(fmt::format("Could not find variable '{}' in {} {}, file 2.\n", name, + entity->short_label(), entity->Id())); + *diff_flag = true; + return vals; + } + + if (Invalid_Values(vals, entity->Size())) { + Warning(fmt::format("NaN found for variable '{}' in {} {}, file 2.\n", name, + entity->short_label(), entity->Id())); + *diff_flag = true; + } + return vals; +} + +template +bool summarize_element(ExoII_Read &file, int step, const std::vector &elmt_map, + std::vector &mm_elmt) +{ + bool diff_flag = false; + + for (unsigned e_idx = 0; e_idx < interFace.elmt_var_names.size(); ++e_idx) { + const std::string &name = (interFace.elmt_var_names)[e_idx]; + int vidx = find_string(file.Element_Var_Names(), name, interFace.nocase_var_names); + if (vidx < 0) { + Error(fmt::format("Unable to find element variable named '{}' on database.\n", name)); + } + + size_t global_elmt_index = 0; + for (size_t b = 0; b < file.Num_Element_Blocks(); ++b) { + Exo_Block *eblock = file.Get_Element_Block_by_Index(b); + const double * vals = get_validated_variable(eblock, step, vidx, name, &diff_flag); + if (vals == nullptr) { + global_elmt_index += eblock->Size(); + continue; + } + + size_t ecount = eblock->Size(); + for (size_t e = 0; e < ecount; ++e) { + INT el_flag = 1; + if (!elmt_map.empty()) { + el_flag = elmt_map[global_elmt_index]; + } + + if (el_flag >= 0) { + mm_elmt[e_idx].spec_min_max(vals[e], step, global_elmt_index, eblock->Id()); + } + ++global_elmt_index; + } + + eblock->Free_Results(); + } + } + return diff_flag; +} + +template +bool summarize_nodeset(ExoII_Read &file, int step, std::vector &mm_ns) +{ + bool diff_flag = false; + for (unsigned e_idx = 0; e_idx < interFace.ns_var_names.size(); ++e_idx) { + const std::string &name = (interFace.ns_var_names)[e_idx]; + int vidx = find_string(file.NS_Var_Names(), name, interFace.nocase_var_names); + if (vidx < 0) { + Error(fmt::format("Unable to find nodeset variable named '{}' on database.\n", name)); + } + + for (size_t b = 0; b < file.Num_Node_Sets(); ++b) { + Node_Set *nset = file.Get_Node_Set_by_Index(b); + + const double *vals = get_validated_variable(nset, step, vidx, name, &diff_flag); + if (vals == nullptr) { + continue; + } + + size_t ncount = nset->Size(); + for (size_t e = 0; e < ncount; ++e) { + int idx = nset->Node_Index(e); + mm_ns[e_idx].spec_min_max(vals[idx], step, e, nset->Id()); + } + nset->Free_Results(); + } + } + return diff_flag; +} + +template +bool summarize_sideset(ExoII_Read &file, int step, std::vector &mm_ss) +{ + bool diff_flag = false; + for (unsigned e_idx = 0; e_idx < interFace.ss_var_names.size(); ++e_idx) { + const std::string &name = (interFace.ss_var_names)[e_idx]; + int vidx = find_string(file.SS_Var_Names(), name, interFace.nocase_var_names); + if (vidx < 0) { + Error(fmt::format("Unable to find sideset variable named '{}' on database.\n", name)); + } + + for (size_t b = 0; b < file.Num_Side_Sets(); ++b) { + Side_Set *sset = file.Get_Side_Set_by_Index(b); + + const double *vals = get_validated_variable(sset, step, vidx, name, &diff_flag); + if (vals == nullptr) { + continue; + } + + size_t ecount = sset->Size(); + for (size_t e = 0; e < ecount; ++e) { + size_t ind = sset->Side_Index(e); + mm_ss[e_idx].spec_min_max(vals[ind], step, e, sset->Id()); + } + sset->Free_Results(); + } + } + return diff_flag; +} + +template +bool summarize_edgeblock(ExoII_Read &file, int step, std::vector &mm_eb) +{ + bool diff_flag = false; + for (unsigned e_idx = 0; e_idx < interFace.eb_var_names.size(); ++e_idx) { + const std::string &name = (interFace.eb_var_names)[e_idx]; + int vidx = find_string(file.EB_Var_Names(), name, interFace.nocase_var_names); + if (vidx < 0) { + Error(fmt::format("Unable to find edge block variable named '{}' on database.\n", name)); + } + + for (size_t b = 0; b < file.Num_Edge_Blocks(); ++b) { + Edge_Block *eblock = file.Get_Edge_Block_by_Index(b); + + const double *vals = get_validated_variable(eblock, step, vidx, name, &diff_flag); + if (vals == nullptr) { + continue; + } + + size_t ecount = eblock->Size(); + for (size_t e = 0; e < ecount; ++e) { + size_t ind = eblock->Edge_Index(e); + mm_eb[e_idx].spec_min_max(vals[ind], step, e, eblock->Id()); + } + + eblock->Free_Results(); + } + } + return diff_flag; +} + +template +bool summarize_faceblock(ExoII_Read &file, int step, std::vector &mm_fb) +{ + bool diff_flag = false; + for (unsigned f_idx = 0; f_idx < interFace.fb_var_names.size(); ++f_idx) { + const std::string &name = (interFace.fb_var_names)[f_idx]; + int vidx = find_string(file.FB_Var_Names(), name, interFace.nocase_var_names); + if (vidx < 0) { + Error(fmt::format("Unable to find face block variable named '{}' on database.\n", name)); + } + + for (size_t b = 0; b < file.Num_Face_Blocks(); ++b) { + Face_Block *fblock = file.Get_Face_Block_by_Index(b); + + const double *vals = get_validated_variable(fblock, step, vidx, name, &diff_flag); + if (vals == nullptr) { + continue; + } + + size_t fcount = fblock->Size(); + for (size_t f = 0; f < fcount; ++f) { + size_t ind = fblock->Face_Index(f); + mm_fb[f_idx].spec_min_max(vals[ind], step, f, fblock->Id()); + } + + fblock->Free_Results(); + } + } + return diff_flag; +} + template void do_diffs(ExoII_Read &file1, ExoII_Read &file2, int time_step1, const TimeInterp &t2, - int out_file_id, std::vector &mm_glob, std::vector &mm_node, - std::vector &mm_elmt, std::vector &mm_ns, - std::vector &mm_ss, std::vector &mm_eb, - std::vector &mm_fb, INT *node_map, const INT *node_id_map, INT *elmt_map, - const INT *elem_id_map, Exo_Block **blocks2, double *var_vals, bool *diff_flag) + int out_file_id, const std::vector &node_map, const INT *node_id_map, + const std::vector &elmt_map, const INT *elem_id_map, Exo_Block **blocks2, + std::vector &var_vals, bool *diff_flag) { - if (diff_globals(file1, file2, time_step1, t2, out_file_id, mm_glob, var_vals)) { + SMART_ASSERT(!interFace.summary_flag); + if (diff_globals(file1, file2, time_step1, t2, out_file_id, var_vals)) { *diff_flag = true; } // Nodal variables. - if (diff_nodals(file1, file2, time_step1, t2, out_file_id, node_map, node_id_map, mm_node, - var_vals)) { + if (diff_nodals(file1, file2, time_step1, t2, out_file_id, node_map, node_id_map, var_vals)) { *diff_flag = true; } // Element variables. if (diff_element(file1, file2, time_step1, t2, out_file_id, elmt_map, elem_id_map, blocks2, - mm_elmt, var_vals)) { + var_vals)) { *diff_flag = true; } if (interFace.map_flag != MapType::PARTIAL) { // Nodeset variables. - if (diff_nodeset(file1, file2, time_step1, t2, out_file_id, node_id_map, mm_ns, var_vals)) { + if (diff_nodeset(file1, file2, time_step1, t2, out_file_id, node_id_map, var_vals)) { *diff_flag = true; } // Sideset variables. - if (diff_sideset(file1, file2, time_step1, t2, out_file_id, elem_id_map, mm_ss, var_vals)) { + if (diff_sideset(file1, file2, time_step1, t2, out_file_id, elem_id_map, var_vals)) { *diff_flag = true; } // Edge Block variables. - if (diff_edgeblock(file1, file2, time_step1, t2, out_file_id, elem_id_map, mm_eb, var_vals)) { + if (diff_edgeblock(file1, file2, time_step1, t2, out_file_id, elem_id_map, var_vals)) { *diff_flag = true; } // Face Block variables. - if (diff_faceblock(file1, file2, time_step1, t2, out_file_id, elem_id_map, mm_fb, var_vals)) { + if (diff_faceblock(file1, file2, time_step1, t2, out_file_id, elem_id_map, var_vals)) { *diff_flag = true; } } @@ -1164,9 +1422,56 @@ void do_diffs(ExoII_Read &file1, ExoII_Read &file2, int time_step1, co } } +template +void do_summaries(ExoII_Read &file, int time_step, std::vector &mm_glob, + std::vector &mm_node, std::vector &mm_elmt, + std::vector &mm_ns, std::vector &mm_ss, + std::vector &mm_eb, std::vector &mm_fb, + const std::vector &elmt_map, bool *diff_flag) +{ + SMART_ASSERT(interFace.summary_flag); + if (summarize_globals(file, time_step, mm_glob)) { + *diff_flag = true; + } + if (summarize_nodals(file, time_step, mm_node)) { + *diff_flag = true; + } + if (summarize_element(file, time_step, elmt_map, mm_elmt)) { + *diff_flag = true; + } + if (summarize_nodeset(file, time_step, mm_ns)) { + *diff_flag = true; + } + if (summarize_sideset(file, time_step, mm_ss)) { + *diff_flag = true; + } + if (summarize_edgeblock(file, time_step, mm_eb)) { + *diff_flag = true; + } + if (summarize_faceblock(file, time_step, mm_fb)) { + *diff_flag = true; + } +} + +void output_norms(Norm &norm, const std::string &name) +{ + if (interFace.doL1Norm && norm.diff(1) > 0.0) { + std::string buf = + fmt::format(" {:<{}} L1 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", name, + name_length(), norm.diff(1), norm.left(1), norm.right(1), norm.relative(1)); + DIFF_OUT(buf, fmt::color::green); + } + if (interFace.doL2Norm && norm.diff(2) > 0.0) { + std::string buf = + fmt::format(" {:<{}} L2 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", name, + name_length(), norm.diff(2), norm.left(2), norm.right(2), norm.relative(2)); + DIFF_OUT(buf, fmt::color::green); + } +} + template bool diff_globals(ExoII_Read &file1, ExoII_Read &file2, int step1, const TimeInterp &t2, - int out_file_id, std::vector &mm_glob, double *gvals) + int out_file_id, std::vector &gvals) { bool diff_flag = false; if (interFace.glob_var_names.empty()) { @@ -1178,56 +1483,34 @@ bool diff_globals(ExoII_Read &file1, ExoII_Read &file2, int step1, con const double *vals1 = file1.Get_Global_Results(); if (vals1 == nullptr) { Error("Could not find global variables on file 1.\n"); - exit(1); } - const double *vals2 = nullptr; - if (!interFace.summary_flag) { - file2.Load_Global_Results(t2.step1, t2.step2, t2.proportion); - vals2 = file2.Get_Global_Results(); - if (vals2 == nullptr) { - Error("Could not find global variables on file 2.\n"); - exit(1); - } + file2.Load_Global_Results(t2.step1, t2.step2, t2.proportion); + const double *vals2 = file2.Get_Global_Results(); + if (vals2 == nullptr) { + Error("Could not find global variables on file 2.\n"); } // ---------------------------------------------------------------------- // Output file containing differences... if (out_file_id >= 0) { - SMART_ASSERT(gvals != nullptr); + SMART_ASSERT(!gvals.empty()); for (unsigned out_idx = 0; out_idx < interFace.glob_var_names.size(); ++out_idx) { const std::string &name = (interFace.glob_var_names)[out_idx]; int idx1 = find_string(file1.Global_Var_Names(), name, interFace.nocase_var_names); int idx2 = find_string(file2.Global_Var_Names(), name, interFace.nocase_var_names); if (idx1 < 0 || idx2 < 0 || vals2 == nullptr) { Error(fmt::format("Unable to find global variable named '{}' on database.\n", name)); - exit(1); } gvals[out_idx] = FileDiff(vals1[idx1], vals2[idx2], interFace.output_type); } - ex_put_var(out_file_id, t2.step1, EX_GLOBAL, 1, 0, interFace.glob_var_names.size(), gvals); - return diff_flag; - } - - // -------------------------------------------------------------------- - // Summary output - if (interFace.summary_flag) { - for (unsigned out_idx = 0; out_idx < interFace.glob_var_names.size(); ++out_idx) { - const std::string &name = (interFace.glob_var_names)[out_idx]; - int idx1 = find_string(file1.Global_Var_Names(), name, interFace.nocase_var_names); - if (idx1 < 0) { - Error(fmt::format("Unable to find global variable named '{}' on database.\n", name)); - exit(1); - } - - mm_glob[out_idx].spec_min_max(vals1[idx1], step1); - } + ex_put_var(out_file_id, t2.step1, EX_GLOBAL, 1, 0, interFace.glob_var_names.size(), + gvals.data()); return diff_flag; } // ------------------------------------------------------------------- // Determine if any diffs and output to terminal - int name_length = max_string_length(file1.Global_Var_Names()) + 1; if (!interFace.quiet_flag && !interFace.glob_var_names.empty()) { fmt::print("Global variables:\n"); } @@ -1237,16 +1520,15 @@ bool diff_globals(ExoII_Read &file1, ExoII_Read &file2, int step1, con int idx2 = find_string(file2.Global_Var_Names(), name, interFace.nocase_var_names); if (idx1 < 0 || idx2 < 0) { Error(fmt::format("Unable to find global variable named '{}' on database.\n", name)); - exit(1); } if (Invalid_Values(&vals1[idx1], 1)) { - Error(fmt::format("NaN found for global variable '{}' in file 1\n", name)); + Warning(fmt::format("NaN found for global variable '{}' in file 1\n", name)); diff_flag = true; } if (Invalid_Values(&vals2[idx2], 1)) { - Error(fmt::format("NaN found for global variable '{}' in file 2\n", name)); + Warning(fmt::format("NaN found for global variable '{}' in file 2\n", name)); diff_flag = true; } @@ -1254,9 +1536,10 @@ bool diff_globals(ExoII_Read &file1, ExoII_Read &file2, int step1, con diff_flag = true; if (!interFace.quiet_flag) { - buf = fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (FAILED)", name, - name_length, interFace.glob_var[out_idx].abrstr(), vals1[idx1], - vals2[idx2], interFace.glob_var[out_idx].Delta(vals1[idx1], vals2[idx2])); + std::string buf = + fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (FAILED)", name, + name_length(), interFace.glob_var[out_idx].abrstr(), vals1[idx1], + vals2[idx2], interFace.glob_var[out_idx].Delta(vals1[idx1], vals2[idx2])); DIFF_OUT(buf); } else { @@ -1269,15 +1552,15 @@ bool diff_globals(ExoII_Read &file1, ExoII_Read &file2, int step1, con template bool diff_nodals(ExoII_Read &file1, ExoII_Read &file2, int step1, const TimeInterp &t2, - int out_file_id, INT *node_map, const INT *id_map, - std::vector &mm_node, double *nvals) + int out_file_id, const std::vector &node_map, const INT *id_map, + std::vector &nvals) { bool diff_flag = false; // --------------------------------------------------------------------- // Output file containing differences... if (out_file_id >= 0) { - SMART_ASSERT(nvals != nullptr); + SMART_ASSERT(!nvals.empty()); int step2 = t2.step1; for (unsigned n_idx = 0; n_idx < interFace.node_var_names.size(); ++n_idx) { const std::string &name = (interFace.node_var_names)[n_idx]; @@ -1285,7 +1568,6 @@ bool diff_nodals(ExoII_Read &file1, ExoII_Read &file2, int step1, cons int idx2 = find_string(file2.Nodal_Var_Names(), name, interFace.nocase_var_names); if (idx1 < 0 || idx2 < 0) { Error(fmt::format("Unable to find nodal variable named '{}' on database.\n", name)); - exit(1); } const double *vals1 = get_nodal_values(file1, step1, idx1, 1, name, &diff_flag); @@ -1293,27 +1575,25 @@ bool diff_nodals(ExoII_Read &file1, ExoII_Read &file2, int step1, cons if (vals1 == nullptr) { Error("Could not find nodal variables on file 1\n"); - exit(1); } if (vals2 == nullptr) { Error("Could not find nodal variables on file 2\n"); - exit(1); } size_t ncount = file1.Num_Nodes(); for (size_t n = 0; n < ncount; ++n) { // Should this node be processed... - if (node_map == nullptr || node_map[n] >= 0) { - INT n2 = node_map != nullptr ? node_map[n] : n; + if (node_map.empty() || node_map[n] >= 0) { + INT n2 = node_map.empty() ? n : node_map[n]; nvals[n] = FileDiff(vals1[n], vals2[n2], interFace.output_type); } else { nvals[n] = 0.; } } // End of node iteration... - ex_put_var(out_file_id, step2, EX_NODAL, n_idx + 1, 0, file1.Num_Nodes(), nvals); + ex_put_var(out_file_id, step2, EX_NODAL, n_idx + 1, 0, file1.Num_Nodes(), nvals.data()); file1.Free_Nodal_Results(idx1); file2.Free_Nodal_Results(idx2); } @@ -1322,61 +1602,31 @@ bool diff_nodals(ExoII_Read &file1, ExoII_Read &file2, int step1, cons return diff_flag; } - // ------------------------------------------------------------------- - // Summary output - if (interFace.summary_flag) { - for (unsigned n_idx = 0; n_idx < interFace.node_var_names.size(); ++n_idx) { - const std::string &name = (interFace.node_var_names)[n_idx]; - int idx1 = find_string(file1.Nodal_Var_Names(), name, interFace.nocase_var_names); - if (idx1 < 0) { - Error(fmt::format("Unable to find nodal variable named '{}' on database.\n", name)); - exit(1); - } - const double *vals1 = get_nodal_values(file1, step1, idx1, 1, name, &diff_flag); - - if (vals1 == nullptr) { - Error("Could not find nodal variables on file 1\n"); - exit(1); - } - - size_t ncount = file1.Num_Nodes(); - for (size_t n = 0; n < ncount; ++n) { - mm_node[n_idx].spec_min_max(vals1[n], step1, n); - } - file1.Free_Nodal_Results(idx1); - } - file1.Free_Nodal_Results(); - return diff_flag; - } - SMART_ASSERT(!interFace.summary_flag && out_file_id < 0); // ---------------------------------------------------------------------- // Determine if any diffs and output to terminal if (!interFace.quiet_flag && !interFace.node_var_names.empty()) { fmt::print("Nodal variables:\n"); } - int name_length = max_string_length(file1.Nodal_Var_Names()) + 1; - for (unsigned n_idx = 0; n_idx < interFace.node_var_names.size(); ++n_idx) { const std::string &name = (interFace.node_var_names)[n_idx]; int idx1 = find_string(file1.Nodal_Var_Names(), name, interFace.nocase_var_names); int idx2 = find_string(file2.Nodal_Var_Names(), name, interFace.nocase_var_names); if (idx1 < 0 || idx2 < 0) { Error(fmt::format("Unable to find nodal variable named '{}' on database.\n", name)); - exit(1); } const double *vals1 = get_nodal_values(file1, step1, idx1, 1, name, &diff_flag); const double *vals2 = get_nodal_values(file2, t2, idx2, 2, name, &diff_flag); if (vals1 == nullptr) { - Error(fmt::format("Could not find nodal variable '{}' on file 1.\n", name)); + Warning(fmt::format("Could not find nodal variable '{}' on file 1.\n", name)); diff_flag = true; continue; } if (vals2 == nullptr) { - Error(fmt::format("Could not find nodal variable '{}' on file 2.\n", name)); + Warning(fmt::format("Could not find nodal variable '{}' on file 2.\n", name)); diff_flag = true; continue; } @@ -1388,15 +1638,15 @@ bool diff_nodals(ExoII_Read &file1, ExoII_Read &file2, int step1, cons for (size_t n = 0; n < ncount; ++n) { // Should this node be processed... - if (node_map == nullptr || node_map[n] >= 0) { - INT n2 = node_map != nullptr ? node_map[n] : n; + if (node_map.empty() || node_map[n] >= 0) { + INT n2 = node_map.empty() ? n : node_map[n]; double d = interFace.node_var[n_idx].Delta(vals1[n], vals2[n2]); if (interFace.show_all_diffs) { if (d > interFace.node_var[n_idx].value) { - diff_flag = true; - buf = fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (node {})", name, - name_length, interFace.node_var[n_idx].abrstr(), vals1[n], vals2[n2], - d, id_map[n]); + diff_flag = true; + std::string buf = fmt::format( + " {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (node {})", name, name_length(), + interFace.node_var[n_idx].abrstr(), vals1[n], vals2[n2], d, id_map[n]); DIFF_OUT(buf); } } @@ -1407,25 +1657,15 @@ bool diff_nodals(ExoII_Read &file1, ExoII_Read &file2, int step1, cons } } // End of node iteration... - if (interFace.doL1Norm && norm.diff(1) > 0.0) { - buf = - fmt::format(" {:<{}} L1 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", name, - name_length, norm.diff(1), norm.left(1), norm.right(1), norm.relative(1)); - DIFF_OUT(buf, fmt::color::green); - } - if (interFace.doL2Norm && norm.diff(2) > 0.0) { - buf = - fmt::format(" {:<{}} L2 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", name, - name_length, norm.diff(2), norm.left(2), norm.right(2), norm.relative(2)); - DIFF_OUT(buf, fmt::color::green); - } + output_norms(norm, name); if (max_diff.diff > interFace.node_var[n_idx].value) { diff_flag = true; if (!interFace.quiet_flag) { - buf = fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (node {})", name, - name_length, interFace.node_var[n_idx].abrstr(), max_diff.val1, - max_diff.val2, max_diff.diff, id_map[max_diff.id]); + std::string buf = + fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (node {})", name, + name_length(), interFace.node_var[n_idx].abrstr(), max_diff.val1, + max_diff.val2, max_diff.diff, id_map[max_diff.id]); DIFF_OUT(buf); } else { @@ -1442,48 +1682,40 @@ bool diff_nodals(ExoII_Read &file1, ExoII_Read &file2, int step1, cons template bool diff_element(ExoII_Read &file1, ExoII_Read &file2, int step1, const TimeInterp &t2, - int out_file_id, INT *elmt_map, const INT *id_map, Exo_Block **blocks2, - std::vector &mm_elmt, double *evals) + int out_file_id, const std::vector &elmt_map, const INT *id_map, + Exo_Block **blocks2, std::vector &evals) { bool diff_flag = false; if (out_file_id >= 0) { - SMART_ASSERT(evals != nullptr); + SMART_ASSERT(!evals.empty()); } - int name_length = max_string_length(interFace.elmt_var_names) + 1; - - if (out_file_id < 0 && !interFace.quiet_flag && !interFace.summary_flag && - !interFace.elmt_var_names.empty()) { + if (out_file_id < 0 && !interFace.quiet_flag && !interFace.elmt_var_names.empty()) { fmt::print("Element variables:\n"); } for (unsigned e_idx = 0; e_idx < interFace.elmt_var_names.size(); ++e_idx) { const std::string &name = (interFace.elmt_var_names)[e_idx]; - int vidx1 = find_string(file1.Elmt_Var_Names(), name, interFace.nocase_var_names); - int vidx2 = 0; - if (!interFace.summary_flag) { - vidx2 = find_string(file2.Elmt_Var_Names(), name, interFace.nocase_var_names); - } + int vidx1 = find_string(file1.Element_Var_Names(), name, interFace.nocase_var_names); + int vidx2 = find_string(file2.Element_Var_Names(), name, interFace.nocase_var_names); if (vidx1 < 0 || vidx2 < 0) { Error(fmt::format("Unable to find element variable named '{}' on database.\n", name)); - exit(1); } Norm norm; - if (elmt_map != nullptr) { // Load variable for all blocks in file 2. - for (size_t b = 0; b < file2.Num_Elmt_Blocks(); ++b) { - Exo_Block *block2 = file2.Get_Elmt_Block_by_Index(b); + if (!elmt_map.empty()) { // Load variable for all blocks in file 2. + for (size_t b = 0; b < file2.Num_Element_Blocks(); ++b) { + Exo_Block *block2 = file2.Get_Element_Block_by_Index(b); block2->Load_Results(t2.step1, t2.step2, t2.proportion, vidx2); } } size_t global_elmt_index = 0; DiffData max_diff; - size_t e2; - for (size_t b = 0; b < file1.Num_Elmt_Blocks(); ++b) { - Exo_Block *eblock1 = file1.Get_Elmt_Block_by_Index(b); + for (size_t b = 0; b < file1.Num_Element_Blocks(); ++b) { + Exo_Block *eblock1 = file1.Get_Element_Block_by_Index(b); if (!eblock1->is_valid_var(vidx1)) { global_elmt_index += eblock1->Size(); continue; @@ -1493,13 +1725,12 @@ bool diff_element(ExoII_Read &file1, ExoII_Read &file2, int step1, con } Exo_Block *eblock2 = nullptr; - int b2 = b; - if (elmt_map == nullptr && !interFace.summary_flag) { + if (elmt_map.empty()) { if (interFace.by_name) { - eblock2 = file2.Get_Elmt_Block_by_Name(eblock1->Name()); + eblock2 = file2.Get_Element_Block_by_Name(eblock1->Name()); } else { - eblock2 = file2.Get_Elmt_Block_by_Id(eblock1->Id()); + eblock2 = file2.Get_Element_Block_by_Id(eblock1->Id()); } SMART_ASSERT(eblock2 != nullptr); @@ -1511,45 +1742,34 @@ bool diff_element(ExoII_Read &file1, ExoII_Read &file2, int step1, con eblock1->Load_Results(step1, vidx1); const double *vals1 = eblock1->Get_Results(vidx1); if (vals1 == nullptr) { - Error(fmt::format("Could not find element variable '{}' in block {}, file 1.\n", name, - eblock1->Id())); + Warning(fmt::format("Could not find element variable '{}' in block {}, file 1.\n", name, + eblock1->Id())); diff_flag = true; continue; } if (Invalid_Values(vals1, eblock1->Size())) { - Error(fmt::format("NaN found for element variable '{}' in block {}, file 1\n", name, - eblock1->Id())); + Warning(fmt::format("NaN found for element variable '{}' in block {}, file 1\n", name, + eblock1->Id())); diff_flag = true; } double v2 = 0; const double *vals2 = nullptr; - if (elmt_map == nullptr && !interFace.summary_flag) { + if (elmt_map.empty()) { // Without mapping, get result for this block. size_t id = eblock1->Id(); if (interFace.by_name) { - eblock2 = file2.Get_Elmt_Block_by_Name(eblock1->Name()); + eblock2 = file2.Get_Element_Block_by_Name(eblock1->Name()); } else { - eblock2 = file2.Get_Elmt_Block_by_Id(id); + eblock2 = file2.Get_Element_Block_by_Id(id); } - eblock2->Load_Results(t2.step1, t2.step2, t2.proportion, vidx2); - vals2 = eblock2->Get_Results(vidx2); - + vals2 = get_validated_variable(eblock2, t2, vidx2, name, &diff_flag); if (vals2 == nullptr) { - Error(fmt::format("Could not find element variable '{}' in block {}, file 2.\n", name, - eblock2->Id())); - diff_flag = true; continue; } - - if (Invalid_Values(vals2, eblock2->Size())) { - Error(fmt::format("NaN found for element variable '{}' in block {}, file 2\n", name, - eblock2->Id())); - diff_flag = true; - } } size_t ecount = eblock1->Size(); @@ -1559,44 +1779,45 @@ bool diff_element(ExoII_Read &file1, ExoII_Read &file2, int step1, con evals[e] = 0.; } INT el_flag = 1; - if (elmt_map != nullptr) { + if (!elmt_map.empty()) { el_flag = elmt_map[global_elmt_index]; } if (el_flag >= 0) { - if (!interFace.summary_flag) { - if (elmt_map == nullptr) { - v2 = vals2[e]; - } - else { - // With mapping, map global index from file 1 to global index - // for file 2. Then convert to block index and elmt index. - file2.Global_to_Block_Local(elmt_map[global_elmt_index] + 1, b2, e2); - SMART_ASSERT(blocks2[b2] != nullptr); - if (blocks2[b2]->is_valid_var(vidx2)) { - v2 = blocks2[b2]->Get_Results(vidx2)[e2]; // Get value from file 2. + if (elmt_map.empty()) { + v2 = vals2[e]; + } + else { + // With mapping, map global index from file 1 to global index + // for file 2. Then convert to block index and elmt index. + auto bl_idx = file2.Global_to_Block_Local(elmt_map[global_elmt_index] + 1); + SMART_ASSERT(blocks2[bl_idx.first] != nullptr); + if (blocks2[bl_idx.first]->is_valid_var(vidx2)) { + auto *tmp = blocks2[bl_idx.first]->Get_Results(vidx2); + if (tmp != nullptr) { + v2 = tmp[bl_idx.second]; // Get value from file 2. } else { - // Easiest from logic standpoint to just set v2 equal to v1 at - // this point and continue through rest of loop. - v2 = vals1[e]; + v2 = vals1[e]; // Should never happen... } } + else { + // Easiest from logic standpoint to just set v2 equal to v1 at + // this point and continue through rest of loop. + v2 = vals1[e]; + } } - if (interFace.summary_flag) { - mm_elmt[e_idx].spec_min_max(vals1[e], step1, global_elmt_index, block_id); - } - else if (out_file_id >= 0) { + if (out_file_id >= 0) { evals[e] = FileDiff(vals1[e], v2, interFace.output_type); } else if (interFace.show_all_diffs) { double d = interFace.elmt_var[e_idx].Delta(vals1[e], v2); if (d > interFace.elmt_var[e_idx].value) { - diff_flag = true; - buf = fmt::format( + diff_flag = true; + std::string buf = fmt::format( " {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (block {}, elmt {})", name, - name_length, interFace.elmt_var[e_idx].abrstr(), vals1[e], v2, d, block_id, + name_length(), interFace.elmt_var[e_idx].abrstr(), vals1[e], v2, d, block_id, id_map[global_elmt_index]); DIFF_OUT(buf); } @@ -1612,36 +1833,26 @@ bool diff_element(ExoII_Read &file1, ExoII_Read &file2, int step1, con if (out_file_id >= 0) { ex_put_var(out_file_id, t2.step1, EX_ELEM_BLOCK, e_idx + 1, eblock1->Id(), eblock1->Size(), - evals); + evals.data()); } eblock1->Free_Results(); - if (!interFace.summary_flag && elmt_map == nullptr) { + if (elmt_map.empty()) { eblock2->Free_Results(); } } // End of element block loop. - if (interFace.doL1Norm && norm.diff(1) > 0.0) { - buf = - fmt::format(" {:<{}} L1 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", name, - name_length, norm.diff(1), norm.left(1), norm.right(1), norm.relative(1)); - DIFF_OUT(buf, fmt::color::green); - } - if (interFace.doL2Norm && norm.diff(2) > 0.0) { - buf = - fmt::format(" {:<{}} L2 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", name, - name_length, norm.diff(2), norm.left(2), norm.right(2), norm.relative(2)); - DIFF_OUT(buf, fmt::color::green); - } + output_norms(norm, name); - if (!interFace.summary_flag && max_diff.diff > interFace.elmt_var[e_idx].value) { + if (max_diff.diff > interFace.elmt_var[e_idx].value) { diff_flag = true; if (!interFace.quiet_flag) { - buf = fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (block {}, elmt {})", - name, name_length, interFace.elmt_var[e_idx].abrstr(), max_diff.val1, - max_diff.val2, max_diff.diff, max_diff.blk, id_map[max_diff.id]); + std::string buf = + fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (block {}, elmt {})", + name, name_length(), interFace.elmt_var[e_idx].abrstr(), max_diff.val1, + max_diff.val2, max_diff.diff, max_diff.blk, id_map[max_diff.id]); DIFF_OUT(buf); } else { @@ -1655,30 +1866,23 @@ bool diff_element(ExoII_Read &file1, ExoII_Read &file2, int step1, con template bool diff_nodeset(ExoII_Read &file1, ExoII_Read &file2, int step1, const TimeInterp &t2, - int out_file_id, const INT *id_map, std::vector &mm_ns, double *vals) + int out_file_id, const INT *id_map, std::vector &vals) { bool diff_flag = false; if (out_file_id >= 0) { - SMART_ASSERT(vals != nullptr); + SMART_ASSERT(!vals.empty()); } - int name_length = max_string_length(file1.NS_Var_Names()) + 1; - - if (out_file_id < 0 && !interFace.quiet_flag && !interFace.summary_flag && - !interFace.ns_var_names.empty()) { + if (out_file_id < 0 && !interFace.quiet_flag && !interFace.ns_var_names.empty()) { fmt::print("Nodeset variables:\n"); } for (unsigned e_idx = 0; e_idx < interFace.ns_var_names.size(); ++e_idx) { const std::string &name = (interFace.ns_var_names)[e_idx]; int vidx1 = find_string(file1.NS_Var_Names(), name, interFace.nocase_var_names); - int vidx2 = 0; - if (!interFace.summary_flag) { - vidx2 = find_string(file2.NS_Var_Names(), name, interFace.nocase_var_names); - } + int vidx2 = find_string(file2.NS_Var_Names(), name, interFace.nocase_var_names); if (vidx1 < 0 || vidx2 < 0) { Error(fmt::format("Unable to find nodeset variable named '{}' on database.\n", name)); - exit(1); } DiffData max_diff; @@ -1686,144 +1890,79 @@ bool diff_nodeset(ExoII_Read &file1, ExoII_Read &file2, int step1, con for (size_t b = 0; b < file1.Num_Node_Sets(); ++b) { Node_Set *nset1 = file1.Get_Node_Set_by_Index(b); - SMART_ASSERT(nset1 != nullptr); - if (nset1->Size() == 0) { - continue; - } - if (!nset1->is_valid_var(vidx1)) { + const double * vals1 = get_validated_variable(nset1, step1, vidx1, name, &diff_flag); + if (vals1 == nullptr) { continue; } Node_Set *nset2 = nullptr; - if (!interFace.summary_flag) { - size_t id = nset1->Id(); - if (interFace.by_name) { - nset2 = file2.Get_Node_Set_by_Name(nset1->Name()); - } - else { - nset2 = file2.Get_Node_Set_by_Id(id); - } - if (nset2 == nullptr || !nset2->is_valid_var(vidx2)) { - continue; - } + size_t id = nset1->Id(); + if (interFace.by_name) { + nset2 = file2.Get_Node_Set_by_Name(nset1->Name()); } - - nset1->Load_Results(step1, vidx1); - const double *vals1 = nset1->Get_Results(vidx1); - - if (vals1 == nullptr) { - Error(fmt::format("Could not find variable '{}' in nodeset {}, file 1.\n", name, - nset1->Id())); - diff_flag = true; - continue; - } - - if (Invalid_Values(vals1, nset1->Size())) { - Error(fmt::format("NAN found for nodeset variable '{}' in nodeset {}, file 1.\n", name, - nset1->Id())); - diff_flag = true; + else { + nset2 = file2.Get_Node_Set_by_Id(id); } - - double v2 = 0; - const double *vals2 = nullptr; - if (!interFace.summary_flag) { - // Without mapping, get result for this nset - nset2->Load_Results(t2.step1, t2.step2, t2.proportion, vidx2); - vals2 = nset2->Get_Results(vidx2); - - if (vals2 == nullptr) { - Error(fmt::format("Could not find variable '{}' in nodeset {}, file 2.\n", name, - nset2->Id())); - diff_flag = true; - continue; - } - - if (Invalid_Values(vals2, nset2->Size())) { - Error(fmt::format("NAN found for nodeset variable '{}' in nodeset {}, file 2.\n", name, - nset2->Id())); - diff_flag = true; - } + const double *vals2 = get_validated_variable(nset2, t2, vidx2, name, &diff_flag); + if (vals2 == nullptr) { + continue; } size_t ncount = nset1->Size(); - if (interFace.summary_flag || nset2->Size() == ncount) { + if (nset2->Size() == ncount) { for (size_t e = 0; e < ncount; ++e) { int idx1 = nset1->Node_Index(e); - int idx2 = 0; + int idx2 = nset2->Node_Index(e); if (out_file_id >= 0) { - vals[idx1] = 0.; - } - if (!interFace.summary_flag) { - idx2 = nset2->Node_Index(e); - v2 = vals2[idx2]; - } - - if (interFace.summary_flag) { - mm_ns[e_idx].spec_min_max(vals1[idx1], step1, e, nset1->Id()); - } - else if (out_file_id >= 0) { - vals[idx1] = FileDiff(vals1[idx1], v2, interFace.output_type); + vals[idx1] = FileDiff(vals1[idx1], vals2[idx2], interFace.output_type); } else if (interFace.show_all_diffs) { - double d = interFace.ns_var[e_idx].Delta(vals1[idx1], v2); + double d = interFace.ns_var[e_idx].Delta(vals1[idx1], vals2[idx2]); if (d > interFace.ns_var[e_idx].value) { diff_flag = true; - buf = + std::string buf = fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (set {}, node {})", - name, name_length, interFace.ns_var[e_idx].abrstr(), vals1[idx1], v2, - d, nset1->Id(), e); + name, name_length(), interFace.ns_var[e_idx].abrstr(), vals1[idx1], + vals2[idx2], d, nset1->Id(), e); DIFF_OUT(buf); } } else { - double d = interFace.ns_var[e_idx].Delta(vals1[idx1], v2); - max_diff.set_max(d, vals1[idx1], v2, e, nset1->Id()); + double d = interFace.ns_var[e_idx].Delta(vals1[idx1], vals2[idx2]); + max_diff.set_max(d, vals1[idx1], vals2[idx2], e, nset1->Id()); } - norm.add_value(vals1[idx1], v2); + norm.add_value(vals1[idx1], vals2[idx2]); } if (out_file_id >= 0) { ex_put_var(out_file_id, t2.step1, EX_NODE_SET, e_idx + 1, nset1->Id(), nset1->Size(), - vals); + vals.data()); } } else { - buf = fmt::format(" {:<{}} diff: nodeset node counts differ for nodeset {}", name, - name_length, nset1->Id()); + std::string buf = + fmt::format(" {:<{}} diff: nodeset node counts differ for nodeset {}", name, + name_length(), nset1->Id()); DIFF_OUT(buf); diff_flag = true; } nset1->Free_Results(); - if (!interFace.summary_flag) { - nset2->Free_Results(); - } - + nset2->Free_Results(); } // End of nodeset loop. - if (interFace.doL1Norm && norm.diff(1) > 0.0) { - buf = - fmt::format(" {:<{}} L1 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", name, - name_length, norm.diff(1), norm.left(1), norm.right(1), norm.relative(1)); - DIFF_OUT(buf, fmt::color::green); - } - if (interFace.doL2Norm && norm.diff(2) > 0.0) { - buf = - fmt::format(" {:<{}} L2 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", name, - name_length, norm.diff(2), norm.left(2), norm.right(2), norm.relative(2)); - DIFF_OUT(buf, fmt::color::green); - } + output_norms(norm, name); - if (!interFace.summary_flag && max_diff.diff > interFace.ns_var[e_idx].value) { + if (max_diff.diff > interFace.ns_var[e_idx].value) { diff_flag = true; if (!interFace.quiet_flag) { Node_Set *nset = file1.Get_Node_Set_by_Id(max_diff.blk); - buf = - fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (set {}, node {})", name, - name_length, interFace.ns_var[e_idx].abrstr(), max_diff.val1, max_diff.val2, - max_diff.diff, max_diff.blk, id_map[nset->Node_Id(max_diff.id) - 1]); + std::string buf = fmt::format( + " {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (set {}, node {})", name, + name_length(), interFace.ns_var[e_idx].abrstr(), max_diff.val1, max_diff.val2, + max_diff.diff, max_diff.blk, id_map[nset->Node_Id(max_diff.id) - 1]); DIFF_OUT(buf); } else { @@ -1836,31 +1975,25 @@ bool diff_nodeset(ExoII_Read &file1, ExoII_Read &file2, int step1, con template bool diff_sideset(ExoII_Read &file1, ExoII_Read &file2, int step1, const TimeInterp &t2, - int out_file_id, const INT *id_map, std::vector &mm_ss, double *vals) + int out_file_id, const INT *id_map, std::vector &vals) { bool diff_flag = false; if (out_file_id >= 0) { - SMART_ASSERT(vals != nullptr); + SMART_ASSERT(!vals.empty()); } - int name_length = max_string_length(file1.SS_Var_Names()) + 1; - - if (out_file_id < 0 && !interFace.quiet_flag && !interFace.summary_flag && - !interFace.ss_var_names.empty()) { + if (out_file_id < 0 && !interFace.quiet_flag && !interFace.ss_var_names.empty()) { fmt::print("Sideset variables:\n"); } for (unsigned e_idx = 0; e_idx < interFace.ss_var_names.size(); ++e_idx) { const std::string &name = (interFace.ss_var_names)[e_idx]; int vidx1 = find_string(file1.SS_Var_Names(), name, interFace.nocase_var_names); - int vidx2 = 0; - if (!interFace.summary_flag) { - vidx2 = find_string(file2.SS_Var_Names(), name, interFace.nocase_var_names); - } + int vidx2 = find_string(file2.SS_Var_Names(), name, interFace.nocase_var_names); + if (vidx1 < 0 || vidx2 < 0) { Error(fmt::format("Unable to find sideset variable named '{}' on database.\n", name)); - exit(1); } DiffData max_diff; @@ -1869,139 +2002,78 @@ bool diff_sideset(ExoII_Read &file1, ExoII_Read &file2, int step1, con for (size_t b = 0; b < file1.Num_Side_Sets(); ++b) { Side_Set *sset1 = file1.Get_Side_Set_by_Index(b); SMART_ASSERT(sset1 != nullptr); - if (sset1->Size() == 0) { - continue; - } - if (!sset1->is_valid_var(vidx1)) { + const double *vals1 = get_validated_variable(sset1, step1, vidx1, name, &diff_flag); + if (vals1 == nullptr) { continue; } Side_Set *sset2 = nullptr; - if (!interFace.summary_flag) { - if (interFace.by_name) { - sset2 = file2.Get_Side_Set_by_Name(sset1->Name()); - } - else { - sset2 = file2.Get_Side_Set_by_Id(sset1->Id()); - } - if (sset2 == nullptr || !sset2->is_valid_var(vidx2)) { - continue; - } + if (interFace.by_name) { + sset2 = file2.Get_Side_Set_by_Name(sset1->Name()); } - - sset1->Load_Results(step1, vidx1); - const double *vals1 = sset1->Get_Results(vidx1); - - if (vals1 == nullptr) { - Error(fmt::format("Could not find variable '{}' in sideset {}, file 1.\n", name, - sset1->Id())); - diff_flag = true; - continue; - } - - if (Invalid_Values(vals1, sset1->Size())) { - Error(fmt::format("NaN found for sideset variable '{}' in sideset {}, file 1.\n", name, - sset1->Id())); - diff_flag = true; + else { + sset2 = file2.Get_Side_Set_by_Id(sset1->Id()); } - - double v2 = 0; - const double *vals2 = nullptr; - if (!interFace.summary_flag) { - sset2->Load_Results(t2.step1, t2.step2, t2.proportion, vidx2); - vals2 = sset2->Get_Results(vidx2); - - if (vals2 == nullptr) { - Error(fmt::format("Could not find variable '{}' in sideset {}, file 2.\n", name, - sset2->Id())); - diff_flag = true; - continue; - } - - if (Invalid_Values(vals2, sset2->Size())) { - Error(fmt::format("NaN found for sideset variable '{}' in sideset {}, file 2.\n", name, - sset2->Id())); - diff_flag = true; - } + const double *vals2 = get_validated_variable(sset2, t2, vidx2, name, &diff_flag); + if (vals2 == nullptr) { + continue; } size_t ecount = sset1->Size(); - if (interFace.summary_flag || sset2->Size() == ecount) { + if (sset2->Size() == ecount) { for (size_t e = 0; e < ecount; ++e) { size_t ind1 = sset1->Side_Index(e); - size_t ind2 = 0; - if (out_file_id >= 0) { - vals[ind1] = 0.; - } - if (!interFace.summary_flag) { - ind2 = sset2->Side_Index(e); - v2 = vals2[ind2]; - } + size_t ind2 = sset2->Side_Index(e); - if (interFace.summary_flag) { - mm_ss[e_idx].spec_min_max(vals1[ind1], step1, e, sset1->Id()); - } - else if (out_file_id >= 0) { - vals[ind1] = FileDiff(vals1[ind1], v2, interFace.output_type); + if (out_file_id >= 0) { + vals[ind1] = FileDiff(vals1[ind1], vals2[ind2], interFace.output_type); } else if (interFace.show_all_diffs) { - double d = interFace.ss_var[e_idx].Delta(vals1[ind1], v2); + double d = interFace.ss_var[e_idx].Delta(vals1[ind1], vals2[ind2]); if (d > interFace.ss_var[e_idx].value) { - diff_flag = true; - buf = fmt::format( + diff_flag = true; + std::string buf = fmt::format( " {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (set {}, side {}.{})", name, - name_length, interFace.ss_var[e_idx].abrstr(), vals1[ind1], v2, d, sset1->Id(), - id_map[sset1->Side_Id(e).first - 1], (int)sset1->Side_Id(e).second); + name_length(), interFace.ss_var[e_idx].abrstr(), vals1[ind1], vals2[ind2], d, + sset1->Id(), id_map[sset1->Side_Id(e).first - 1], (int)sset1->Side_Id(e).second); DIFF_OUT(buf); } } else { - double d = interFace.ss_var[e_idx].Delta(vals1[ind1], v2); - max_diff.set_max(d, vals1[ind1], v2, e, sset1->Id()); + double d = interFace.ss_var[e_idx].Delta(vals1[ind1], vals2[ind2]); + max_diff.set_max(d, vals1[ind1], vals2[ind2], e, sset1->Id()); } - norm.add_value(vals1[ind1], v2); + norm.add_value(vals1[ind1], vals2[ind2]); } if (out_file_id >= 0) { ex_put_var(out_file_id, t2.step1, EX_SIDE_SET, e_idx + 1, sset1->Id(), sset1->Size(), - vals); + vals.data()); } } else { - buf = fmt::format(" {:<{}} diff: sideset side counts differ for sideset {}", name, - name_length, sset1->Id()); + std::string buf = + fmt::format(" {:<{}} diff: sideset side counts differ for sideset {}", name, + name_length(), sset1->Id()); DIFF_OUT(buf); diff_flag = true; } sset1->Free_Results(); - if (!interFace.summary_flag) { - sset2->Free_Results(); - } + sset2->Free_Results(); } // End of sideset loop. - if (!interFace.summary_flag && max_diff.diff > interFace.ss_var[e_idx].value) { + if (max_diff.diff > interFace.ss_var[e_idx].value) { diff_flag = true; - if (interFace.doL1Norm && norm.diff(1) > 0.0) { - buf = fmt::format(" {:<{}} L1 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", - name, name_length, norm.diff(1), norm.left(1), norm.right(1), - norm.relative(1)); - DIFF_OUT(buf, fmt::color::green); - } - if (interFace.doL2Norm && norm.diff(2) > 0.0) { - buf = fmt::format(" {:<{}} L2 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", - name, name_length, norm.diff(2), norm.left(2), norm.right(2), - norm.relative(2)); - DIFF_OUT(buf, fmt::color::green); - } + output_norms(norm, name); if (!interFace.quiet_flag) { Side_Set *sset = file1.Get_Side_Set_by_Id(max_diff.blk); - buf = fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (set {}, side {}.{})", - name, name_length, interFace.ss_var[e_idx].abrstr(), max_diff.val1, - max_diff.val2, max_diff.diff, max_diff.blk, - id_map[sset->Side_Id(max_diff.id).first - 1], - (int)sset->Side_Id(max_diff.id).second); + std::string buf = fmt::format( + " {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (set {}, side {}.{})", name, + name_length(), interFace.ss_var[e_idx].abrstr(), max_diff.val1, max_diff.val2, + max_diff.diff, max_diff.blk, id_map[sset->Side_Id(max_diff.id).first - 1], + (int)sset->Side_Id(max_diff.id).second); DIFF_OUT(buf); } else { @@ -2047,7 +2119,7 @@ bool diff_sideset_df(ExoII_Read &file1, ExoII_Read &file2, const INT * const double *vals1 = sset1->Distribution_Factors(); if (vals1 == nullptr) { - Error( + Warning( fmt::format("Could not read distribution factors in sideset {}, file 1.\n", sset1->Id())); diff_flag = true; continue; @@ -2063,8 +2135,8 @@ bool diff_sideset_df(ExoII_Read &file1, ExoII_Read &file2, const INT * { std::pair range1 = sset1->Distribution_Factor_Range(ecount - 1); if (Invalid_Values(vals1, range1.second)) { - Error(fmt::format("NaN found for distribution factors in sideset {}, file 1.\n", - sset1->Id())); + Warning(fmt::format("NaN found for distribution factors in sideset {}, file 1.\n", + sset1->Id())); diff_flag = true; } @@ -2075,7 +2147,7 @@ bool diff_sideset_df(ExoII_Read &file1, ExoII_Read &file2, const INT * auto *vals2 = sset2->Distribution_Factors(); if (vals2 == nullptr) { - Error( + Warning( fmt::format("Could not read distribution factors in sideset {}, file 2.\n", sset2->Id())); diff_flag = true; continue; @@ -2084,8 +2156,8 @@ bool diff_sideset_df(ExoII_Read &file1, ExoII_Read &file2, const INT * { std::pair range2 = sset2->Distribution_Factor_Range(sset2->Size() - 1); if (Invalid_Values(vals2, range2.second)) { - Error(fmt::format("NaN found for distribution factors in sideset {}, file 2.\n", - sset2->Id())); + Warning(fmt::format("NaN found for distribution factors in sideset {}, file 2.\n", + sset2->Id())); diff_flag = true; } @@ -2110,12 +2182,12 @@ bool diff_sideset_df(ExoII_Read &file1, ExoII_Read &file2, const INT * if (interFace.show_all_diffs) { double d = interFace.ss_df_tol.Delta(v1, v2); if (d > interFace.ss_df_tol.value) { - diff_flag = true; - buf = fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (set {}, side {}" - ".{}-{})", - name, name_length, interFace.ss_df_tol.abrstr(), v1, v2, d, - sset1->Id(), id_map[sset1->Side_Id(e).first - 1], - (int)sset1->Side_Id(e).second, (int)i + 1); + diff_flag = true; + std::string buf = fmt::format( + " {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (set {}, side {}" + ".{}-{})", + name, name_length, interFace.ss_df_tol.abrstr(), v1, v2, d, sset1->Id(), + id_map[sset1->Side_Id(e).first - 1], (int)sset1->Side_Id(e).second, (int)i + 1); DIFF_OUT(buf); } } @@ -2127,8 +2199,8 @@ bool diff_sideset_df(ExoII_Read &file1, ExoII_Read &file2, const INT * } } else { - buf = fmt::format(" {:<{}} diff: sideset side counts differ for sideset {}", name, - name_length, sset1->Id()); + std::string buf = fmt::format(" {:<{}} diff: sideset side counts differ for sideset {}", + name, name_length, sset1->Id()); DIFF_OUT(buf); diff_flag = true; } @@ -2142,7 +2214,7 @@ bool diff_sideset_df(ExoII_Read &file1, ExoII_Read &file2, const INT * if (!interFace.quiet_flag) { Side_Set *sset = file1.Get_Side_Set_by_Id(max_diff.blk); - buf = + std::string buf = fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (set {}, side {}.{})", name, name_length, interFace.ss_df_tol.abrstr(), max_diff.val1, max_diff.val2, max_diff.diff, max_diff.blk, id_map[sset->Side_Id(max_diff.id).first - 1], @@ -2159,32 +2231,24 @@ bool diff_sideset_df(ExoII_Read &file1, ExoII_Read &file2, const INT * template bool diff_edgeblock(ExoII_Read &file1, ExoII_Read &file2, int step1, const TimeInterp &t2, - int out_file_id, const INT *id_map, std::vector &mm_eb, - double *vals) + int out_file_id, const INT *id_map, std::vector &vals) { bool diff_flag = false; if (out_file_id >= 0) { - SMART_ASSERT(vals != nullptr); + SMART_ASSERT(!vals.empty()); } - int name_length = max_string_length(file1.EB_Var_Names()) + 1; - - if (out_file_id < 0 && !interFace.quiet_flag && !interFace.summary_flag && - !interFace.eb_var_names.empty()) { + if (out_file_id < 0 && !interFace.quiet_flag && !interFace.eb_var_names.empty()) { fmt::print("Edge Block variables:\n"); } for (unsigned e_idx = 0; e_idx < interFace.eb_var_names.size(); ++e_idx) { const std::string &name = (interFace.eb_var_names)[e_idx]; int vidx1 = find_string(file1.EB_Var_Names(), name, interFace.nocase_var_names); - int vidx2 = 0; - if (!interFace.summary_flag) { - vidx2 = find_string(file2.EB_Var_Names(), name, interFace.nocase_var_names); - } + int vidx2 = find_string(file2.EB_Var_Names(), name, interFace.nocase_var_names); if (vidx1 < 0 || vidx2 < 0) { Error(fmt::format("Unable to find edge block variable named '{}' on database.\n", name)); - exit(1); } DiffData max_diff; @@ -2192,140 +2256,77 @@ bool diff_edgeblock(ExoII_Read &file1, ExoII_Read &file2, int step1, c for (size_t b = 0; b < file1.Num_Edge_Blocks(); ++b) { Edge_Block *eblock1 = file1.Get_Edge_Block_by_Index(b); - SMART_ASSERT(eblock1 != nullptr); - if (eblock1->Size() == 0) { - std::cout << "eblock1->Size() == 0 ...continuing...\n"; - continue; - } - if (!eblock1->is_valid_var(vidx1)) { + const double * vals1 = get_validated_variable(eblock1, step1, vidx1, name, &diff_flag); + if (vals1 == nullptr) { continue; } Edge_Block *eblock2 = nullptr; - if (!interFace.summary_flag) { - if (interFace.by_name) { - eblock2 = file2.Get_Edge_Block_by_Name(eblock1->Name()); - } - else { - eblock2 = file2.Get_Edge_Block_by_Id(eblock1->Id()); - } - if (eblock2 == nullptr || !eblock2->is_valid_var(vidx2)) { - continue; - } - } - - eblock1->Load_Results(step1, vidx1); - const double *vals1 = eblock1->Get_Results(vidx1); - - if (vals1 == nullptr) { - Error(fmt::format("Could not find variable '{}' in edge block {}, file 1.\n", name, - eblock1->Id())); - diff_flag = true; - continue; + if (interFace.by_name) { + eblock2 = file2.Get_Edge_Block_by_Name(eblock1->Name()); } - - if (Invalid_Values(vals1, eblock1->Size())) { - Error(fmt::format("NaN found for edge block variable '{}' in edge block {}, file 1.\n", - name, eblock1->Id())); - diff_flag = true; + else { + eblock2 = file2.Get_Edge_Block_by_Id(eblock1->Id()); } - - double v2 = 0; - const double *vals2 = nullptr; - if (!interFace.summary_flag) { - eblock2->Load_Results(t2.step1, t2.step2, t2.proportion, vidx2); - vals2 = eblock2->Get_Results(vidx2); - - if (vals2 == nullptr) { - Error(fmt::format("Could not find variable '{}' in edge block {}, file 2.\n", name, - eblock2->Id())); - diff_flag = true; - continue; - } - - if (Invalid_Values(vals2, eblock2->Size())) { - Error(fmt::format("NaN found for edge block variable '{}' in edge block {}, file 2.\n", - name, eblock2->Id())); - diff_flag = true; - } + const double *vals2 = get_validated_variable(eblock2, t2, vidx2, name, &diff_flag); + if (vals2 == nullptr) { + continue; } size_t ecount = eblock1->Size(); - if (interFace.summary_flag || eblock2->Size() == ecount) { + if (eblock2->Size() == ecount) { for (size_t e = 0; e < ecount; ++e) { size_t ind1 = eblock1->Edge_Index(e); - size_t ind2 = 0; - if (out_file_id >= 0) { - vals[ind1] = 0.; - } - if (!interFace.summary_flag) { - ind2 = eblock2->Edge_Index(e); - v2 = vals2[ind2]; - } + size_t ind2 = eblock2->Edge_Index(e); - if (interFace.summary_flag) { - mm_eb[e_idx].spec_min_max(vals1[ind1], step1, e, eblock1->Id()); - } - else if (out_file_id >= 0) { - vals[ind1] = FileDiff(vals1[ind1], v2, interFace.output_type); + if (out_file_id >= 0) { + vals[ind1] = FileDiff(vals1[ind1], vals2[ind2], interFace.output_type); } else if (interFace.show_all_diffs) { - double d = interFace.eb_var[e_idx].Delta(vals1[ind1], v2); + double d = interFace.eb_var[e_idx].Delta(vals1[ind1], vals2[ind2]); if (d > interFace.eb_var[e_idx].value) { - diff_flag = true; - buf = fmt::format( + diff_flag = true; + std::string buf = fmt::format( " {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (edge block {}, edge {})", name, - name_length, interFace.eb_var[e_idx].abrstr(), vals1[ind1], v2, d, eblock1->Id(), - eblock1->Edge_Index(e)); + name_length(), interFace.eb_var[e_idx].abrstr(), vals1[ind1], vals2[ind2], d, + eblock1->Id(), eblock1->Edge_Index(e) + 1); DIFF_OUT(buf); } } else { - double d = interFace.eb_var[e_idx].Delta(vals1[ind1], v2); - max_diff.set_max(d, vals1[ind1], v2, e, eblock1->Id()); + double d = interFace.eb_var[e_idx].Delta(vals1[ind1], vals2[ind2]); + max_diff.set_max(d, vals1[ind1], vals2[ind2], e, eblock1->Id()); } - norm.add_value(vals1[ind1], v2); + norm.add_value(vals1[ind1], vals2[ind2]); } if (out_file_id >= 0) { ex_put_var(out_file_id, t2.step1, EX_EDGE_BLOCK, e_idx + 1, eblock1->Id(), - eblock1->Size(), vals); + eblock1->Size(), vals.data()); } } else { - buf = fmt::format(" {:<{}} diff: edge block edge counts differ for edge block {}", - name, name_length, eblock1->Id()); + std::string buf = + fmt::format(" {:<{}} diff: edge block edge counts differ for edge block {}", name, + name_length(), eblock1->Id()); DIFF_OUT(buf); diff_flag = true; } eblock1->Free_Results(); - if (!interFace.summary_flag) { - eblock2->Free_Results(); - } + eblock2->Free_Results(); } // End of edgeblock loop. - if (!interFace.summary_flag && max_diff.diff > interFace.eb_var[e_idx].value) { + if (max_diff.diff > interFace.eb_var[e_idx].value) { diff_flag = true; - if (interFace.doL1Norm && norm.diff(1) > 0.0) { - buf = fmt::format(" {:<{}} L1 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", - name, name_length, norm.diff(1), norm.left(1), norm.right(1), - norm.relative(1)); - DIFF_OUT(buf, fmt::color::green); - } - if (interFace.doL2Norm && norm.diff(2) > 0.0) { - buf = fmt::format(" {:<{}} L2 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", - name, name_length, norm.diff(2), norm.left(2), norm.right(2), - norm.relative(2)); - DIFF_OUT(buf, fmt::color::green); - } + output_norms(norm, name); if (!interFace.quiet_flag) { Edge_Block *eblock = file1.Get_Edge_Block_by_Id(max_diff.blk); - buf = fmt::format( + std::string buf = fmt::format( " {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (edge block {}, edge {})", name, - name_length, interFace.eb_var[e_idx].abrstr(), max_diff.val1, max_diff.val2, - max_diff.diff, max_diff.blk, eblock->Edge_Index(max_diff.id)); + name_length(), interFace.eb_var[e_idx].abrstr(), max_diff.val1, max_diff.val2, + max_diff.diff, max_diff.blk, eblock->Edge_Index(max_diff.id) + 1); DIFF_OUT(buf); } else { @@ -2339,32 +2340,24 @@ bool diff_edgeblock(ExoII_Read &file1, ExoII_Read &file2, int step1, c template bool diff_faceblock(ExoII_Read &file1, ExoII_Read &file2, int step1, const TimeInterp &t2, - int out_file_id, const INT *id_map, std::vector &mm_fb, - double *vals) + int out_file_id, const INT *id_map, std::vector &vals) { bool diff_flag = false; if (out_file_id >= 0) { - SMART_ASSERT(vals != nullptr); + SMART_ASSERT(!vals.empty()); } - int name_length = max_string_length(file1.FB_Var_Names()) + 1; - - if (out_file_id < 0 && !interFace.quiet_flag && !interFace.summary_flag && - !interFace.fb_var_names.empty()) { + if (out_file_id < 0 && !interFace.quiet_flag && !interFace.fb_var_names.empty()) { fmt::print("Face Block variables:\n"); } for (unsigned f_idx = 0; f_idx < interFace.fb_var_names.size(); ++f_idx) { const std::string &name = (interFace.fb_var_names)[f_idx]; int vidx1 = find_string(file1.FB_Var_Names(), name, interFace.nocase_var_names); - int vidx2 = 0; - if (!interFace.summary_flag) { - vidx2 = find_string(file2.FB_Var_Names(), name, interFace.nocase_var_names); - } + int vidx2 = find_string(file2.FB_Var_Names(), name, interFace.nocase_var_names); if (vidx1 < 0 || vidx2 < 0) { Error(fmt::format("Unable to find face block variable named '{}' on database.\n", name)); - exit(1); } DiffData max_diff; @@ -2372,138 +2365,76 @@ bool diff_faceblock(ExoII_Read &file1, ExoII_Read &file2, int step1, c for (size_t b = 0; b < file1.Num_Face_Blocks(); ++b) { Face_Block *fblock1 = file1.Get_Face_Block_by_Index(b); - SMART_ASSERT(fblock1 != nullptr); - if (fblock1->Size() == 0) { - continue; - } - if (!fblock1->is_valid_var(vidx1)) { + const double * vals1 = get_validated_variable(fblock1, step1, vidx1, name, &diff_flag); + if (vals1 == nullptr) { continue; } Face_Block *fblock2 = nullptr; - if (!interFace.summary_flag) { - if (interFace.by_name) { - fblock2 = file2.Get_Face_Block_by_Name(fblock1->Name()); - } - else { - fblock2 = file2.Get_Face_Block_by_Id(fblock1->Id()); - } - if (fblock2 == nullptr || !fblock2->is_valid_var(vidx2)) { - continue; - } - } - - fblock1->Load_Results(step1, vidx1); - const double *vals1 = fblock1->Get_Results(vidx1); - - if (vals1 == nullptr) { - Error(fmt::format("Could not find variable '{}' in face block {}, file 1.\n", name, - fblock1->Id())); - diff_flag = true; - continue; + if (interFace.by_name) { + fblock2 = file2.Get_Face_Block_by_Name(fblock1->Name()); } - - if (Invalid_Values(vals1, fblock1->Size())) { - Error(fmt::format("NaN found for face block variable '{}' in face block {}, file 1.\n", - name, fblock1->Id())); - diff_flag = true; + else { + fblock2 = file2.Get_Face_Block_by_Id(fblock1->Id()); } - - double v2 = 0; - const double *vals2 = nullptr; - if (!interFace.summary_flag) { - fblock2->Load_Results(t2.step1, t2.step2, t2.proportion, vidx2); - vals2 = fblock2->Get_Results(vidx2); - - if (vals2 == nullptr) { - Error(fmt::format("Could not find variable '{}' in face block {}, file 2.\n", name, - fblock2->Id())); - diff_flag = true; - continue; - } - - if (Invalid_Values(vals2, fblock2->Size())) { - Error(fmt::format("NaN found for face block variable '{}' in face block {}, file 2.\n", - name, fblock2->Id())); - diff_flag = true; - } + const double *vals2 = get_validated_variable(fblock2, t2, vidx2, name, &diff_flag); + if (vals2 == nullptr) { + continue; } size_t fcount = fblock1->Size(); - if (interFace.summary_flag || fblock2->Size() == fcount) { + if (fblock2->Size() == fcount) { for (size_t f = 0; f < fcount; ++f) { size_t ind1 = fblock1->Face_Index(f); - size_t ind2 = 0; - if (out_file_id >= 0) { - vals[ind1] = 0.; - } - if (!interFace.summary_flag) { - ind2 = fblock2->Face_Index(f); - v2 = vals2[ind2]; - } + size_t ind2 = fblock2->Face_Index(f); - if (interFace.summary_flag) { - mm_fb[f_idx].spec_min_max(vals1[ind1], step1, f, fblock1->Id()); - } - else if (out_file_id >= 0) { - vals[ind1] = FileDiff(vals1[ind1], v2, interFace.output_type); + if (out_file_id >= 0) { + vals[ind1] = FileDiff(vals1[ind1], vals2[ind2], interFace.output_type); } else if (interFace.show_all_diffs) { - double d = interFace.fb_var[f_idx].Delta(vals1[ind1], v2); + double d = interFace.fb_var[f_idx].Delta(vals1[ind1], vals2[ind2]); if (d > interFace.fb_var[f_idx].value) { - diff_flag = true; - buf = fmt::format( + diff_flag = true; + std::string buf = fmt::format( " {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (face block {}, face {})", name, - name_length, interFace.fb_var[f_idx].abrstr(), vals1[ind1], v2, d, fblock1->Id(), - fblock1->Face_Index(f)); + name_length(), interFace.fb_var[f_idx].abrstr(), vals1[ind1], vals2[ind2], d, + fblock1->Id(), fblock1->Face_Index(f) + 1); DIFF_OUT(buf); } } else { - double d = interFace.fb_var[f_idx].Delta(vals1[ind1], v2); - max_diff.set_max(d, vals1[ind1], v2, f, fblock1->Id()); + double d = interFace.fb_var[f_idx].Delta(vals1[ind1], vals2[ind2]); + max_diff.set_max(d, vals1[ind1], vals2[ind2], f, fblock1->Id()); } - norm.add_value(vals1[ind1], v2); + norm.add_value(vals1[ind1], vals2[ind2]); } if (out_file_id >= 0) { ex_put_var(out_file_id, t2.step1, EX_FACE_BLOCK, f_idx + 1, fblock1->Id(), - fblock1->Size(), vals); + fblock1->Size(), vals.data()); } } else { - buf = fmt::format(" {:<{}} diff: face block face counts differ for face block {}", - name, name_length, fblock1->Id()); + std::string buf = + fmt::format(" {:<{}} diff: face block face counts differ for face block {}", name, + name_length(), fblock1->Id()); DIFF_OUT(buf); diff_flag = true; } fblock1->Free_Results(); - if (!interFace.summary_flag) { - fblock2->Free_Results(); - } + fblock2->Free_Results(); } // End of faceblock loop. - if (!interFace.summary_flag && max_diff.diff > interFace.fb_var[f_idx].value) { + if (max_diff.diff > interFace.fb_var[f_idx].value) { diff_flag = true; - if (interFace.doL1Norm && norm.diff(1) > 0.0) { - buf = fmt::format(" {:<{}} L1 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", - name, name_length, norm.diff(1), norm.left(1), norm.right(1), - norm.relative(1)); - DIFF_OUT(buf, fmt::color::green); - } - if (interFace.doL2Norm && norm.diff(2) > 0.0) { - buf = fmt::format(" {:<{}} L2 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", - name, name_length, norm.diff(2), norm.left(2), norm.right(2), - norm.relative(2)); - DIFF_OUT(buf, fmt::color::green); - } + output_norms(norm, name); if (!interFace.quiet_flag) { - buf = + std::string buf = fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (face block {}, face {})", - name, name_length, interFace.fb_var[f_idx].abrstr(), max_diff.val1, - max_diff.val2, max_diff.diff, max_diff.blk, max_diff.id); + name, name_length(), interFace.fb_var[f_idx].abrstr(), max_diff.val1, + max_diff.val2, max_diff.diff, max_diff.blk, max_diff.id + 1); DIFF_OUT(buf); } else { @@ -2516,25 +2447,30 @@ bool diff_faceblock(ExoII_Read &file1, ExoII_Read &file2, int step1, c } template -bool diff_element_attributes(ExoII_Read &file1, ExoII_Read &file2, INT * /*elmt_map*/, - const INT *id_map, Exo_Block ** /*blocks2*/) +bool diff_element_attributes(ExoII_Read &file1, ExoII_Read & file2, + const std::vector & /*elmt_map*/, const INT *id_map, + Exo_Block ** /*blocks2*/) { + if (interFace.summary_flag) { + return false; + } + bool diff_was_output = false; bool diff_flag = false; size_t global_elmt_offset = 0; - for (size_t b = 0; b < file1.Num_Elmt_Blocks(); ++b) { - Exo_Block *eblock1 = file1.Get_Elmt_Block_by_Index(b); + for (size_t b = 0; b < file1.Num_Element_Blocks(); ++b) { + Exo_Block *eblock1 = file1.Get_Element_Block_by_Index(b); SMART_ASSERT(eblock1 != nullptr); size_t block_id = eblock1->Id(); Exo_Block *eblock2 = nullptr; if (interFace.by_name) { - eblock2 = file2.Get_Elmt_Block_by_Name(eblock1->Name()); + eblock2 = file2.Get_Element_Block_by_Name(eblock1->Name()); } else { - eblock2 = file2.Get_Elmt_Block_by_Id(block_id); + eblock2 = file2.Get_Element_Block_by_Id(block_id); } SMART_ASSERT(eblock2 != nullptr); @@ -2544,7 +2480,6 @@ bool diff_element_attributes(ExoII_Read &file1, ExoII_Read &file2, INT fmt::print("Element attributes:\n"); } - int name_length = max_string_length(eblock1->Attribute_Names()) + 1; for (int idx1 = 0; idx1 < eblock1->attr_count(); idx1++) { size_t global_elmt_index = global_elmt_offset; @@ -2576,15 +2511,15 @@ bool diff_element_attributes(ExoII_Read &file1, ExoII_Read &file2, INT const double *vals1 = eblock1->Get_Attributes(idx1); if (vals1 == nullptr) { - Error(fmt::format("Could not find element attribute '{}' in block {}, file 1.\n", name, - eblock1->Id())); + Warning(fmt::format("Could not find element attribute '{}' in block {}, file 1.\n", name, + eblock1->Id())); diff_flag = true; continue; } if (Invalid_Values(vals1, eblock1->Size())) { - Error(fmt::format("NaN found for element attribute '{}' in block {}, file 1.\n", name, - eblock1->Id())); + Warning(fmt::format("NaN found for element attribute '{}' in block {}, file 1.\n", name, + eblock1->Id())); diff_flag = true; } @@ -2593,15 +2528,15 @@ bool diff_element_attributes(ExoII_Read &file1, ExoII_Read &file2, INT const double *vals2 = eblock2->Get_Attributes(idx2); if (vals2 == nullptr) { - Error(fmt::format("Could not find element attribute '{}' in block {}, file 2.\n", name, - eblock2->Id())); + Warning(fmt::format("Could not find element attribute '{}' in block {}, file 2.\n", name, + eblock2->Id())); diff_flag = true; continue; } if (Invalid_Values(vals2, eblock2->Size())) { - Error(fmt::format("NaN found for element attribute '{}' in block {}, file 2.\n", name, - eblock2->Id())); + Warning(fmt::format("NaN found for element attribute '{}' in block {}, file 2.\n", name, + eblock2->Id())); diff_flag = true; } @@ -2612,9 +2547,9 @@ bool diff_element_attributes(ExoII_Read &file1, ExoII_Read &file2, INT double d = interFace.elmt_att[tol_idx].Delta(vals1[e], vals2[e]); if (d > interFace.elmt_att[tol_idx].value) { diff_flag = true; - buf = + std::string buf = fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (block {}, elmt {})", - name, name_length, interFace.elmt_att[tol_idx].abrstr(), vals1[e], + name, name_length(), interFace.elmt_att[tol_idx].abrstr(), vals1[e], vals2[e], d, block_id, id_map[global_elmt_index]); DIFF_OUT(buf); } @@ -2627,26 +2562,16 @@ bool diff_element_attributes(ExoII_Read &file1, ExoII_Read &file2, INT ++global_elmt_index; } - if (interFace.doL1Norm && norm.diff(1) > 0.0) { - buf = fmt::format(" {:<{}} L1 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", - name, name_length, norm.diff(1), norm.left(1), norm.right(1), - norm.relative(1)); - DIFF_OUT(buf, fmt::color::green); - } - if (interFace.doL2Norm && norm.diff(2) > 0.0) { - buf = fmt::format(" {:<{}} L2 norm of diff={:14.7e} ({:11.5e} ~ {:11.5e}) rel={:14.7e}", - name, name_length, norm.diff(2), norm.left(2), norm.right(2), - norm.relative(2)); - DIFF_OUT(buf, fmt::color::green); - } + output_norms(norm, name); - if (!interFace.summary_flag && max_diff.diff > interFace.elmt_att[tol_idx].value) { + if (max_diff.diff > interFace.elmt_att[tol_idx].value) { diff_flag = true; if (!interFace.quiet_flag) { - buf = fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (block {}, elmt {})", - name, name_length, interFace.elmt_att[tol_idx].abrstr(), max_diff.val1, - max_diff.val2, max_diff.diff, max_diff.blk, id_map[max_diff.id]); + std::string buf = + fmt::format(" {:<{}} {} diff: {:14.7e} ~ {:14.7e} ={:12.5e} (block {}, elmt {})", + name, name_length(), interFace.elmt_att[tol_idx].abrstr(), max_diff.val1, + max_diff.val2, max_diff.diff, max_diff.blk, id_map[max_diff.id]); DIFF_OUT(buf); } else { @@ -2669,7 +2594,6 @@ void output_summary(ExoII_Read &file1, MinMaxData &mm_time, std::vector &mm_eb, std::vector &mm_fb, const INT *node_id_map, const INT *elem_id_map) { - int name_length = 0; int i; int n; @@ -2698,10 +2622,9 @@ void output_summary(ExoII_Read &file1, MinMaxData &mm_time, std::vector 0) { fmt::print("GLOBAL VARIABLES relative 1.e-6 floor 0.0\n"); - name_length = max_string_length(interFace.glob_var_names); for (i = 0; i < n; ++i) { fmt::print("\t{:<{}} # min: {:15.8g} @ t{}\tmax: {:15.8g} @ t{}\n", - ((interFace.glob_var_names)[i]), name_length, mm_glob[i].min_val, + ((interFace.glob_var_names)[i]), name_length(), mm_glob[i].min_val, mm_glob[i].min_step, mm_glob[i].max_val, mm_glob[i].max_step); } } @@ -2712,10 +2635,9 @@ void output_summary(ExoII_Read &file1, MinMaxData &mm_time, std::vector 0 && file1.Num_Nodes() > 0) { fmt::print("\nNODAL VARIABLES relative 1.e-6 floor 0.0\n"); - name_length = max_string_length(interFace.node_var_names); for (i = 0; i < n; ++i) { fmt::print("\t{:<{}} # min: {:15.8g} @ t{},n{}\tmax: {:15.8g} @ t{},n{}\n", - ((interFace.node_var_names)[i]), name_length, mm_node[i].min_val, + ((interFace.node_var_names)[i]), name_length(), mm_node[i].min_val, mm_node[i].min_step, node_id_map[mm_node[i].min_id], mm_node[i].max_val, mm_node[i].max_step, node_id_map[mm_node[i].max_id]); } @@ -2725,13 +2647,12 @@ void output_summary(ExoII_Read &file1, MinMaxData &mm_time, std::vector 0 && file1.Num_Elmts() > 0) { + if (n > 0 && file1.Num_Elements() > 0) { fmt::print("\nELEMENT VARIABLES relative 1.e-6 floor 0.0\n"); - name_length = max_string_length(interFace.elmt_var_names); for (i = 0; i < n; ++i) { fmt::print("\t{:<{}} # min: {:15.8g} @ t{},b{},e{}\tmax: {:15.8g} @ t{},b{}" ",e{}\n", - ((interFace.elmt_var_names)[i]), name_length, mm_elmt[i].min_val, + ((interFace.elmt_var_names)[i]), name_length(), mm_elmt[i].min_val, mm_elmt[i].min_step, mm_elmt[i].min_blk, elem_id_map[mm_elmt[i].min_id], mm_elmt[i].max_val, mm_elmt[i].max_step, mm_elmt[i].max_blk, elem_id_map[mm_elmt[i].max_id]); @@ -2744,13 +2665,12 @@ void output_summary(ExoII_Read &file1, MinMaxData &mm_time, std::vector 0) { fmt::print("\nNODESET VARIABLES relative 1.e-6 floor 0.0\n"); - name_length = max_string_length(interFace.ns_var_names); for (i = 0; i < n; ++i) { Node_Set *nsmin = file1.Get_Node_Set_by_Id(mm_ns[i].min_blk); Node_Set *nsmax = file1.Get_Node_Set_by_Id(mm_ns[i].max_blk); fmt::print("\t{:<{}} # min: {:15.8g} @ t{},s{},n{}\tmax: {:15.8g} @ t{},s{}" ",n{}\n", - ((interFace.ns_var_names)[i]), name_length, mm_ns[i].min_val, mm_ns[i].min_step, + ((interFace.ns_var_names)[i]), name_length(), mm_ns[i].min_val, mm_ns[i].min_step, mm_ns[i].min_blk, node_id_map[nsmin->Node_Id(mm_ns[i].min_id) - 1], mm_ns[i].max_val, mm_ns[i].max_step, mm_ns[i].max_blk, node_id_map[nsmax->Node_Id(mm_ns[i].max_id) - 1]); @@ -2763,7 +2683,6 @@ void output_summary(ExoII_Read &file1, MinMaxData &mm_time, std::vector 0) { fmt::print("\nSIDESET VARIABLES relative 1.e-6 floor 0.0\n"); - name_length = max_string_length(interFace.ss_var_names); for (i = 0; i < n; ++i) { Side_Set * ssmin = file1.Get_Side_Set_by_Id(mm_ss[i].min_blk); Side_Set * ssmax = file1.Get_Side_Set_by_Id(mm_ss[i].max_blk); @@ -2771,7 +2690,7 @@ void output_summary(ExoII_Read &file1, MinMaxData &mm_time, std::vector max_side = ssmax->Side_Id(mm_ss[i].max_id); fmt::print("\t{:<{}} # min: {:15.8g} @ t{},s{},f{}.{}\tmax: {:15.8g} @ t{},s{}" ",f{}.{}\n", - ((interFace.ss_var_names)[i]), name_length, mm_ss[i].min_val, mm_ss[i].min_step, + ((interFace.ss_var_names)[i]), name_length(), mm_ss[i].min_val, mm_ss[i].min_step, mm_ss[i].min_blk, elem_id_map[min_side.first - 1], min_side.second, mm_ss[i].max_val, mm_ss[i].max_step, mm_ss[i].max_blk, elem_id_map[max_side.first - 1], max_side.second); @@ -2784,13 +2703,12 @@ void output_summary(ExoII_Read &file1, MinMaxData &mm_time, std::vector 0) { fmt::print("\nEDGE BLOCK VARIABLES relative 1.e-6 floor 0.0\n"); - name_length = max_string_length(interFace.eb_var_names); for (i = 0; i < n; ++i) { fmt::print("\t{:<{}} # min: {:15.8g} @ t{},b{},e{}\tmax: {:15.8g} @ t{},b{}" ",e{}\n", - ((interFace.eb_var_names)[i]), name_length, mm_eb[i].min_val, mm_eb[i].min_step, - mm_eb[i].min_blk, elem_id_map[mm_eb[i].min_id], mm_eb[i].max_val, - mm_eb[i].max_step, mm_eb[i].max_blk, elem_id_map[mm_eb[i].max_id]); + ((interFace.eb_var_names)[i]), name_length(), mm_eb[i].min_val, mm_eb[i].min_step, + mm_eb[i].min_blk, mm_eb[i].min_id + 1, mm_eb[i].max_val, mm_eb[i].max_step, + mm_eb[i].max_blk, mm_eb[i].max_id + 1); } } else { @@ -2800,13 +2718,12 @@ void output_summary(ExoII_Read &file1, MinMaxData &mm_time, std::vector 0) { fmt::print("\nFACE BLOCK VARIABLES relative 1.e-6 floor 0.0\n"); - name_length = max_string_length(interFace.fb_var_names); for (i = 0; i < n; ++i) { - fmt::print("\t{:<{}} # min: {:15.8g} @ t{},b{},e{}\tmax: {:15.8g} @ t{},b{}" - ",e{}\n", - ((interFace.fb_var_names)[i]), name_length, mm_fb[i].min_val, mm_fb[i].min_step, - mm_fb[i].min_blk, elem_id_map[mm_fb[i].min_id], mm_fb[i].max_val, - mm_fb[i].max_step, mm_fb[i].max_blk, elem_id_map[mm_fb[i].max_id]); + fmt::print("\t{:<{}} # min: {:15.8g} @ t{},b{},f{}\tmax: {:15.8g} @ t{},b{}" + ",f{}\n", + ((interFace.fb_var_names)[i]), name_length(), mm_fb[i].min_val, mm_fb[i].min_step, + mm_fb[i].min_blk, mm_fb[i].min_id + 1, mm_fb[i].max_val, mm_fb[i].max_step, + mm_fb[i].max_blk, mm_fb[i].max_id + 1); } } else { diff --git a/packages/seacas/applications/exodiff/face_block.C b/packages/seacas/applications/exodiff/face_block.C index 91a813154075..fb035e422c60 100644 --- a/packages/seacas/applications/exodiff/face_block.C +++ b/packages/seacas/applications/exodiff/face_block.C @@ -26,12 +26,7 @@ Face_Block::Face_Block(int file_id, size_t id, size_t ne) : Exo_Entity(file SMART_ASSERT(id > 0); } -template Face_Block::~Face_Block() -{ - SMART_ASSERT(Check_State()); - - delete[] faceIndex; -} +template Face_Block::~Face_Block() { SMART_ASSERT(Check_State()); } template EXOTYPE Face_Block::exodus_type() const { return EX_FACE_BLOCK; } @@ -46,7 +41,6 @@ template void Face_Block::entity_load_params() if (err < 0) { Error("Face_Block::entity_load_params(): Failed to get face" " block parameters! Aborting...\n"); - exit(1); } numEntity = block.num_entry; @@ -62,35 +56,19 @@ template void Face_Block::entity_load_params() "\tnum attributes = {}\n" " ... Aborting...\n", numEntity, num_faces_per_elmt, num_attr)); - exit(1); - } -} - -template void Face_Block::load_faces(const INT *elmt_map) const -{ - if ((faceIndex == nullptr) && numEntity > 0) { - faceIndex = new INT[numEntity]; - SMART_ASSERT(faceIndex != nullptr); - - for (size_t i = 0; i < numEntity; i++) { - faceIndex[i] = i; - } - SMART_ASSERT(Check_State()); } } template size_t Face_Block::Face_Index(size_t position) const { - load_faces(); SMART_ASSERT(position < numEntity); - return faceIndex[position]; + return position; } template int Face_Block::Check_State() const { SMART_ASSERT(id_ >= EX_INVALID_ID); SMART_ASSERT(!(id_ == EX_INVALID_ID && numEntity > 0)); - SMART_ASSERT(!(id_ == EX_INVALID_ID && faceIndex)); return 1; } diff --git a/packages/seacas/applications/exodiff/face_block.h b/packages/seacas/applications/exodiff/face_block.h index a5f1a3d9fece..1148e1961896 100644 --- a/packages/seacas/applications/exodiff/face_block.h +++ b/packages/seacas/applications/exodiff/face_block.h @@ -1,4 +1,4 @@ -// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// Copyright(C) 1999-2021 National Technology & Engineering Solutions // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with // NTESS, the U.S. Government retains certain rights in this software. // @@ -28,15 +28,12 @@ template class Face_Block : public Exo_Entity Face_Block(const Face_Block &) = delete; // Not written. const Face_Block &operator=(const Face_Block &) = delete; // Not written. - void load_faces(const INT *elmt_map = nullptr) const; void entity_load_params() override; EXOTYPE exodus_type() const override; const char *label() const override { return "Faceblock"; } const char *short_label() const override { return "faceblock"; } - mutable INT *faceIndex{nullptr}; - std::string elmt_type; int num_faces_per_elmt{-1}; diff --git a/packages/seacas/applications/exodiff/map.C b/packages/seacas/applications/exodiff/map.C index ad0ecf45154c..c62912d4626f 100644 --- a/packages/seacas/applications/exodiff/map.C +++ b/packages/seacas/applications/exodiff/map.C @@ -23,60 +23,62 @@ namespace { double find_range(const double *x, size_t num_nodes); template - INT Find(double x0, double y0, double z0, const double *x, const double *y, const double *z, - INT *id, size_t N, int dim, bool ignore_dups); + INT Find(double x0, double y0, double z0, const std::vector &x, + const std::vector &y, const std::vector &z, const std::vector &id, + int dim, bool ignore_dups); template - void Compute_Node_Map(INT *&node_map, ExoII_Read &file1, ExoII_Read &file2); + void Compute_Node_Map(std::vector &node_map, ExoII_Read &file1, ExoII_Read &file2); } // namespace template -void Compute_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, ExoII_Read &file2) +void Compute_Maps(std::vector &node_map, std::vector &elmt_map, ExoII_Read &file1, + ExoII_Read &file2) { SMART_ASSERT(file1.Open()); SMART_ASSERT(file2.Open()); size_t num_nodes = file1.Num_Nodes(); - size_t num_elmts = file1.Num_Elmts(); + size_t num_elmts = file1.Num_Elements(); int dim = file1.Dimension(); // ******************** elements ******************** // // Load global ids (0-offset) into id array. - auto id = new INT[num_elmts]; - std::iota(id, id + num_elmts, 0); + std::vector id(num_elmts); + std::iota(id.begin(), id.end(), 0); // Get map storage. - node_map = new INT[num_nodes]; - SMART_ASSERT(node_map != nullptr); - std::fill(node_map, node_map + num_nodes, -1); + node_map.resize(num_nodes); + std::fill(node_map.begin(), node_map.end(), -1); - elmt_map = new INT[num_elmts]; - SMART_ASSERT(elmt_map != nullptr); - std::fill(elmt_map, elmt_map + num_elmts, -1); + elmt_map.resize(num_elmts); + std::fill(elmt_map.begin(), elmt_map.end(), -1); // Create storage for midpoints. - auto * x2 = new double[num_elmts]; - double *y2 = dim > 1 ? new double[num_elmts] : nullptr; - double *z2 = dim > 2 ? new double[num_elmts] : nullptr; + std::vector x2; + x2.reserve(num_elmts); + std::vector y2; + y2.reserve(dim > 1 ? num_elmts : 0); + std::vector z2; + z2.reserve(dim > 2 ? num_elmts : 0); // Load coordinates for file 2 and get pointers to them. file2.Load_Nodal_Coordinates(); - const double *x2_f = (double *)file2.X_Coords(); - const double *y2_f = (double *)file2.Y_Coords(); - const double *z2_f = (double *)file2.Z_Coords(); + const auto *x2_f = file2.X_Coords(); + const auto *y2_f = file2.Y_Coords(); + const auto *z2_f = file2.Z_Coords(); // Load connectivities for all blocks in second file. - file2.Load_Elmt_Block_Descriptions(); + file2.Load_Element_Block_Descriptions(); { // Compute midpoints of each element and place into x,y,z arrays. - size_t num_blocks = file2.Num_Elmt_Blocks(); - size_t e = 0; + size_t num_blocks = file2.Num_Element_Blocks(); for (size_t b = 0; b < num_blocks; ++b) { - const Exo_Block *block = file2.Get_Elmt_Block_by_Index(b); + const Exo_Block *block = file2.Get_Element_Block_by_Index(b); size_t num_elmts_in_block = block->Size(); - size_t num_nodes_per_elmt = block->Num_Nodes_per_Elmt(); + size_t num_nodes_per_elmt = block->Num_Nodes_per_Element(); for (size_t i = 0; i < num_elmts_in_block; ++i) { const INT *conn = block->Connectivity(i); // Connectivity for element i. double sum_x = 0.0; @@ -91,21 +93,19 @@ void Compute_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, ExoII_ sum_z += z2_f[conn[j] - 1]; } } - x2[e] = sum_x / static_cast(num_nodes_per_elmt); + x2.push_back(sum_x / static_cast(num_nodes_per_elmt)); if (dim > 1) { - y2[e] = sum_y / static_cast(num_nodes_per_elmt); + y2.push_back(sum_y / static_cast(num_nodes_per_elmt)); } if (dim > 2) { - z2[e] = sum_z / static_cast(num_nodes_per_elmt); + z2.push_back(sum_z / static_cast(num_nodes_per_elmt)); } - - ++e; } } } // Sort by x value. - index_qsort(x2, id, num_elmts); + index_qsort(x2.data(), id.data(), num_elmts); #if 0 fmt::print("****************** elmts ******************** \n"); @@ -116,9 +116,9 @@ void Compute_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, ExoII_ #endif // Load and get nodal coordinates for first file. file1.Load_Nodal_Coordinates(); - const double *x1_f = (double *)file1.X_Coords(); - const double *y1_f = (double *)file1.Y_Coords(); - const double *z1_f = (double *)file1.Z_Coords(); + const auto *x1_f = file1.X_Coords(); + const auto *y1_f = file1.Y_Coords(); + const auto *z1_f = file1.Z_Coords(); // Cannot ignore the comparisons, so make sure the coord_tol_type // is not -1 which is "ignore" @@ -128,15 +128,15 @@ void Compute_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, ExoII_ } // Match elmts in first file to their corresponding elmts in second. - size_t num_blocks = file1.Num_Elmt_Blocks(); + size_t num_blocks = file1.Num_Element_Blocks(); size_t e1 = 0; size_t e2 = 0; for (size_t b = 0; b < num_blocks; ++b) { - const Exo_Block *block1 = file1.Get_Elmt_Block_by_Index(b); - file1.Load_Elmt_Block_Description(b); + const Exo_Block *block1 = file1.Get_Element_Block_by_Index(b); + file1.Load_Element_Block_Description(b); size_t num_elmts_in_block = block1->Size(); - size_t num_nodes_per_elmt = block1->Num_Nodes_per_Elmt(); + size_t num_nodes_per_elmt = block1->Num_Nodes_per_Element(); for (size_t i = 0; i < num_elmts_in_block; ++i) { // Connectivity for element i. const INT *conn1 = block1->Connectivity(i); @@ -165,14 +165,12 @@ void Compute_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, ExoII_ } // Locate midpoint in sorted array. - INT sort_idx = - Find(mid_x, mid_y, mid_z, x2, y2, z2, id, num_elmts, dim, interFace.ignore_dups); + INT sort_idx = Find(mid_x, mid_y, mid_z, x2, y2, z2, id, dim, interFace.ignore_dups); if (sort_idx < 0) { Error(fmt::format("Files are different (couldn't match element {} from block {} from first " "file to second)\n", i + 1, file1.Block_Id(b))); - exit(1); } e2 = id[sort_idx]; @@ -181,25 +179,22 @@ void Compute_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, ExoII_ { // Determine the block and elmt index of matched element. - int b2; - size_t l2; - file2.Global_to_Block_Local(e2 + 1, b2, l2); + auto bl_idx = file2.Global_to_Block_Local(e2 + 1); - const Exo_Block *block2 = file2.Get_Elmt_Block_by_Index(b2); + const Exo_Block *block2 = file2.Get_Element_Block_by_Index(bl_idx.first); SMART_ASSERT(block2 != nullptr); // Check that the element types are the same. - if (num_nodes_per_elmt != block2->Num_Nodes_per_Elmt()) { + if (num_nodes_per_elmt != block2->Num_Nodes_per_Element()) { Error(fmt::format("Files are different.\n" " In File 1: Element {:L} in Block {} has {} and\n" " In File 2: Element {:L} in Block {} has {}\n", - i + 1, file1.Block_Id(b), num_nodes_per_elmt, l2 + 1, - file2.Block_Id(b2), block2->Num_Nodes_per_Elmt())); - exit(1); + i + 1, file1.Block_Id(b), num_nodes_per_elmt, bl_idx.second + 1, + file2.Block_Id(bl_idx.first), block2->Num_Nodes_per_Element())); } // Get connectivity for file2 element. - const INT *conn2 = block2->Connectivity(l2); + const INT *conn2 = block2->Connectivity(bl_idx.second); // Match each node in the first elmt with a node in the second // and assign node_map. @@ -251,7 +246,6 @@ void Compute_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, ExoII_ "\tFile 2, Node {:L} at ({}, {}, {}) and\n" "\tFile 2, Node {:L} at ({}, {}, {})\n\n", node1, x1a, y1a, z1a, n1, x2a, y2a, z2a, n2, x2b, y2b, z2b)); - exit(1); } found = 1; break; @@ -263,11 +257,10 @@ void Compute_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, ExoII_ } if (!found) { std::ostringstream out; - fmt::print( - out, - "\nexodiff: ERROR: Cannot find a match for node at position {} in first element.\n" - "\tFile 1: Element {:L} in Block {} nodes:\n", - ln1 + 1, i + 1, file1.Block_Id(b)); + fmt::print(out, + "\nCannot find a match for node at position {} in first element.\n" + "\tFile 1: Element {:L} in Block {} nodes:\n", + ln1 + 1, i + 1, file1.Block_Id(b)); for (size_t l1 = 0; l1 < num_nodes_per_elmt; ++l1) { double x_val = x1_f[conn1[l1] - 1]; double y_val = dim > 1 ? y1_f[conn1[l1] - 1] : 0.0; @@ -275,7 +268,7 @@ void Compute_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, ExoII_ fmt::print(out, "\t({})\t{:L}\t{:.9e}\t{:.9e}\t{:.9e}\n", l1 + 1, conn1[l1], x_val, y_val, z_val); } - fmt::print(out, "\tFile 2: Element {:L} in Block {} nodes:\n", l2 + 1, + fmt::print(out, "\tFile 2: Element {:L} in Block {} nodes:\n", bl_idx.second + 1, file1.Block_Id(b)); for (size_t l3 = 0; l3 < num_nodes_per_elmt; ++l3) { double x_val = x2_f[conn2[l3] - 1]; @@ -287,8 +280,7 @@ void Compute_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, ExoII_ fmt::print(out, "Coordinates compared using tolerance: {} ({}), floor: {}\n", interFace.coord_tol.value, interFace.coord_tol.typestr(), interFace.coord_tol.floor); - ERR_OUT(out); - exit(1); + Error(out); } } // End of local node loop on file1's element. } // End of local node search block. @@ -297,7 +289,7 @@ void Compute_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, ExoII_ } // End of loop on elements in file1 element block. - file1.Free_Elmt_Block(b); + file1.Free_Element_Block(b); } // End of loop on file1 blocks. @@ -313,77 +305,70 @@ void Compute_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, ExoII_ file1.Free_Nodal_Coordinates(); file2.Free_Nodal_Coordinates(); - file2.Free_Elmt_Blocks(); - - delete[] x2; - delete[] y2; - delete[] z2; - delete[] id; + file2.Free_Element_Blocks(); interFace.coord_tol.type = save_tolerance_type; } template -void Compute_Partial_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, - ExoII_Read &file2) +void Compute_Partial_Maps(std::vector &node_map, std::vector &elmt_map, + ExoII_Read &file1, ExoII_Read &file2) { SMART_ASSERT(file1.Open()); SMART_ASSERT(file2.Open()); size_t num_nodes1 = file1.Num_Nodes(); - size_t num_elmts1 = file1.Num_Elmts(); + size_t num_elmts1 = file1.Num_Elements(); size_t num_nodes2 = file2.Num_Nodes(); - size_t num_elmts2 = file2.Num_Elmts(); + size_t num_elmts2 = file2.Num_Elements(); int dim = file1.Dimension(); SMART_ASSERT(dim == file2.Dimension()); // ******************** elements ******************** // // Load global ids (0-offset) into id array. - auto id2 = new INT[num_elmts2]; - std::iota(id2, id2 + num_elmts2, 0); + std::vector id2(num_elmts2); + std::iota(id2.begin(), id2.end(), 0); // Get map storage. - node_map = new INT[num_nodes1]; - SMART_ASSERT(node_map != nullptr); - std::fill(node_map, node_map + num_nodes1, -1); + node_map.resize(num_nodes1); + std::fill(node_map.begin(), node_map.end(), -1); - elmt_map = new INT[num_elmts1]; - SMART_ASSERT(elmt_map != nullptr); - std::fill(elmt_map, elmt_map + num_elmts1, -1); + elmt_map.resize(num_elmts1); + std::fill(elmt_map.begin(), elmt_map.end(), -1); // Create storage for midpoints. - auto * x2 = new double[num_elmts2]; - double *y2 = nullptr; - double *z2 = nullptr; + std::vector x2; + std::vector y2; + std::vector z2; + x2.reserve(num_elmts2); if (dim > 1) { - y2 = new double[num_elmts2]; + y2.reserve(num_elmts2); } if (dim > 2) { - z2 = new double[num_elmts2]; + z2.reserve(num_elmts2); } // Load coordinates for file 2 and get pointers to them. file2.Load_Nodal_Coordinates(); - const double *x2_f = (double *)file2.X_Coords(); - const double *y2_f = (double *)file2.Y_Coords(); - const double *z2_f = (double *)file2.Z_Coords(); + const auto *x2_f = file2.X_Coords(); + const auto *y2_f = file2.Y_Coords(); + const auto *z2_f = file2.Z_Coords(); // Load connectivities for all blocks in second file. - file2.Load_Elmt_Block_Descriptions(); + file2.Load_Element_Block_Descriptions(); { // Compute midpoints of each element and place into x,y,z arrays. - size_t num_blocks2 = file2.Num_Elmt_Blocks(); - size_t e = 0; + size_t num_blocks2 = file2.Num_Element_Blocks(); double sum_x; double sum_y; double sum_z; for (size_t b = 0; b < num_blocks2; ++b) { - const Exo_Block *block = file2.Get_Elmt_Block_by_Index(b); + const Exo_Block *block = file2.Get_Element_Block_by_Index(b); size_t num_elmts_in_block = block->Size(); - size_t num_nodes_per_elmt = block->Num_Nodes_per_Elmt(); + size_t num_nodes_per_elmt = block->Num_Nodes_per_Element(); for (size_t i = 0; i < num_elmts_in_block; ++i) { const INT *conn = block->Connectivity(i); // Connectivity for element i. sum_x = 0.0; @@ -398,21 +383,19 @@ void Compute_Partial_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1 sum_z += z2_f[conn[j] - 1]; } } - x2[e] = sum_x / static_cast(num_nodes_per_elmt); + x2.push_back(sum_x / static_cast(num_nodes_per_elmt)); if (dim > 1) { - y2[e] = sum_y / static_cast(num_nodes_per_elmt); + y2.push_back(sum_y / static_cast(num_nodes_per_elmt)); } if (dim > 2) { - z2[e] = sum_z / static_cast(num_nodes_per_elmt); + z2.push_back(sum_z / static_cast(num_nodes_per_elmt)); } - - ++e; } } } // Sort by x value. - index_qsort(x2, id2, num_elmts2); + index_qsort(x2.data(), id2.data(), num_elmts2); #if 0 fmt::print("****************** elmts ******************** \n"); @@ -423,9 +406,9 @@ void Compute_Partial_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1 #endif // Load and get nodal coordinates for first file. file1.Load_Nodal_Coordinates(); - const double *x1_f = (double *)file1.X_Coords(); - const double *y1_f = (double *)file1.Y_Coords(); - const double *z1_f = (double *)file1.Z_Coords(); + const auto *x1_f = file1.X_Coords(); + const auto *y1_f = file1.Y_Coords(); + const auto *z1_f = file1.Z_Coords(); // Cannot ignore the comparisons, so make sure the coord_tol_type // is not -1 which is "ignore" @@ -435,17 +418,17 @@ void Compute_Partial_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1 } // Match elmts in first file to their corresponding elmts in second. - size_t num_blocks1 = file1.Num_Elmt_Blocks(); + size_t num_blocks1 = file1.Num_Element_Blocks(); size_t e1 = 0; size_t e2 = 0; bool first = true; size_t unmatched = 0; for (size_t b = 0; b < num_blocks1; ++b) { - const Exo_Block *block1 = file1.Get_Elmt_Block_by_Index(b); - file1.Load_Elmt_Block_Description(b); + const Exo_Block *block1 = file1.Get_Element_Block_by_Index(b); + file1.Load_Element_Block_Description(b); size_t num_elmts_in_block = block1->Size(); - size_t num_nodes_per_elmt = block1->Num_Nodes_per_Elmt(); + size_t num_nodes_per_elmt = block1->Num_Nodes_per_Element(); for (size_t i = 0; i < num_elmts_in_block; ++i) { // Connectivity for element i. const INT *conn1 = block1->Connectivity(i); @@ -474,8 +457,7 @@ void Compute_Partial_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1 } // Locate midpoint in sorted array. - INT sort_idx = - Find(mid_x, mid_y, mid_z, x2, y2, z2, id2, num_elmts2, dim, interFace.ignore_dups); + INT sort_idx = Find(mid_x, mid_y, mid_z, x2, y2, z2, id2, dim, interFace.ignore_dups); if (sort_idx < 0) { unmatched++; if (first && interFace.show_unmatched) { @@ -493,25 +475,22 @@ void Compute_Partial_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1 // Assign element map for this element. // Determine the block and elmt index of matched element. - int b2; - size_t l2; - file2.Global_to_Block_Local(e2 + 1, b2, l2); + auto bl_idx = file2.Global_to_Block_Local(e2 + 1); - const Exo_Block *block2 = file2.Get_Elmt_Block_by_Index(b2); + const Exo_Block *block2 = file2.Get_Element_Block_by_Index(bl_idx.first); SMART_ASSERT(block2 != nullptr); // Check that the element types are the same. - if (num_nodes_per_elmt != block2->Num_Nodes_per_Elmt()) { + if (num_nodes_per_elmt != block2->Num_Nodes_per_Element()) { Error(fmt::format("Files are different.\n" " In File 1: Element {:L} in Block {} has {} and\n" " In File 2: Element {:L} in Block {} has {}\n", - i + 1, file1.Block_Id(b), num_nodes_per_elmt, l2 + 1, - file2.Block_Id(b2), block2->Num_Nodes_per_Elmt())); - exit(1); + i + 1, file1.Block_Id(b), num_nodes_per_elmt, bl_idx.second + 1, + file2.Block_Id(bl_idx.first), block2->Num_Nodes_per_Element())); } // Get connectivity for file2 element. - const INT *conn2 = block2->Connectivity(l2); + const INT *conn2 = block2->Connectivity(bl_idx.second); // Match each node in the first elmt with a node in the second // and assign node_map. @@ -537,11 +516,10 @@ void Compute_Partial_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1 } if (!found) { std::ostringstream out; - fmt::print( - out, - "\nexodiff: ERROR: Cannot find a match for node at position {} in first element.\n" - "\tFile 1: Element {:L} in Block {} nodes:\n", - ln1 + 1, i + 1, file1.Block_Id(b)); + fmt::print(out, + "\nCannot find a match for node at position {} in first element.\n" + "\tFile 1: Element {:L} in Block {} nodes:\n", + ln1 + 1, i + 1, file1.Block_Id(b)); for (size_t l1 = 0; l1 < num_nodes_per_elmt; ++l1) { double x_val = x1_f[conn1[l1] - 1]; double y_val = dim > 1 ? y1_f[conn1[l1] - 1] : 0.0; @@ -549,7 +527,7 @@ void Compute_Partial_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1 fmt::print(out, "\t({})\t{:L}\t{:.9e}\t{:.9e}\t{:.9e}\n", l1 + 1, conn1[l1], x_val, y_val, z_val); } - fmt::print(out, "\tFile 2: Element {:L} in Block {} nodes:\n", l2 + 1, + fmt::print(out, "\tFile 2: Element {:L} in Block {} nodes:\n", bl_idx.second + 1, file1.Block_Id(b)); for (size_t l3 = 0; l3 < num_nodes_per_elmt; ++l3) { double x_val = x2_f[conn2[l3] - 1]; @@ -561,8 +539,7 @@ void Compute_Partial_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1 fmt::print(out, "Coordinates compared using tolerance: {} ({}), floor: {}\n", interFace.coord_tol.value, interFace.coord_tol.typestr(), interFace.coord_tol.floor); - ERR_OUT(out); - exit(1); + Error(out); } } // End of local node loop on file1's element. } // End of local node search block. @@ -570,7 +547,7 @@ void Compute_Partial_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1 ++e1; } // End of loop on elements in file1 element block. - file1.Free_Elmt_Block(b); + file1.Free_Element_Block(b); } // End of loop on file1 blocks. if (!first) { @@ -595,12 +572,7 @@ void Compute_Partial_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1 file1.Free_Nodal_Coordinates(); file2.Free_Nodal_Coordinates(); - file2.Free_Elmt_Blocks(); - - delete[] x2; - delete[] y2; - delete[] z2; - delete[] id2; + file2.Free_Element_Blocks(); interFace.coord_tol.type = save_tolerance_type; } @@ -617,8 +589,8 @@ namespace { } template - bool internal_compute_maps(INT *map, const INT *file1_id_map, const INT *file2_id_map, - size_t count, const char *type) + bool internal_compute_maps(std::vector &map, const INT *file1_id_map, + const INT *file2_id_map, size_t count, const char *type) { std::vector id1; id1.reserve(count); @@ -647,7 +619,6 @@ namespace { else { Error(fmt::format("Unable to match {0} {1} in first file with {0} in second file.\n", type, file1_id_map[id1[i]])); - exit(1); } } @@ -664,8 +635,8 @@ namespace { } // namespace template -void Compute_FileId_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, - ExoII_Read &file2) +void Compute_FileId_Maps(std::vector &node_map, std::vector &elmt_map, + ExoII_Read &file1, ExoII_Read &file2) { // Compute map of nodes and elements in file1 to nodes and elements in file2 // Use the internal exodus node and element number maps in file1 and file2 to @@ -679,43 +650,40 @@ void Compute_FileId_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, size_t num_nodes = file1.Num_Nodes(); SMART_ASSERT(num_nodes == file2.Num_Nodes()); - node_map = new INT[num_nodes]; - SMART_ASSERT(node_map != nullptr); + node_map.resize(num_nodes); file1.Load_Node_Map(); file2.Load_Node_Map(); const INT *node_id_map1 = file1.Get_Node_Map(); const INT *node_id_map2 = file2.Get_Node_Map(); if (!internal_compute_maps(node_map, node_id_map1, node_id_map2, num_nodes, "node")) { - delete[] node_map; - node_map = nullptr; + node_map.clear(); } } { - size_t num_elmts = file1.Num_Elmts(); - SMART_ASSERT(num_elmts == file2.Num_Elmts()); - elmt_map = new INT[num_elmts]; - SMART_ASSERT(elmt_map != nullptr); - file1.Load_Elmt_Map(); - file2.Load_Elmt_Map(); - const INT *elem_id_map1 = file1.Get_Elmt_Map(); - const INT *elem_id_map2 = file2.Get_Elmt_Map(); + size_t num_elmts = file1.Num_Elements(); + SMART_ASSERT(num_elmts == file2.Num_Elements()); + elmt_map.resize(num_elmts); + file1.Load_Element_Map(); + file2.Load_Element_Map(); + const INT *elem_id_map1 = file1.Get_Element_Map(); + const INT *elem_id_map2 = file2.Get_Element_Map(); if (!internal_compute_maps(elmt_map, elem_id_map1, elem_id_map2, num_elmts, "element")) { - delete[] elmt_map; - elmt_map = nullptr; + elmt_map.clear(); } } } template -void Dump_Maps(const INT *node_map, const INT *elmt_map, ExoII_Read &file1) +void Dump_Maps(const std::vector &node_map, const std::vector &elmt_map, + ExoII_Read &file1) { size_t ijk; fmt::print("\n=== node number map (file1 -> file2) local ids\n"); bool one_to_one = true; - if (node_map != nullptr) { + if (!node_map.empty()) { for (ijk = 0; ijk < file1.Num_Nodes(); ++ijk) { if ((INT)ijk != node_map[ijk]) { one_to_one = false; @@ -734,8 +702,8 @@ void Dump_Maps(const INT *node_map, const INT *elmt_map, ExoII_Read &file1) fmt::print("\n=== element number map (file1 -> file2) local ids\n"); one_to_one = true; - if (elmt_map != nullptr) { - for (ijk = 0; ijk < file1.Num_Elmts(); ++ijk) { + if (!elmt_map.empty()) { + for (ijk = 0; ijk < file1.Num_Elements(); ++ijk) { if ((INT)ijk != elmt_map[ijk]) { one_to_one = false; break; @@ -743,7 +711,7 @@ void Dump_Maps(const INT *node_map, const INT *elmt_map, ExoII_Read &file1) } } if (!one_to_one) { - for (ijk = 0; ijk < file1.Num_Elmts(); ++ijk) { + for (ijk = 0; ijk < file1.Num_Elements(); ++ijk) { fmt::print("{} -> {}\n", (ijk + 1), (elmt_map[ijk] + 1)); } } @@ -755,7 +723,7 @@ void Dump_Maps(const INT *node_map, const INT *elmt_map, ExoII_Read &file1) namespace { template - void Compute_Node_Map(INT *&node_map, ExoII_Read &file1, ExoII_Read &file2) + void Compute_Node_Map(std::vector &node_map, ExoII_Read &file1, ExoII_Read &file2) { // This function is called if and only if there are nodes that were // not matched in the Compute_Map function. This is typically the @@ -807,19 +775,18 @@ namespace { Error(fmt::format("Files are different (free node count in file1 is " "{:L} but file2 free node count is {:L})\n", count_1, count_2)); - exit(1); } // Now, need to match all nodes in 'mapped_1' with nodes in // 'mapped_2' // Get pointer to coordinates... - const double *x1_f = (double *)file1.X_Coords(); - const double *y1_f = (double *)file1.Y_Coords(); - const double *z1_f = (double *)file1.Z_Coords(); + const auto *x1_f = file1.X_Coords(); + const auto *y1_f = file1.Y_Coords(); + const auto *z1_f = file1.Z_Coords(); - const double *x2_f = (double *)file2.X_Coords(); - const double *y2_f = (double *)file2.Y_Coords(); - const double *z2_f = (double *)file2.Z_Coords(); + const auto *x2_f = file2.X_Coords(); + const auto *y2_f = file2.Y_Coords(); + const auto *z2_f = file2.Z_Coords(); // For now, we will try a brute force matching with the hopes that // the number of unmatched nodes is 'small'. If this proves to be a @@ -853,17 +820,16 @@ namespace { Error(fmt::format("Unable to match all free nodes in the model. There are {:L}" " unmatched nodes remaining.\n", count_1 - matched)); - exit(1); } interFace.coord_tol.type = save_tolerance_type; } template - INT Find(double x0, double y0, double z0, const double *x, const double *y, const double *z, - INT *id, size_t N, int dim, bool ignore_dups) + INT Find(double x0, double y0, double z0, const std::vector &x, + const std::vector &y, const std::vector &z, const std::vector &id, + int dim, bool ignore_dups) { - SMART_ASSERT(x != nullptr); - if (N == 0) { + if (x.empty()) { return -1; } @@ -876,6 +842,7 @@ namespace { // Find the index such that x0 > x[0,1,...,low-1] and x0 >= x[low] // where x[N] is infinity. + auto N = x.size(); size_t low = 0; size_t high = N; while (low < high) { @@ -926,7 +893,6 @@ namespace { "\tLocal element {:L} at ({}, {}, {})\n" "\tNo unique element mapping possible.\n", id[i] + 1, x1, y1, z1, id[index] + 1, x2, y2, z2)); - return -1; } index = i; @@ -974,12 +940,12 @@ template double Find_Min_Coord_Sep(ExoII_Read &file) } file.Load_Nodal_Coordinates(); - const double *x = (double *)file.X_Coords(); - const double *y = (double *)file.Y_Coords(); - const double *z = (double *)file.Z_Coords(); + const auto *x = file.X_Coords(); + const auto *y = file.Y_Coords(); + const auto *z = file.Z_Coords(); - auto indx = new INT[num_nodes]; - std::iota(indx, indx + num_nodes, 0); + std::vector indx(num_nodes); + std::iota(indx.begin(), indx.end(), 0); // Find coordinate with largest range... const double *r = x; @@ -1001,7 +967,7 @@ template double Find_Min_Coord_Sep(ExoII_Read &file) } // Sort based on coordinate with largest range... - index_qsort(r, indx, num_nodes); + index_qsort(r, indx.data(), num_nodes); double min = DBL_MAX; switch (file.Dimension()) { @@ -1047,19 +1013,18 @@ template double Find_Min_Coord_Sep(ExoII_Read &file) break; } } - delete[] indx; return sqrt(min); } template -bool Compare_Maps_Internal(const INT *entity_map, bool partial_flag, const INT *entity_id_map1, - const INT *entity_id_map2, size_t num_entities1, size_t num_entities2, - const char *type) +bool Compare_Maps_Internal(const std::vector &entity_map, bool partial_flag, + const INT *entity_id_map1, const INT *entity_id_map2, + size_t num_entities1, size_t num_entities2, const char *type) { bool diff = false; int warn_count = 0; - if (entity_map != nullptr) { + if (!entity_map.empty()) { if (!interFace.dump_mapping) { // There is a map between file1 and file2, but all entities are // used in both files. @@ -1115,8 +1080,8 @@ bool Compare_Maps_Internal(const INT *entity_map, bool partial_flag, const INT * } template -bool Compare_Maps(ExoII_Read &file1, ExoII_Read &file2, const INT *node_map, - const INT *elmt_map, bool partial_flag) +bool Compare_Maps(ExoII_Read &file1, ExoII_Read &file2, const std::vector &node_map, + const std::vector &elmt_map, bool partial_flag) { // Check whether the node and element number maps from both file1 // and file2 match which indicates that we are comparing the same @@ -1135,19 +1100,19 @@ bool Compare_Maps(ExoII_Read &file1, ExoII_Read &file2, const INT *nod num_nodes1, num_nodes2, "node"); file2.Free_Node_Map(); - size_t num_elmts1 = file1.Num_Elmts(); - size_t num_elmts2 = file2.Num_Elmts(); + size_t num_elmts1 = file1.Num_Elements(); + size_t num_elmts2 = file2.Num_Elements(); // NOTE: file1 maps are already loaded... - file2.Load_Elmt_Map(); + file2.Load_Element_Map(); - const INT *elem_id_map1 = file1.Get_Elmt_Map(); - const INT *elem_id_map2 = file2.Get_Elmt_Map(); + const INT *elem_id_map1 = file1.Get_Element_Map(); + const INT *elem_id_map2 = file2.Get_Element_Map(); bool diff_elems = Compare_Maps_Internal(elmt_map, partial_flag, elem_id_map1, elem_id_map2, num_elmts1, num_elmts2, "element"); - file2.Free_Elmt_Map(); + file2.Free_Element_Map(); if (diff_nodes || diff_elems) { fmt::print("\n"); @@ -1155,27 +1120,30 @@ bool Compare_Maps(ExoII_Read &file1, ExoII_Read &file2, const INT *nod return diff_nodes || diff_elems; } -template void Compute_Maps(int *&node_map, int *&elmt_map, ExoII_Read &file1, - ExoII_Read &file2); -template bool Compare_Maps(ExoII_Read &file1, ExoII_Read &file2, const int *node_map, - const int *elmt_map, bool partial_flag); - -template void Compute_Partial_Maps(int *&node_map, int *&elmt_map, ExoII_Read &file1, - ExoII_Read &file2); -template void Compute_FileId_Maps(int *&node_map, int *&elmt_map, ExoII_Read &file1, - ExoII_Read &file2); -template void Dump_Maps(const int *node_map, const int *elmt_map, ExoII_Read &file1); +template void Compute_Maps(std::vector &node_map, std::vector &elmt_map, + ExoII_Read &file1, ExoII_Read &file2); +template bool Compare_Maps(ExoII_Read &file1, ExoII_Read &file2, + const std::vector &node_map, const std::vector &elmt_map, + bool partial_flag); + +template void Compute_Partial_Maps(std::vector &node_map, std::vector &elmt_map, + ExoII_Read &file1, ExoII_Read &file2); +template void Compute_FileId_Maps(std::vector &node_map, std::vector &elmt_map, + ExoII_Read &file1, ExoII_Read &file2); +template void Dump_Maps(const std::vector &node_map, const std::vector &elmt_map, + ExoII_Read &file1); template double Find_Min_Coord_Sep(ExoII_Read &file); -template void Compute_Maps(int64_t *&node_map, int64_t *&elmt_map, ExoII_Read &file1, - ExoII_Read &file2); +template void Compute_Maps(std::vector &node_map, std::vector &elmt_map, + ExoII_Read &file1, ExoII_Read &file2); template bool Compare_Maps(ExoII_Read &file1, ExoII_Read &file2, - const int64_t *node_map, const int64_t *elmt_map, bool partial_flag); - -template void Compute_Partial_Maps(int64_t *&node_map, int64_t *&elmt_map, - ExoII_Read &file1, ExoII_Read &file2); -template void Compute_FileId_Maps(int64_t *&node_map, int64_t *&elmt_map, - ExoII_Read &file1, ExoII_Read &file2); -template void Dump_Maps(const int64_t *node_map, const int64_t *elmt_map, - ExoII_Read &file1); + const std::vector &node_map, + const std::vector &elmt_map, bool partial_flag); + +template void Compute_Partial_Maps(std::vector &node_map, std::vector &elmt_map, + ExoII_Read &file1, ExoII_Read &file2); +template void Compute_FileId_Maps(std::vector &node_map, std::vector &elmt_map, + ExoII_Read &file1, ExoII_Read &file2); +template void Dump_Maps(const std::vector &node_map, const std::vector &elmt_map, + ExoII_Read &file1); template double Find_Min_Coord_Sep(ExoII_Read &file); diff --git a/packages/seacas/applications/exodiff/map.h b/packages/seacas/applications/exodiff/map.h index c0cbd3f4bcff..1426a169faae 100644 --- a/packages/seacas/applications/exodiff/map.h +++ b/packages/seacas/applications/exodiff/map.h @@ -10,25 +10,27 @@ enum class MapType { FILE_ORDER = 0, PARTIAL, USE_FILE_IDS, DISTANCE }; template -void Compute_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, ExoII_Read &file2); +void Compute_Maps(std::vector &node_map, std::vector &elmt_map, ExoII_Read &file1, + ExoII_Read &file2); template -void Compute_Partial_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, - ExoII_Read &file2); +void Compute_Partial_Maps(std::vector &node_map, std::vector &elmt_map, + ExoII_Read &file1, ExoII_Read &file2); template -void Compute_FileId_Maps(INT *&node_map, INT *&elmt_map, ExoII_Read &file1, - ExoII_Read &file2); +void Compute_FileId_Maps(std::vector &node_map, std::vector &elmt_map, + ExoII_Read &file1, ExoII_Read &file2); template -void Dump_Maps(const INT *node_map, const INT *elmt_map, ExoII_Read &file1); +void Dump_Maps(const std::vector &node_map, const std::vector &elmt_map, + ExoII_Read &file1); template -bool Check_Maps(const INT *node_map, const INT *elmt_map, const ExoII_Read &file1, - const ExoII_Read &file2); +bool Check_Maps(const std::vector &node_map, const std::vector &elmt_map, + const ExoII_Read &file1, const ExoII_Read &file2); template -bool Compare_Maps(ExoII_Read &file1, ExoII_Read &file2, const INT *node_map, - const INT *elmt_map, bool partial_flag); +bool Compare_Maps(ExoII_Read &file1, ExoII_Read &file2, const std::vector &node_map, + const std::vector &elmt_map, bool partial_flag); #endif diff --git a/packages/seacas/applications/exodiff/node_set.C b/packages/seacas/applications/exodiff/node_set.C index 43f746ddf754..6f5b6699e290 100644 --- a/packages/seacas/applications/exodiff/node_set.C +++ b/packages/seacas/applications/exodiff/node_set.C @@ -36,7 +36,8 @@ template const INT *Node_Set::Nodes() const { // See if already loaded... if (!nodes) { - load_nodes(); + std::vector tmp; + load_nodes(tmp); } return nodes; } @@ -49,7 +50,8 @@ template size_t Node_Set::Node_Id(size_t position) const // See if already loaded... if (!nodes) { - load_nodes(); + std::vector tmp; + load_nodes(tmp); } SMART_ASSERT(position < numEntity); return nodes[nodeIndex[position]]; @@ -63,16 +65,17 @@ template size_t Node_Set::Node_Index(size_t position) const // See if already loaded... if (!nodes) { - load_nodes(); + std::vector tmp; + load_nodes(tmp); } SMART_ASSERT(position < numEntity); SMART_ASSERT(nodeIndex != nullptr); return nodeIndex[position]; } -template void Node_Set::apply_map(const INT *node_map) +template void Node_Set::apply_map(const std::vector &node_map) { - SMART_ASSERT(node_map != nullptr); + SMART_ASSERT(!node_map.empty()); if (nodes != nullptr) { delete[] nodes; nodes = nullptr; @@ -82,7 +85,7 @@ template void Node_Set::apply_map(const INT *node_map) load_nodes(node_map); } -template void Node_Set::load_nodes(const INT *node_map) const +template void Node_Set::load_nodes(const std::vector &node_map) const { if (numEntity > 0) { nodes = new INT[numEntity]; @@ -91,7 +94,7 @@ template void Node_Set::load_nodes(const INT *node_map) cons SMART_ASSERT(nodeIndex != nullptr); ex_get_set(fileId, EX_NODE_SET, id_, nodes, nullptr); - if (node_map != nullptr) { + if (!node_map.empty()) { for (size_t i = 0; i < numEntity; i++) { nodes[i] = 1 + node_map[nodes[i] - 1]; } @@ -148,7 +151,6 @@ template void Node_Set::entity_load_params() if (err < 0) { Error(fmt::format("Failed to get nodeset parameters for nodeset {}. ! Aborting...\n", id_)); - exit(1); } numEntity = sets[0].num_entry; diff --git a/packages/seacas/applications/exodiff/node_set.h b/packages/seacas/applications/exodiff/node_set.h index fa9f1958fc95..a4c30d26ed78 100644 --- a/packages/seacas/applications/exodiff/node_set.h +++ b/packages/seacas/applications/exodiff/node_set.h @@ -23,7 +23,7 @@ template class Node_Set : public Exo_Entity ~Node_Set() override; - void apply_map(const INT *node_map); + void apply_map(const std::vector &node_map); const INT *Nodes() const; size_t Node_Id(size_t position) const; size_t Node_Index(size_t position) const; @@ -40,7 +40,7 @@ template class Node_Set : public Exo_Entity const char *label() const override { return "Nodeset"; } const char *short_label() const override { return "nodeset"; } - void load_nodes(const INT *node_map = nullptr) const; + void load_nodes(const std::vector &node_map) const; size_t num_dist_factors{0}; diff --git a/packages/seacas/applications/exodiff/side_set.C b/packages/seacas/applications/exodiff/side_set.C index 228359d07a19..713f191a73b7 100644 --- a/packages/seacas/applications/exodiff/side_set.C +++ b/packages/seacas/applications/exodiff/side_set.C @@ -53,16 +53,15 @@ template void Side_Set::entity_load_params() if (err < 0) { Error(fmt::format("{}: Failed to get sideset parameters for sideset {}. ! Aborting...\n", __func__, id_)); - exit(1); } numEntity = sets[0].num_entry; num_dist_factors = sets[0].num_distribution_factor; } -template void Side_Set::apply_map(const INT *elmt_map) +template void Side_Set::apply_map(const std::vector &elmt_map) { - SMART_ASSERT(elmt_map != nullptr); + SMART_ASSERT(!elmt_map.empty()); if (elmts != nullptr) { delete[] elmts; elmts = nullptr; @@ -74,7 +73,7 @@ template void Side_Set::apply_map(const INT *elmt_map) load_sides(elmt_map); } -template void Side_Set::load_sides(const INT *elmt_map) const +template void Side_Set::load_sides(const std::vector &elmt_map) const { int err = 0; if ((elmts == nullptr || sides == nullptr) && numEntity > 0) { @@ -86,10 +85,9 @@ template void Side_Set::load_sides(const INT *elmt_map) cons if (err < 0) { Error(fmt::format("{}: Failed to read side set {}! Aborting...\n", __func__, id_)); - exit(1); } - if (elmt_map != nullptr) { + if (!elmt_map.empty()) { for (size_t i = 0; i < numEntity; i++) { elmts[i] = 1 + elmt_map[elmts[i] - 1]; } @@ -120,7 +118,8 @@ template void Side_Set::load_sides(const INT *elmt_map) cons template void Side_Set::load_df() const { if (elmts == nullptr) { - load_sides(); + std::vector tmp; + load_sides(tmp); } if (dist_factors != nullptr) { @@ -142,7 +141,6 @@ template void Side_Set::load_df() const if (err < 0) { Error(fmt::format("{}: Failed to read side set node count for sideset {}! Aborting...\n", __func__, id_)); - exit(1); } } @@ -160,7 +158,6 @@ template void Side_Set::load_df() const "file says there should be {},\n\t\tbut ex_get_side_set_node_count says " "there should be {}! Aborting...\n", __func__, id_, num_dist_factors, index)); - exit(1); } SMART_ASSERT(index == num_dist_factors); dist_factors = new double[index]; @@ -169,32 +166,35 @@ template void Side_Set::load_df() const Error(fmt::format( "{}: Failed to read side set distribution factors for sideset {}! Aborting...\n", __func__, id_)); - exit(1); } } template const INT *Side_Set::Elements() const { - load_sides(); + std::vector tmp; + load_sides(tmp); return elmts; } template const INT *Side_Set::Sides() const { - load_sides(); + std::vector tmp; + load_sides(tmp); return sides; } template std::pair Side_Set::Side_Id(size_t position) const { - load_sides(); + std::vector tmp; + load_sides(tmp); SMART_ASSERT(position < numEntity); return std::make_pair(elmts[sideIndex[position]], sides[sideIndex[position]]); } template size_t Side_Set::Side_Index(size_t position) const { - load_sides(); + std::vector tmp; + load_sides(tmp); SMART_ASSERT(position < numEntity); return sideIndex[position]; } @@ -224,7 +224,6 @@ std::pair Side_Set::Distribution_Factor_Range(size_t side) const if (dfIndex == nullptr) { Error(fmt::format("{}: Failed to get distribution factors for sideset {}! Aborting...\n", __func__, id_)); - exit(1); } size_t side_index = sideIndex[side]; return std::make_pair(dfIndex[side_index], dfIndex[side_index + 1]); diff --git a/packages/seacas/applications/exodiff/side_set.h b/packages/seacas/applications/exodiff/side_set.h index 3106de629095..3ee4694b7fb0 100644 --- a/packages/seacas/applications/exodiff/side_set.h +++ b/packages/seacas/applications/exodiff/side_set.h @@ -20,7 +20,7 @@ template class Side_Set : public Exo_Entity Side_Set(int file_id, size_t id, size_t ns, size_t ndf = 0); ~Side_Set() override; - void apply_map(const INT *elmt_map); + void apply_map(const std::vector &elmt_map); const INT * Elements() const; const INT * Sides() const; std::pair Side_Id(size_t position) const; @@ -37,7 +37,7 @@ template class Side_Set : public Exo_Entity Side_Set(const Side_Set &); // Not written. const Side_Set &operator=(const Side_Set &); // Not written. - void load_sides(const INT *elmt_map = nullptr) const; + void load_sides(const std::vector &elmt_map) const; void load_df() const; void entity_load_params() override; diff --git a/packages/seacas/applications/exodiff/stringx.h b/packages/seacas/applications/exodiff/stringx.h index 5c1852313918..b947f0133d80 100644 --- a/packages/seacas/applications/exodiff/stringx.h +++ b/packages/seacas/applications/exodiff/stringx.h @@ -1,4 +1,4 @@ -// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// Copyright(C) 1999-2021 National Technology & Engineering Solutions // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with // NTESS, the U.S. Government retains certain rights in this software. // @@ -7,6 +7,9 @@ #ifndef STRINGX_H #define STRINGX_H +#include +#include + //! Compare a string against another "master" string, where the string, str, //! can be abbreiviated to as little as min_length characters. Returns true //! only if str has at least min_length characters and those that it does diff --git a/packages/seacas/applications/exodiff/util.C b/packages/seacas/applications/exodiff/util.C index 200d0c266b66..f2e71b71f075 100644 --- a/packages/seacas/applications/exodiff/util.C +++ b/packages/seacas/applications/exodiff/util.C @@ -1,4 +1,4 @@ -// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// Copyright(C) 1999-2021 National Technology & Engineering Solutions // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with // NTESS, the U.S. Government retains certain rights in this software. // @@ -9,13 +9,33 @@ #include "util.h" #include // for nullptr, memset #include +#include #include +#include "ED_SystemInterface.h" // for SystemInterface, interFace + #if defined(_MSC_VER) #include #define isatty _isatty #endif +int name_length() +{ + static int max_name_length = -1; + if (max_name_length < 0) { + max_name_length = std::max(max_name_length, max_string_length(interFace.glob_var_names)); + max_name_length = std::max(max_name_length, max_string_length(interFace.node_var_names)); + max_name_length = std::max(max_name_length, max_string_length(interFace.elmt_var_names)); + max_name_length = std::max(max_name_length, max_string_length(interFace.elmt_att_names)); + max_name_length = std::max(max_name_length, max_string_length(interFace.ns_var_names)); + max_name_length = std::max(max_name_length, max_string_length(interFace.ss_var_names)); + max_name_length = std::max(max_name_length, max_string_length(interFace.eb_var_names)); + max_name_length = std::max(max_name_length, max_string_length(interFace.fb_var_names)); + max_name_length++; + } + return max_name_length; +} + char **get_name_array(int size, int length) { char **names = nullptr; @@ -51,7 +71,23 @@ namespace { } } // namespace +void Error(std::ostringstream &x) +{ + std::ostringstream out; + fmt::print(out, "exodiff: ERROR: {}", x.str()); + ERR_OUT(out); + exit(EXIT_FAILURE); +} + void Error(const std::string &x) +{ + std::ostringstream out; + fmt::print(out, "exodiff: ERROR: {}", x); + ERR_OUT(out); + exit(EXIT_FAILURE); +} + +void Warning(const std::string &x) { std::ostringstream out; fmt::print(out, "exodiff: ERROR: {}", x); diff --git a/packages/seacas/applications/exodiff/util.h b/packages/seacas/applications/exodiff/util.h index 0faa8ea67ab7..3ae90624656c 100644 --- a/packages/seacas/applications/exodiff/util.h +++ b/packages/seacas/applications/exodiff/util.h @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -12,10 +12,13 @@ #include #include +int name_length(); char **get_name_array(int size, int length); void free_name_array(char **names, int size); void Error(const std::string &x); +void Error(std::ostringstream &buf); +void Warning(const std::string &x); void ERR_OUT(std::ostringstream &buf); void DIFF_OUT(std::ostringstream &buf, fmt::detail::color_type color = fmt::color::red); void DIFF_OUT(const std::string &buf, fmt::detail::color_type color = fmt::color::red); diff --git a/packages/seacas/applications/mapvar-kd/optkd.c b/packages/seacas/applications/mapvar-kd/optkd.c index ab3d7e79cf72..630dd4a9bb2b 100644 --- a/packages/seacas/applications/mapvar-kd/optkd.c +++ b/packages/seacas/applications/mapvar-kd/optkd.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -35,6 +35,7 @@ #include #include +#include #include #include #include @@ -153,15 +154,12 @@ void rf_select(real *a, int count, int l, int r, int k, int discrim) int findmaxspread(int l, int u, int dimension, real *points, int N) { int maxdim = 0; - real max = -FLT_MAX; - real min = FLT_MAX; real maxspread = -FLT_MAX; - int i, j; - for (i = 0; i < dimension; i++) { - max = -FLT_MAX; - min = FLT_MAX; - for (j = l; j <= u; j++) { + for (int i = 0; i < dimension; i++) { + real max = -FLT_MAX; + real min = FLT_MAX; + for (int j = l; j <= u; j++) { real val = points[N * i + perm[j]]; if (max < val) { max = val; @@ -182,15 +180,14 @@ int findmaxvariance(int l, int u, int dimension, real *points, int N) { int maxdim = 0; real max_var = 0.0; - int i, j; - for (i = 0; i < dimension; i++) { + for (int i = 0; i < dimension; i++) { real prev_mean = 0.0; real mean = 0.0; real variance = 0.0; real count = 0.0; - for (j = l; j <= u; j++) { + for (int j = l; j <= u; j++) { real val = points[N * i + perm[j]]; prev_mean = mean; count += 1.0; @@ -210,10 +207,9 @@ int findmaxvariance(int l, int u, int dimension, real *points, int N) /*******************************************************************************/ int check(real *points, int N, int l, int u, int m, int discrim) { - int i; - for (i = l; i < m; i++) + for (int i = l; i < m; i++) assert(points[discrim * N + perm[i]] <= points[discrim * N + perm[m]]); - for (i = m; i < u; i++) + for (int i = m; i < u; i++) assert(points[discrim * N + perm[i]] >= points[discrim * N + perm[m]]); return 1; } @@ -221,7 +217,6 @@ int check(real *points, int N, int l, int u, int m, int discrim) optkdNode *BuildkdTree(real *points, int N, int l, int u, int dimension) { optkdNode *p; - int m; NEWTREE(p); if (u - l + 1 <= BUCKETSIZE) { @@ -240,7 +235,7 @@ optkdNode *BuildkdTree(real *points, int N, int l, int u, int dimension) p->discrim = findmaxvariance(l, u, dimension, points, N); #endif - m = (l + u) / 2; + int m = (l + u) / 2; rf_select(points, N, l, u, m, p->discrim); assert(check(points, N, l, u, m, p->discrim)); @@ -255,14 +250,11 @@ optkdNode *BuildkdTree(real *points, int N, int l, int u, int dimension) /*******************************************************************************/ void KDBUILDTREE(real *points, int *numPoints, int *dimension) { - - int j; - /* initialize perm array */ assert(perm == NULL); perm = (int *)malloc(*numPoints * sizeof(int)); assert(perm != NULL); - for (j = 0; j < *numPoints; j++) { + for (int j = 0; j < *numPoints; j++) { perm[j] = j; } assert(Root == NULL); @@ -308,10 +300,8 @@ void KDKILLTREE() void optInRegion(optkdNode *P, int Dimension, real *Points, int N, real *xmin, real *xmax, int *found, int *count) { - int index, dc, InsideRange; - - for (index = P->lopt; index <= P->hipt; index++) { - InsideRange = 1; + for (int index = P->lopt; index <= P->hipt; index++) { + bool InsideRange = true; if (Dimension == 3) { int inval = perm[index]; @@ -319,14 +309,14 @@ void optInRegion(optkdNode *P, int Dimension, real *Points, int N, real *xmin, r Points[1 * N + inval] < xmin[1] || Points[1 * N + inval] > xmax[1] || Points[2 * N + inval] < xmin[2] || Points[2 * N + inval] > xmax[2]) { /* P is not in the region */ - InsideRange = 0; + InsideRange = false; } } else { - for (dc = 0; dc < Dimension; dc++) { + for (int dc = 0; dc < Dimension; dc++) { if (Points[dc * N + perm[index]] < xmin[dc] || Points[dc * N + perm[index]] > xmax[dc]) { /* P is not in the region */ - InsideRange = 0; + InsideRange = false; break; } } @@ -344,9 +334,7 @@ void optInRegion(optkdNode *P, int Dimension, real *Points, int N, real *xmin, r /***************************************************************************/ void optAddRegion(optkdNode *P, int Dimension, int *found, int *count) { - int index; - - for (index = P->lopt; index <= P->hipt; index++) { + for (int index = P->lopt; index <= P->hipt; index++) { found[(*count)++] = perm[index] + 1; } } @@ -408,8 +396,6 @@ void optRangeSearch(optkdNode *P, real *Points, int N, int Dimension, real *xmin static real BLow[MAX_DEPTH][6]; static real BHigh[MAX_DEPTH][6]; - int disc; - if (depth >= MAX_DEPTH) { fprintf(stderr, "Internal Error in optRangeSearch -- recursion depth too large.\n"); abort(); @@ -431,7 +417,7 @@ void optRangeSearch(optkdNode *P, real *Points, int N, int Dimension, real *xmin } /* Claim: P is not a bucket node */ - disc = P->discrim; + int disc = P->discrim; /* copy the region B into BLow, BHigh */ if (Dimension == 3) { diff --git a/packages/seacas/applications/nem_slice/elb_graph.C b/packages/seacas/applications/nem_slice/elb_graph.C index 49ce2ca3b696..e0119b1da355 100644 --- a/packages/seacas/applications/nem_slice/elb_graph.C +++ b/packages/seacas/applications/nem_slice/elb_graph.C @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -283,15 +283,14 @@ namespace { } /* Cycle through the elements */ - E_Type etype_last = NULL_EL; - E_Type etype = NULL_EL; int element_3d = 0; int nnodes = mesh->num_dims; int nsides = 0; for (size_t ecnt = 0; ecnt < mesh->num_elems; ecnt++) { - etype = mesh->elem_type[ecnt]; + E_Type etype = mesh->elem_type[ecnt]; + E_Type etype_last = NULL_EL; if (etype != etype_last) { etype_last = etype; element_3d = is_3d_element(mesh->elem_type[ecnt]); diff --git a/packages/seacas/applications/nem_slice/elb_loadbal.C b/packages/seacas/applications/nem_slice/elb_loadbal.C index 7c8607f08411..fe5c04025c8b 100644 --- a/packages/seacas/applications/nem_slice/elb_loadbal.C +++ b/packages/seacas/applications/nem_slice/elb_loadbal.C @@ -1052,14 +1052,8 @@ namespace { Mesh_Description *mesh, LB_Description *lb, Graph_Description *graph, int check_type) { - std::vector pt_list; - size_t nelem; - size_t nhold; - size_t count; size_t num_found = 0; - std::vector list_ptr; - size_t end; /* * look for discontinuities in the graph @@ -1134,6 +1128,7 @@ namespace { for (size_t ecnt = 0; ecnt < mesh->num_elems; ecnt++) { int proc = lb->vertex2proc[ecnt]; assert(proc < machine->num_procs); + size_t end = 0; if (proc == pcnt) { if (ecnt < (mesh->num_elems - 1)) { end = graph->start[ecnt + 1]; @@ -1164,6 +1159,7 @@ namespace { for (size_t ecnt = 0; ecnt < mesh->num_elems; ecnt++) { int proc = lb->vertex2proc[ecnt]; assert(proc < machine->num_procs); + size_t end = 0; if (proc == pcnt) { global_index[ki++] = ecnt; columns[kf++] = local_number[ecnt]; @@ -1222,7 +1218,7 @@ namespace { if (problem->global_mech == 1 || problem->local_mech == 1) { - pt_list.resize(graph->max_nsur); + std::vector pt_list(graph->max_nsur); std::vector hold_elem(graph->max_nsur); std::vector problems(mesh->num_nodes); std::vector proc_cnt(machine->num_procs); @@ -1271,8 +1267,8 @@ namespace { for (int ncnt = 0; ncnt < side_cnt; ncnt++) { - size_t node = side_nodes[ncnt]; - nhold = graph->sur_elem[node].size(); + size_t node = side_nodes[ncnt]; + size_t nhold = graph->sur_elem[node].size(); /* * look for the following cases @@ -1306,7 +1302,7 @@ namespace { int nsides2 = get_elem_info(NSIDES, etype2); - count = 0; + size_t count = 0; for (int cnt = 0; cnt < nsides2; cnt++) { ss_to_node_list(etype2, mesh->connect[el2], (cnt + 1), side_nodes2); @@ -1321,9 +1317,9 @@ namespace { hold_elem[i] = graph->sur_elem[side_nodes2[0]][pt_list[i]]; } - nelem = find_inter(hold_elem.data(), graph->sur_elem[side_nodes2[2]].data(), - nhold2, graph->sur_elem[side_nodes2[2]].size(), - pt_list.data()); + size_t nelem = find_inter( + hold_elem.data(), graph->sur_elem[side_nodes2[2]].data(), nhold2, + graph->sur_elem[side_nodes2[2]].size(), pt_list.data()); if (nelem >= 1) { count++; diff --git a/packages/seacas/applications/nem_slice/elb_main.C b/packages/seacas/applications/nem_slice/elb_main.C index b207278d1f45..03e1c6ef6910 100644 --- a/packages/seacas/applications/nem_slice/elb_main.C +++ b/packages/seacas/applications/nem_slice/elb_main.C @@ -33,10 +33,6 @@ #include // for MPI_Finalize, etc #endif -#ifdef SGI10K -#include -#endif - namespace { template void print_input(Machine_Description * /*machine*/, LB_Description * /*lb*/, diff --git a/packages/seacas/applications/slice/Slice.C b/packages/seacas/applications/slice/Slice.C index 9a33bcf107c8..a776e4d75ece 100644 --- a/packages/seacas/applications/slice/Slice.C +++ b/packages/seacas/applications/slice/Slice.C @@ -24,6 +24,7 @@ #include #include +#include #include #include #include diff --git a/packages/seacas/applications/zellij/Cell.h b/packages/seacas/applications/zellij/Cell.h index ec35cca0616e..719ee7822f94 100644 --- a/packages/seacas/applications/zellij/Cell.h +++ b/packages/seacas/applications/zellij/Cell.h @@ -20,6 +20,7 @@ // -- rank (for parallel -- which rank does this entry exist on) // +#include #include #include diff --git a/packages/seacas/applications/zellij/Doxyfile b/packages/seacas/applications/zellij/Doxyfile index 584cb3518032..ed9c4045a74d 100644 --- a/packages/seacas/applications/zellij/Doxyfile +++ b/packages/seacas/applications/zellij/Doxyfile @@ -1,4 +1,4 @@ -# Doxyfile 1.9.1 +# Doxyfile 1.9.2 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. @@ -32,7 +32,7 @@ DOXYFILE_ENCODING = UTF-8 # title of most generated pages and in a few other places. # The default value is: My Project. -PROJECT_NAME = "Zellij" +PROJECT_NAME = Zellij # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version @@ -93,14 +93,6 @@ ALLOW_UNICODE_NAMES = NO OUTPUT_LANGUAGE = English -# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all generated output in the proper direction. -# Possible values are: None, LTR, RTL and Context. -# The default value is: None. - -OUTPUT_TEXT_DIRECTION = None - # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. @@ -258,16 +250,16 @@ TAB_SIZE = 4 # the documentation. An alias has the form: # name=value # For example adding -# "sideeffect=@par Side Effects:\n" +# "sideeffect=@par Side Effects:^^" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines (in the resulting output). You can put ^^ in the value part of an -# alias to insert a newline as if a physical newline was in the original file. -# When you need a literal { or } or , in the value part of an alias you have to -# escape them by means of a backslash (\), this can lead to conflicts with the -# commands \{ and \} for these it is advised to use the version @{ and @} or use -# a double escape (\\{ and \\}) +# "Side Effects:". Note that you cannot put \n's in the value part of an alias +# to insert newlines (in the resulting output). You can put ^^ in the value part +# of an alias to insert a newline as if a physical newline was in the original +# file. When you need a literal { or } or , in the value part of an alias you +# have to escape them by means of a backslash (\), this can lead to conflicts +# with the commands \{ and \} for these it is advised to use the version @{ and +# @} or use a double escape (\\{ and \\}) ALIASES = @@ -312,8 +304,8 @@ OPTIMIZE_OUTPUT_SLICE = NO # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by doxygen: IDL, Java, JavaScript, -# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, -# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, +# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: # FortranFree, unknown formatted Fortran: Fortran. In the later case the parser # tries to guess whether the code is fixed or free formatted code, this is the # default for Fortran type files). For instance to make doxygen treat .inc files @@ -466,7 +458,7 @@ LOOKUP_CACHE_SIZE = 0 # than 0 to get more control over the balance between CPU load and processing # speed. At this moment only the input processing can be done using multiple # threads. Since this is still an experimental feature the default is set to 1, -# which efficively disables parallel processing. Please report any issues you +# which effectively disables parallel processing. Please report any issues you # encounter. Generating dot graphs in parallel is controlled by the # DOT_NUM_THREADS setting. # Minimum value: 0, maximum value: 32, default value: 1. @@ -610,6 +602,12 @@ HIDE_SCOPE_NAMES = NO HIDE_COMPOUND_REFERENCE= NO +# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class +# will show which file needs to be included to use the class. +# The default value is: YES. + +SHOW_HEADERFILE = YES + # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. @@ -767,7 +765,8 @@ FILE_VERSION_FILTER = # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. +# will be used as the name of the layout file. See also section "Changing the +# layout of pages" for information. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE @@ -813,18 +812,26 @@ WARNINGS = YES WARN_IF_UNDOCUMENTED = YES # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. +# potential errors in the documentation, such as documenting some parameters in +# a documented function twice, or documenting parameters that don't exist or +# using markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES +# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete +# function parameter documentation. If set to NO, doxygen will accept that some +# parameters have no documentation without warning. +# The default value is: YES. + +WARN_IF_INCOMPLETE_DOC = YES + # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. If -# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# value. If set to NO, doxygen will only warn about wrong parameter +# documentation, but not about the absence of documentation. If EXTRACT_ALL is +# set to YES then this flag will automatically be disabled. See also +# WARN_IF_INCOMPLETE_DOC # The default value is: NO. WARN_NO_PARAMDOC = NO @@ -864,7 +871,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = +INPUT = # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -888,13 +895,13 @@ INPUT_ENCODING = UTF-8 # # If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, # *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), -# *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, *.vhdl, -# *.ucf, *.qsf and *.ice. +# *.hh, *.hxx, *.hpp, *.h++, *.l, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, +# *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C +# comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, +# *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.C \ - *.c \ + *.c \ *.cc \ *.cxx \ *.cpp \ @@ -1260,7 +1267,7 @@ HTML_EXTRA_FILES = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see +# this color. Hue is specified as an angle on a color-wheel, see # https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. @@ -1270,7 +1277,7 @@ HTML_EXTRA_FILES = HTML_COLORSTYLE_HUE = 220 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A +# in the HTML output. For a value of 0 the output will use gray-scales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1377,8 +1384,12 @@ DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: -# https://www.microsoft.com/en-us/download/details.aspx?id=21138) on Windows. +# on Windows. In the beginning of 2021 Microsoft took the original page, with +# a.o. the download links, offline the HTML help workshop was already many years +# in maintenance mode). You can download the HTML help workshop from the web +# archives at Installation executable (see: +# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo +# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML @@ -1537,16 +1548,28 @@ DISABLE_INDEX = NO # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. +# further fine tune the look of the index (see "Fine-tuning the output"). As an +# example, the default style sheet generated by doxygen has an example that +# shows how to put an image at the root of the tree instead of the PROJECT_NAME. +# Since the tree basically has the same information as the tab index, you could +# consider setting DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = NO +# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the +# FULL_SIDEBAR option determines if the side bar is limited to only the treeview +# area (value NO) or if it should extend to the full height of the window (value +# YES). Setting this to YES gives a layout similar to +# https://docs.readthedocs.io with more room for contents, but less room for the +# project logo, title, and description. If either GENERATOR_TREEVIEW or +# DISABLE_INDEX is set to NO, this option has no effect. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FULL_SIDEBAR = NO + # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # @@ -1619,11 +1642,29 @@ FORMULA_MACROFILE = USE_MATHJAX = NO +# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. +# Note that the different versions of MathJax have different requirements with +# regards to the different settings, so it is possible that also other MathJax +# settings have to be changed when switching between the different MathJax +# versions. +# Possible values are: MathJax_2 and MathJax_3. +# The default value is: MathJax_2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_VERSION = MathJax_2 + # When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. +# the MathJax output. For more details about the output format see MathJax +# version 2 (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 +# (see: +# http://docs.mathjax.org/en/latest/web/components/output.html). # Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. +# compatibility. This is the name for Mathjax version 2, for MathJax version 3 +# this will be translated into chtml), NativeMML (i.e. MathML. Only supported +# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This +# is the name for Mathjax version 3, for MathJax version 2 this will be +# translated into HTML-CSS) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1636,15 +1677,21 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. -# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# MathJax from https://www.mathjax.org before deployment. The default value is: +# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 +# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = https://cdn.jsdelivr.net/npm/mathjax@2 # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example +# for MathJax version 2 (see https://docs.mathjax.org/en/v2.7-latest/tex.html +# #tex-and-latex-extensions): # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# For example for MathJax version 3 (see +# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): +# MATHJAX_EXTENSIONS = ams # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = @@ -1824,29 +1871,31 @@ PAPER_TYPE = a4 EXTRA_PACKAGES = -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the -# generated LaTeX document. The header should contain everything until the first -# chapter. If it is left blank doxygen will generate a standard header. See -# section "Doxygen usage" for information on how to let doxygen write the -# default header to a separate file. +# The LATEX_HEADER tag can be used to specify a user-defined LaTeX header for +# the generated LaTeX document. The header should contain everything until the +# first chapter. If it is left blank doxygen will generate a standard header. It +# is highly recommended to start with a default header using +# doxygen -w latex new_header.tex new_footer.tex new_stylesheet.sty +# and then modify the file new_header.tex. See also section "Doxygen usage" for +# information on how to generate the default header that doxygen normally uses. # -# Note: Only use a user-defined header if you know what you are doing! The -# following commands have a special meaning inside the header: $title, -# $datetime, $date, $doxygenversion, $projectname, $projectnumber, -# $projectbrief, $projectlogo. Doxygen will replace $title with the empty -# string, for the replacement values of the other commands the user is referred -# to HTML_HEADER. +# Note: Only use a user-defined header if you know what you are doing! +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. The following +# commands have a special meaning inside the header (and footer): For a +# description of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HEADER = -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the -# generated LaTeX document. The footer should contain everything after the last -# chapter. If it is left blank doxygen will generate a standard footer. See +# The LATEX_FOOTER tag can be used to specify a user-defined LaTeX footer for +# the generated LaTeX document. The footer should contain everything after the +# last chapter. If it is left blank doxygen will generate a standard footer. See # LATEX_HEADER for more information on how to generate a default footer and what -# special commands can be used inside the footer. -# -# Note: Only use a user-defined footer if you know what you are doing! +# special commands can be used inside the footer. See also section "Doxygen +# usage" for information on how to generate the default footer that doxygen +# normally uses. Note: Only use a user-defined footer if you know what you are +# doing! # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_FOOTER = @@ -1891,8 +1940,7 @@ USE_PDFLATEX = YES # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode # command to the generated LaTeX files. This will instruct LaTeX to keep running -# if errors occur, instead of asking the user for help. This option is also used -# when generating formulas in HTML. +# if errors occur, instead of asking the user for help. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1905,16 +1953,6 @@ LATEX_BATCHMODE = NO LATEX_HIDE_INDICES = NO -# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source -# code with syntax highlighting in the LaTeX output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_SOURCE_CODE = NO - # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See # https://en.wikipedia.org/wiki/BibTeX and \cite for more info. @@ -1995,16 +2033,6 @@ RTF_STYLESHEET_FILE = RTF_EXTENSIONS_FILE = -# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code -# with syntax highlighting in the RTF output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_RTF is set to YES. - -RTF_SOURCE_CODE = NO - #--------------------------------------------------------------------------- # Configuration options related to the man page output #--------------------------------------------------------------------------- @@ -2101,15 +2129,6 @@ GENERATE_DOCBOOK = NO DOCBOOK_OUTPUT = docbook -# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the -# program listings (including syntax highlighting and cross-referencing -# information) to the DOCBOOK output. Note that enabling this will significantly -# increase the size of the DOCBOOK output. -# The default value is: NO. -# This tag requires that the tag GENERATE_DOCBOOK is set to YES. - -DOCBOOK_PROGRAMLISTING = NO - #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- @@ -2612,8 +2631,8 @@ GENERATE_LEGEND = YES # If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate # files that are used to generate the various graphs. # -# Note: This setting is not only used for dot files but also for msc and -# plantuml temporary files. +# Note: This setting is not only used for dot files but also for msc temporary +# files. # The default value is: YES. DOT_CLEANUP = YES diff --git a/packages/seacas/applications/zellij/UnitCell.h b/packages/seacas/applications/zellij/UnitCell.h index 3ddb7e24086c..b1e95d477c68 100644 --- a/packages/seacas/applications/zellij/UnitCell.h +++ b/packages/seacas/applications/zellij/UnitCell.h @@ -6,6 +6,7 @@ #ifndef ZE_UnitCell_H #define ZE_UnitCell_H +#include #include #include #include diff --git a/packages/seacas/applications/zellij/Zellij.C b/packages/seacas/applications/zellij/Zellij.C index b1e7d64e5cb9..a39e10a4f247 100644 --- a/packages/seacas/applications/zellij/Zellij.C +++ b/packages/seacas/applications/zellij/Zellij.C @@ -140,7 +140,7 @@ template double zellij(SystemInterface &interFace, INT /*dummy*/) double end = Ioss::Utils::timer(); double hwm = (double)Ioss::Utils::get_hwm_memory_info() / 1024.0 / 1024.0; if (pu.parallel_rank() == 0) { - fmt::print("\n Total Execution time = {:.5} seconds.\n", end - begin); + fmt::print("\n Total Execution Time = {:.5} seconds.\n", end - begin); fmt::print(" High-Water Memory Use = {:.3} MiBytes.\n", hwm); } return (end - begin); diff --git a/packages/seacas/applications/zellij/Zellij.md b/packages/seacas/applications/zellij/Zellij.md index 2a2527593126..34c17a652116 100644 --- a/packages/seacas/applications/zellij/Zellij.md +++ b/packages/seacas/applications/zellij/Zellij.md @@ -42,7 +42,7 @@ usage: zellij [options] -lattice Elements handed out to id % proc_count) -random (Use the random method to decompose the input mesh in a parallel run. Elements assigned randomly to processors in a way that preserves balance - (do *not* use for a real run)) + (do _not_ use for a real run)) -ranks <$val> (Number of ranks to decompose mesh across) -start_rank <$val> (In partial output mode, start outputting decomposed files at this rank) @@ -178,17 +178,17 @@ END_LATTICE ## Unit Cell Template Mesh Requirements Zellij requires that the boundary mesh (`X` and `Y` faces) of each of the unit cell templates be a _regular_ "structured" mesh. Basically this means that the faces of the mesh elements on the boundary are in a regular rectangular grid such that each mesh face is rectangular (90 degree corners) and that the boundary mesh on the minimum `X` face is the same as that on the maximum `X` face and similarly for the minimum `Y` face and the maximum `Y` face. -Additionally, the X faces on *all* unit cells must match and the Y faces on *all* +Additionally, the X faces on _all_ unit cells must match and the Y faces on _all_ unit cells must match both in structure and in coordinate extent. This requirement is verified during execution. The `Z` faces are less constrained with the only requirement being that the coordinate extents of all `Z` faces must be the same (which follows from the `X` and `Y` face requirement); the structure of the mesh on the `Z` faces is arbitrary. -The unit cell meshes can contain any number of element blocks; however, each element block *must* contain hexahedral elements with 8-nodes per element. The element blocks do not need to be the same in each unit cell mesh, but if they do share the same element block `id`, then those elements will be combined into the same element block in the output mesh with the same `id`. +The unit cell meshes can contain any number of element blocks; however, each element block _must_ contain hexahedral elements with 8-nodes per element. The element blocks do not need to be the same in each unit cell mesh, but if they do share the same element block `id`, then those elements will be combined into the same element block in the output mesh with the same `id`. The output mesh will contain the union of all element blocks existing on the input mesh unit cells. For example, if: -* unit cell `0001` has element blocks `1 10 100` -* unit cell `0002` has element blocks `2 20 200` -* unit cell `0003` has element blocks `1 2 10 20` -* unit cell `0004` has element blocks `10 20 100 200` +* unit cell `0001` has element blocks `1 10 100` +* unit cell `0002` has element blocks `2 20 200` +* unit cell `0003` has element blocks `1 2 10 20` +* unit cell `0004` has element blocks `10 20 100 200` The output mesh will have element blocks `1 2 10 20 100 200` @@ -206,12 +206,12 @@ mesh via the command line option `-generate_sidesets ` where on which to generate a sideset. Valid letters are `xyzXYZ` or `ijkIJK` which correspond to: - * `x` or `i` for surface on minimum X coordinate (default name = `min_i`) - * `y` or `j` for surface on minimum Y coordinate (default name = `min_j`) - * `z` or `k` for surface on minimum Z coordinate (default name = `min_k`) - * `X` or `I` for surface on maximum X coordinate (default name = `max_i`) - * `Y` or `J` for surface on maximum Y coordinate (default name = `max_j`) - * `Z` or `K` for surface on maximum Z coordinate (default name = `max_k`) +* `x` or `i` for surface on minimum X coordinate (default name = `min_i`) +* `y` or `j` for surface on minimum Y coordinate (default name = `min_j`) +* `z` or `k` for surface on minimum Z coordinate (default name = `min_k`) +* `X` or `I` for surface on maximum X coordinate (default name = `max_i`) +* `Y` or `J` for surface on maximum Y coordinate (default name = `max_j`) +* `Z` or `K` for surface on maximum Z coordinate (default name = `max_k`) For example `-generate_sidesets xyXY` would generate sideset on the surfaces corresponding to the minimum and maximum X and Y coordinates @@ -226,7 +226,6 @@ and `name` is the name of the specified sideset. For example, minimum x and maximum X faces `left` and `right` respectively. There will be an error if two or more sidesets have the same name. - ## Parallel Execution Zellij can produce a mesh decomposed into a file-per-rank for use in a parallel analysis application. Note that Zellij itself is run @@ -243,12 +242,12 @@ algorithm that is used to break the lattice into `number_of_ranks` pieces each with approximately the same computational complexity. The decomposition methods are: -* `-rcb` Use recursive coordinate bisection method to decompose the input mesh in a parallel run. -* `-rib` Use recursive inertial bisection method to decompose the input mesh in a parallel run. -* `-hsfc` Use hilbert space-filling curve method to decompose the input mesh in a parallel run. -* `-linear` Use the linear method to decompose the input mesh in a parallel run. Elements in order first `n/p` to proc 0, next to proc 1. -* `-cyclic` Use the cyclic method to decompose the input mesh in a parallel run. Elements handed out to `id % proc_count`. -* `-random` Use the random method to decompose the input mesh in a parallel run. Elements are assigned randomly to processors in a way that preserves balance (do *not* use for a real run)) +* `-rcb` Use recursive coordinate bisection method to decompose the input mesh in a parallel run. +* `-rib` Use recursive inertial bisection method to decompose the input mesh in a parallel run. +* `-hsfc` Use hilbert space-filling curve method to decompose the input mesh in a parallel run. +* `-linear` Use the linear method to decompose the input mesh in a parallel run. Elements in order first `n/p` to proc 0, next to proc 1. +* `-cyclic` Use the cyclic method to decompose the input mesh in a parallel run. Elements handed out to `id % proc_count`. +* `-random` Use the random method to decompose the input mesh in a parallel run. Elements are assigned randomly to processors in a way that preserves balance (do _not_ use for a real run)) The `-hsfc` method is the default if no other decomposition method is specified. Note that the decomposition occurs at the _grid_ level so @@ -289,14 +288,18 @@ Most compute systems have a limit on the number of files that a program can have limit is 1024. The files that zellij deals with are (1) the unit cell meshes and (2) the per-rank output files, and (3) the standard input, output, and error files. Because of this, it is somewhat easy for a zellij execution to exceed the open file limit. Zellij attempts to handle this automatically using logic similar to: -* If the unit cell count exceeds the open file limit, then close each unit cell after each access before opening the next unit + +* If the unit cell count exceeds the open file limit, then close each unit cell after each access before opening the next unit cell mesh. -* If the number of `-ranks` that zellij is creating exceeds the open file count, then determine how many output files can be + +* If the number of `-ranks` that zellij is creating exceeds the open file count, then determine how many output files can be open at one time (max_open = open file limit - 3 - number of unit cells open simultaneously) and run zellij in a `subcycle` mode where it is only writing to `max_open` files at one time. -* If the `max_open` calculated in the above bullet is too small, then set the mode to only open a single unit cell mesh at a + +* If the `max_open` calculated in the above bullet is too small, then set the mode to only open a single unit cell mesh at a time and redo the calculation. -* If all else fails, run with only a single unit cell file open and only a single output mesh rank file open. + +* If all else fails, run with only a single unit cell file open and only a single output mesh rank file open. If the above logic fails and Zellij is unable to run without exceeding the open file count, you can specify the behavior manually using a combination of the `-minimize_open_files=` option and the `-subcycle` and `-rank_count <#>` @@ -304,18 +307,18 @@ options. The options to `-minimize_open_files` are: -* `UNIT` - only have a single unit cell mesh open at one time; close before accessing another unit cell mesh. -* `OUTPUT` - only have a single output rank mesh file open at one time. -* `ALL` - both of the above options. +* `UNIT` - only have a single unit cell mesh open at one time; close before accessing another unit cell mesh. +* `OUTPUT` - only have a single output rank mesh file open at one time. +* `ALL` - both of the above options. The `-subcycle` and `-rank_count <#>` options cause zellij to output `#` output files at a time and then cycle to the next `#` output files until all files have been output. For example, `zellij -ranks 1024 -subcycle -rank_count 256` would do the following: -* First cycle would output ranks 0 to 255, -* Second cycle would output ranks 256 to 511, -* Third cycle would output ranks 512 to 767, -* Fourth cycle would output ranks 768 to 1023. +* First cycle would output ranks 0 to 255, +* Second cycle would output ranks 256 to 511, +* Third cycle would output ranks 512 to 767, +* Fourth cycle would output ranks 768 to 1023. In this mode, there will the `#` output files open simultaneously (unless `-minimize_open_files=OUTPUT|ALL` was specified also). So the total number of open files will be `unit cell count + 3 + #` or @@ -330,14 +333,14 @@ time efficiency. Zellij stores the following data: -* For each unit cell template mesh: - * metadata - * 64-bit Ids of nodes on each min_I, max_I, min_J, max_J face -* For each entry in the lattice definition: - * metadata (approximately 1KiByte) - * temporarily it will hold 64-bit Ids of nodes on the max_I and max_J faces. This will be deleted once the upper `I` and upper `J` "neighbor" entry has been processed (see below) -* For the lattice: - * vector containing the lattice definition. +* For each unit cell template mesh: + * metadata + * 64-bit Ids of nodes on each min_I, max_I, min_J, max_J face +* For each entry in the lattice definition: + * metadata (approximately 1KiByte) + * temporarily it will hold 64-bit Ids of nodes on the max_I and max_J faces. This will be deleted once the upper `I` and upper `J` "neighbor" entry has been processed (see below) +* For the lattice: + * vector containing the lattice definition. The main memory use once the output file is being processed is the temporary storage containing the nodes on the `max_I` and `max_J` faces. The lattice is processed cell by cell. For an `II by JJ` sized grid, the cells are processed in the order `(1,1), (2,1), ... , (II, 1), (1,2), (2,2), ..., (II, JJ)`. The temporary storage on the `max_I` face is only needed until the next cell is processed. That is, for cell `(i,j)`, its `max_I` nodes will be used during the processing of cell `(i+1, j)` and then deleted. @@ -345,8 +348,8 @@ The temporary storage on the `max_J` face is retained for a longer time. For ce For a grid of size `(II, JJ)`, there will at most be: -* 1 temporary vector of size `max_I` nodes -* `II` temporary vectors of size `max_J` nodes. +* 1 temporary vector of size `max_I` nodes +* `II` temporary vectors of size `max_J` nodes. If you have a lattice that is rectangular (`II != JJ`), then it is more efficient for memory usage to make the `I` direction the smallest value if possible. @@ -378,9 +381,9 @@ The memory being used by zellij during execution will be output if the `--debug For a large model, the majority of the execution time is related to: -* Read/process/write element block connectivity -* Read/process/write nodal coordinates -* Categorize boundary nodes on each unit cell mesh +* Read/process/write element block connectivity +* Read/process/write nodal coordinates +* Categorize boundary nodes on each unit cell mesh ### Efficiency at the NetCDF level The Exodus format which is used for the unit cell template meshes and the output mesh uses the NetCDF library for on-disk storage. There are several variants of the NetCDF on-disk storage including the format: `netcdf3`, `netcdf4`, and `netcdf5` and the integer size (32-bit integers or 64-bit integers). Although these details are usually transparent to the user, they can affect the execution time especially when very large meshes are being processed. @@ -403,10 +406,10 @@ The benefit of the compression is that it can result in much smaller output (and #### Recommendations For minimal overhead, it is recommended that: -* Use the `netcdf4` format for all input and output meshes -* Use the same integer size for all input and output meshes - * The integer size of the output mesh can be specified using the `-32` or `-64` options. - * The `-64` option is the default. +* Use the `netcdf4` format for all input and output meshes +* Use the same integer size for all input and output meshes + * The integer size of the output mesh can be specified using the `-32` or `-64` options. + * The `-64` option is the default. It is most efficient if the format and integer size of the input mesh matches the output mesh. The format of the input meshes can be converted using the `io_shell` application with the `-netcdf4` and `-64` or `-32` options. The format and integer size of a mesh can be queried using the `exo_format` application. @@ -428,4 +431,3 @@ For illustration, here is the execution time for several runs with different for The fastest option is both input and output using 32-bit integers and the `netcdf4` format. Almost as fast is the case where the input format is `netcdf3` and the output `netcdf4`. The `64-bit` integer options with both input and output using `netcdf4` are slightly slower, but this is probably due to the doubling of the size of the integer data being read and written. The output mesh in this case consisted of 37.3 million elements and 38.5 million nodes in a grid of 46 x 46 unit cells. There were 56 unit cell template meshes. - diff --git a/packages/seacas/cmake/InstallSymLink.cmake b/packages/seacas/cmake/InstallSymLink.cmake new file mode 100644 index 000000000000..bbc7521977e1 --- /dev/null +++ b/packages/seacas/cmake/InstallSymLink.cmake @@ -0,0 +1,41 @@ +# This macro can be used to install symlinks, which turns out to be +# non-trivial due to CMake version differences and limitations on how +# files can be installed when building binary packages. +# +# The rule for binary packaging is that files (including symlinks) must +# be installed with the standard CMake install() macro. +# +# The rule for non-binary packaging is that CMake 2.6 cannot install() +# symlinks, but can create the symlink at install-time via scripting. +# Though, we assume that CMake 2.6 isn't going to be used to generate +# packages because versions later than 2.8.3 are superior for that purpose. +# +# _filepath: the absolute path to the file to symlink +# _sympath: absolute path of the installed symlink + +macro(InstallSymlink _filepath _sympath) + get_filename_component(_symname ${_sympath} NAME) + get_filename_component(_installdir ${_sympath} PATH) + + if (BINARY_PACKAGING_MODE) + execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink + ${_filepath} + ${CMAKE_CURRENT_BINARY_DIR}/${_symname}) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${_symname} + DESTINATION ${_installdir}) + else () + # scripting the symlink installation at install time should work + # for CMake 2.6.x and 2.8.x + install(CODE " + if (\"\$ENV{DESTDIR}\" STREQUAL \"\") + execute_process(COMMAND \"${CMAKE_COMMAND}\" -E create_symlink + ${_filepath} + ${_installdir}/${_symname}) + else () + execute_process(COMMAND \"${CMAKE_COMMAND}\" -E create_symlink + ${_filepath} + \$ENV{DESTDIR}/${_installdir}/${_symname}) + endif () + ") + endif () +endmacro(InstallSymlink) diff --git a/packages/seacas/cmake/tpls/FindTPLCatalyst2.cmake b/packages/seacas/cmake/tpls/FindTPLCatalyst2.cmake new file mode 100644 index 000000000000..d876004db650 --- /dev/null +++ b/packages/seacas/cmake/tpls/FindTPLCatalyst2.cmake @@ -0,0 +1,89 @@ +# @HEADER +# ************************************************************************ +# +# Trilinos: An Object-Oriented Solver Framework +# Copyright (2001) Sandia Corporation +# +# +# Copyright (2001) Sandia Corporation. Under the terms of Contract +# DE-AC04-94AL85000, there is a non-exclusive license for use of this +# work by or on behalf of the U.S. Government. Export of this program +# may require a license from the United States Government. +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the Corporation nor the names of the +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# NOTICE: The United States Government is granted for itself and others +# acting on its behalf a paid-up, nonexclusive, irrevocable worldwide +# license in this data to reproduce, prepare derivative works, and +# perform publicly and display publicly. Beginning five (5) years from +# July 25, 2001, the United States Government is granted for itself and +# others acting on its behalf a paid-up, nonexclusive, irrevocable +# worldwide license in this data to reproduce, prepare derivative works, +# distribute copies to the public, perform publicly and display +# publicly, and to permit others to do so. +# +# NEITHER THE UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT +# OF ENERGY, NOR SANDIA CORPORATION, NOR ANY OF THEIR EMPLOYEES, MAKES +# ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR +# RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY +# INFORMATION, APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS +# THAT ITS USE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS. +# +# ************************************************************************ +# @HEADER + +MESSAGE("-- Using FIND_PACKAGE(catalyst ...) ...") + +FIND_PACKAGE(catalyst) + +IF (catalyst_FOUND) + # Tell TriBITS that we found Catalyst2 and there no need to look any further! + GET_TARGET_PROPERTY(_Catalyst2_INCLUDE_DIRS catalyst::catalyst_headers INTERFACE_INCLUDE_DIRECTORIES) + GET_TARGET_PROPERTY(_Catalyst2_LIBRARY_LOCATION catalyst::catalyst LOCATION) + GET_FILENAME_COMPONENT(Catalyst2_LIBRARY_DIRS ${_Catalyst2_LIBRARY_LOCATION} DIRECTORY) + +# For compatibility with TriBITS: +SET(DOCSTR "List of semi-colon separated paths to look for the TPL Catalyst2") + +GET_TARGET_PROPERTY(Catalyst2_LOCATION catalyst::catalyst LOCATION) + +SET(TPL_Catalyst2_LIBRARIES ${Catalyst2_LOCATION} CACHE PATH ${DOCSTR}) +SET(TPL_Catalyst2_INCLUDE_DIRS ${_Catalyst2_INCLUDE_DIRS} CACHE PATH ${DOCSTR}) +SET(TPL_Catalyst2_LIBRARY_DIRS ${_Catalyst2_LIBRARY_DIRS} CACHE PATH ${DOCSTR}) + +ENDIF() + +# +# Third, call TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES() +# +TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( Catalyst2 + REQUIRED_HEADERS catalyst.h + REQUIRED_LIBS_NAMES catalyst + ) + +# NOTE: If FIND_PACKAGE(Catalyst2 ...) was called and successfully found Catalyst2, then +# TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES() will use the already-set +# variables TPL_Catalyst2_INCLUDE_DIRS and TPL_Catalyst2_LIBRARIES and then print them +# out (and set some other standard variables as well). This is the final +# "hook" into the TriBITS TPL system. diff --git a/packages/seacas/libraries/aprepro_lib/Doxyfile b/packages/seacas/libraries/aprepro_lib/Doxyfile index e6416042e837..6dcd934c9623 100644 --- a/packages/seacas/libraries/aprepro_lib/Doxyfile +++ b/packages/seacas/libraries/aprepro_lib/Doxyfile @@ -1,4 +1,4 @@ -# Doxyfile 1.9.0 +# Doxyfile 1.9.2 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. @@ -93,14 +93,6 @@ ALLOW_UNICODE_NAMES = NO OUTPUT_LANGUAGE = English -# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all generated output in the proper direction. -# Possible values are: None, LTR, RTL and Context. -# The default value is: None. - -OUTPUT_TEXT_DIRECTION = None - # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. @@ -248,16 +240,16 @@ TAB_SIZE = 1 # the documentation. An alias has the form: # name=value # For example adding -# "sideeffect=@par Side Effects:\n" +# "sideeffect=@par Side Effects:^^" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines (in the resulting output). You can put ^^ in the value part of an -# alias to insert a newline as if a physical newline was in the original file. -# When you need a literal { or } or , in the value part of an alias you have to -# escape them by means of a backslash (\), this can lead to conflicts with the -# commands \{ and \} for these it is advised to use the version @{ and @} or use -# a double escape (\\{ and \\}) +# "Side Effects:". Note that you cannot put \n's in the value part of an alias +# to insert newlines (in the resulting output). You can put ^^ in the value part +# of an alias to insert a newline as if a physical newline was in the original +# file. When you need a literal { or } or , in the value part of an alias you +# have to escape them by means of a backslash (\), this can lead to conflicts +# with the commands \{ and \} for these it is advised to use the version @{ and +# @} or use a double escape (\\{ and \\}) ALIASES = func{1}=\1 \ fparam{1}=\1 \ @@ -307,8 +299,8 @@ OPTIMIZE_OUTPUT_SLICE = NO # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by doxygen: IDL, Java, JavaScript, -# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, -# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, +# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: # FortranFree, unknown formatted Fortran: Fortran. In the later case the parser # tries to guess whether the code is fixed or free formatted code, this is the # default for Fortran type files). For instance to make doxygen treat .inc files @@ -461,7 +453,7 @@ LOOKUP_CACHE_SIZE = 0 # than 0 to get more control over the balance between CPU load and processing # speed. At this moment only the input processing can be done using multiple # threads. Since this is still an experimental feature the default is set to 1, -# which efficively disables parallel processing. Please report any issues you +# which effectively disables parallel processing. Please report any issues you # encounter. Generating dot graphs in parallel is controlled by the # DOT_NUM_THREADS setting. # Minimum value: 0, maximum value: 32, default value: 1. @@ -605,6 +597,12 @@ HIDE_SCOPE_NAMES = NO HIDE_COMPOUND_REFERENCE= NO +# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class +# will show which file needs to be included to use the class. +# The default value is: YES. + +SHOW_HEADERFILE = YES + # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. @@ -762,7 +760,8 @@ FILE_VERSION_FILTER = # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. +# will be used as the name of the layout file. See also section "Changing the +# layout of pages" for information. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE @@ -808,18 +807,26 @@ WARNINGS = YES WARN_IF_UNDOCUMENTED = NO # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. +# potential errors in the documentation, such as documenting some parameters in +# a documented function twice, or documenting parameters that don't exist or +# using markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES +# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete +# function parameter documentation. If set to NO, doxygen will accept that some +# parameters have no documentation without warning. +# The default value is: YES. + +WARN_IF_INCOMPLETE_DOC = YES + # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. If -# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# value. If set to NO, doxygen will only warn about wrong parameter +# documentation, but not about the absence of documentation. If EXTRACT_ALL is +# set to YES then this flag will automatically be disabled. See also +# WARN_IF_INCOMPLETE_DOC # The default value is: NO. WARN_NO_PARAMDOC = YES @@ -883,10 +890,10 @@ INPUT_ENCODING = UTF-8 # # If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, # *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), -# *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, *.vhdl, -# *.ucf, *.qsf and *.ice. +# *.hh, *.hxx, *.hpp, *.h++, *.l, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, +# *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C +# comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, +# *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.cc \ *.h @@ -1211,7 +1218,7 @@ HTML_EXTRA_FILES = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see +# this color. Hue is specified as an angle on a color-wheel, see # https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. @@ -1221,7 +1228,7 @@ HTML_EXTRA_FILES = HTML_COLORSTYLE_HUE = 224 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A +# in the HTML output. For a value of 0 the output will use gray-scales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1328,8 +1335,12 @@ DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: -# https://www.microsoft.com/en-us/download/details.aspx?id=21138) on Windows. +# on Windows. In the beginning of 2021 Microsoft took the original page, with +# a.o. the download links, offline the HTML help workshop was already many years +# in maintenance mode). You can download the HTML help workshop from the web +# archives at Installation executable (see: +# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo +# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML @@ -1488,16 +1499,28 @@ DISABLE_INDEX = NO # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. +# further fine tune the look of the index (see "Fine-tuning the output"). As an +# example, the default style sheet generated by doxygen has an example that +# shows how to put an image at the root of the tree instead of the PROJECT_NAME. +# Since the tree basically has the same information as the tab index, you could +# consider setting DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = YES +# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the +# FULL_SIDEBAR option determines if the side bar is limited to only the treeview +# area (value NO) or if it should extend to the full height of the window (value +# YES). Setting this to YES gives a layout similar to +# https://docs.readthedocs.io with more room for contents, but less room for the +# project logo, title, and description. If either GENERATOR_TREEVIEW or +# DISABLE_INDEX is set to NO, this option has no effect. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FULL_SIDEBAR = NO + # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # @@ -1570,11 +1593,29 @@ FORMULA_MACROFILE = USE_MATHJAX = NO +# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. +# Note that the different versions of MathJax have different requirements with +# regards to the different settings, so it is possible that also other MathJax +# settings have to be changed when switching between the different MathJax +# versions. +# Possible values are: MathJax_2 and MathJax_3. +# The default value is: MathJax_2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_VERSION = MathJax_2 + # When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. +# the MathJax output. For more details about the output format see MathJax +# version 2 (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 +# (see: +# http://docs.mathjax.org/en/latest/web/components/output.html). # Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. +# compatibility. This is the name for Mathjax version 2, for MathJax version 3 +# this will be translated into chtml), NativeMML (i.e. MathML. Only supported +# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This +# is the name for Mathjax version 3, for MathJax version 2 this will be +# translated into HTML-CSS) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1587,15 +1628,21 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. -# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# MathJax from https://www.mathjax.org before deployment. The default value is: +# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 +# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = http://www.mathjax.org/mathjax # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example +# for MathJax version 2 (see https://docs.mathjax.org/en/v2.7-latest/tex.html +# #tex-and-latex-extensions): # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# For example for MathJax version 3 (see +# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): +# MATHJAX_EXTENSIONS = ams # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = @@ -1775,29 +1822,31 @@ PAPER_TYPE = letter EXTRA_PACKAGES = -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the -# generated LaTeX document. The header should contain everything until the first -# chapter. If it is left blank doxygen will generate a standard header. See -# section "Doxygen usage" for information on how to let doxygen write the -# default header to a separate file. +# The LATEX_HEADER tag can be used to specify a user-defined LaTeX header for +# the generated LaTeX document. The header should contain everything until the +# first chapter. If it is left blank doxygen will generate a standard header. It +# is highly recommended to start with a default header using +# doxygen -w latex new_header.tex new_footer.tex new_stylesheet.sty +# and then modify the file new_header.tex. See also section "Doxygen usage" for +# information on how to generate the default header that doxygen normally uses. # -# Note: Only use a user-defined header if you know what you are doing! The -# following commands have a special meaning inside the header: $title, -# $datetime, $date, $doxygenversion, $projectname, $projectnumber, -# $projectbrief, $projectlogo. Doxygen will replace $title with the empty -# string, for the replacement values of the other commands the user is referred -# to HTML_HEADER. +# Note: Only use a user-defined header if you know what you are doing! +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. The following +# commands have a special meaning inside the header (and footer): For a +# description of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HEADER = -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the -# generated LaTeX document. The footer should contain everything after the last -# chapter. If it is left blank doxygen will generate a standard footer. See +# The LATEX_FOOTER tag can be used to specify a user-defined LaTeX footer for +# the generated LaTeX document. The footer should contain everything after the +# last chapter. If it is left blank doxygen will generate a standard footer. See # LATEX_HEADER for more information on how to generate a default footer and what -# special commands can be used inside the footer. -# -# Note: Only use a user-defined footer if you know what you are doing! +# special commands can be used inside the footer. See also section "Doxygen +# usage" for information on how to generate the default footer that doxygen +# normally uses. Note: Only use a user-defined footer if you know what you are +# doing! # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_FOOTER = @@ -1842,8 +1891,7 @@ USE_PDFLATEX = YES # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode # command to the generated LaTeX files. This will instruct LaTeX to keep running -# if errors occur, instead of asking the user for help. This option is also used -# when generating formulas in HTML. +# if errors occur, instead of asking the user for help. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1856,16 +1904,6 @@ LATEX_BATCHMODE = NO LATEX_HIDE_INDICES = NO -# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source -# code with syntax highlighting in the LaTeX output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_SOURCE_CODE = NO - # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See # https://en.wikipedia.org/wiki/BibTeX and \cite for more info. @@ -1946,16 +1984,6 @@ RTF_STYLESHEET_FILE = RTF_EXTENSIONS_FILE = -# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code -# with syntax highlighting in the RTF output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_RTF is set to YES. - -RTF_SOURCE_CODE = NO - #--------------------------------------------------------------------------- # Configuration options related to the man page output #--------------------------------------------------------------------------- @@ -2052,15 +2080,6 @@ GENERATE_DOCBOOK = NO DOCBOOK_OUTPUT = docbook -# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the -# program listings (including syntax highlighting and cross-referencing -# information) to the DOCBOOK output. Note that enabling this will significantly -# increase the size of the DOCBOOK output. -# The default value is: NO. -# This tag requires that the tag GENERATE_DOCBOOK is set to YES. - -DOCBOOK_PROGRAMLISTING = NO - #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- @@ -2565,8 +2584,8 @@ GENERATE_LEGEND = YES # If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate # files that are used to generate the various graphs. # -# Note: This setting is not only used for dot files but also for msc and -# plantuml temporary files. +# Note: This setting is not only used for dot files but also for msc temporary +# files. # The default value is: YES. DOT_CLEANUP = YES diff --git a/packages/seacas/libraries/aprepro_lib/apr_aprepro.cc b/packages/seacas/libraries/aprepro_lib/apr_aprepro.cc index 4f8946655dde..fc089af736d2 100644 --- a/packages/seacas/libraries/aprepro_lib/apr_aprepro.cc +++ b/packages/seacas/libraries/aprepro_lib/apr_aprepro.cc @@ -718,14 +718,13 @@ namespace SEAMS { void Aprepro::dumpsym(int type, const char *pre, bool doInternal) const { std::string comment = getsym("_C_")->value.svar; - int width = 10; // controls spacing/padding for the variable names - int fwidth = 20; // controls spacing/padding for the function names std::string spre; if (pre) { spre = pre; } + int width = 10; // controls spacing/padding for the variable names if (type == Parser::token::VAR || type == Parser::token::SVAR || type == Parser::token::AVAR) { (*infoStream) << "\n" << comment << " Variable = Value" << '\n'; @@ -777,6 +776,7 @@ namespace SEAMS { } else if (type == Parser::token::FNCT || type == Parser::token::SFNCT || type == Parser::token::AFNCT) { + int fwidth = 20; // controls spacing/padding for the function names (*infoStream) << trmclr::blue << "\nFunctions returning double:" << trmclr::normal << '\n'; for (unsigned hashval = 0; hashval < HASHSIZE; hashval++) { for (symrec *ptr = sym_table[hashval]; ptr != nullptr; ptr = ptr->next) { diff --git a/packages/seacas/libraries/aprepro_lib/apr_builtin.cc b/packages/seacas/libraries/aprepro_lib/apr_builtin.cc index dbd2a11d555f..2c2851c9efdd 100644 --- a/packages/seacas/libraries/aprepro_lib/apr_builtin.cc +++ b/packages/seacas/libraries/aprepro_lib/apr_builtin.cc @@ -1006,13 +1006,13 @@ namespace SEAMS { { size_t rows_to_skip = static_cast(skip); - const char * delim = ",\t "; - std::fstream *file = aprepro->open_file(filename, "r"); + std::fstream *file = aprepro->open_file(filename, "r"); if (file != nullptr) { size_t rows = 0; size_t cols = 0; + const char *delim = ",\t "; std::string line; while (std::getline(*file, line)) { rows++; diff --git a/packages/seacas/libraries/aprepro_lib/apr_getline_int.c b/packages/seacas/libraries/aprepro_lib/apr_getline_int.c index 175341c9016c..33f691ff4aab 100644 --- a/packages/seacas/libraries/aprepro_lib/apr_getline_int.c +++ b/packages/seacas/libraries/aprepro_lib/apr_getline_int.c @@ -1,6 +1,6 @@ /* - * Copyright (C) 1991, 1992, 1993, 2020 by Chris Thewalt (thewalt@ce.berkeley.edu) + * Copyright (C) 1991, 1992, 1993, 2020, 2021 by Chris Thewalt (thewalt@ce.berkeley.edu) * * Permission to use, copy, modify, and distribute this software * for any purpose and without fee is hereby granted, provided @@ -19,7 +19,7 @@ * Note: This version has been updated by Mike Gleason */ -#if defined(WIN32) || defined(_WINDOWS) || defined(_MSC_VER) +#if defined(_WIN64) || defined(WIN32) || defined(_WINDOWS) || defined(_MSC_VER) #define __windows__ 1 #include @@ -435,19 +435,16 @@ static void ap_gl_putc(int c) static void ap_gl_puts(const char *const buf) { - int len; - if (buf) { - len = strlen(buf); + int len = strlen(buf); write(1, buf, len); } } static void ap_gl_error(const char *const buf) { - int len = strlen(buf); - ap_gl_cleanup(); + int len = strlen(buf); write(2, buf, len); exit(1); } @@ -1536,49 +1533,31 @@ static void ap_gl_beep(void) static int ap_gl_do_tab_completion(char *buf, int *loc, size_t bufsize, int tabtab) { - char * startp; - size_t startoff, amt; - int c; - int qmode; - char * qstart; - char * lastspacestart; - char * cp; - int ntoalloc, nused, nalloced, i; - char **newap_gl_matchlist; - char * strtoadd, *strtoadd1; - int addquotes; - size_t llen, mlen, glen; - int allmatch; - char * curposp; - size_t lenaftercursor; - char * matchpfx; - int wasateol; - char ellipsessave[4]; - /* Zero out the rest of the buffer, so we can move stuff around * and know we'll still be NUL-terminated. */ - llen = strlen(buf); + size_t llen = strlen(buf); memset(buf + llen, 0, bufsize - llen); bufsize -= 4; /* leave room for a NUL, space, and two quotes. */ - curposp = buf + *loc; - wasateol = (*curposp == '\0'); - lenaftercursor = llen - (curposp - buf); + char * curposp = buf + *loc; + int wasateol = (*curposp == '\0'); + size_t lenaftercursor = llen - (curposp - buf); if (ap_gl_ellipses_during_completion != 0) { + char ellipsessave[4]; memcpy(ellipsessave, curposp, (size_t)4); memcpy(curposp, "... ", (size_t)4); ap_gl_fixup(ap_gl_prompt, ap_gl_pos, ap_gl_pos + 3); memcpy(curposp, ellipsessave, (size_t)4); } - qmode = 0; - qstart = NULL; - lastspacestart = NULL; - matchpfx = NULL; + int qmode = 0; + char *qstart = NULL; + char *lastspacestart = NULL; + char *matchpfx = NULL; - cp = buf; + char *cp = buf; while (cp < curposp) { - c = (int)*cp++; + int c = (int)*cp++; if (c == '\0') break; if ((c == '"') || (c == '\'')) { @@ -1605,6 +1584,7 @@ static int ap_gl_do_tab_completion(char *buf, int *loc, size_t bufsize, int tabt } } + char *startp; if (qstart != NULL) startp = qstart + 1; else if (lastspacestart != NULL) @@ -1612,8 +1592,8 @@ static int ap_gl_do_tab_completion(char *buf, int *loc, size_t bufsize, int tabt else startp = buf; - cp = startp; - mlen = (curposp - cp); + cp = startp; + size_t mlen = (curposp - cp); matchpfx = (char *)malloc(mlen + 1); memcpy(matchpfx, cp, mlen); @@ -1621,17 +1601,17 @@ static int ap_gl_do_tab_completion(char *buf, int *loc, size_t bufsize, int tabt #define AP_GL_COMPLETE_VECTOR_BLOCK_SIZE 64 - nused = 0; - ntoalloc = AP_GL_COMPLETE_VECTOR_BLOCK_SIZE; - newap_gl_matchlist = (char **)malloc((size_t)(sizeof(char *) * (ntoalloc + 1))); + int nused = 0; + int ntoalloc = AP_GL_COMPLETE_VECTOR_BLOCK_SIZE; + char **newap_gl_matchlist = (char **)malloc((size_t)(sizeof(char *) * (ntoalloc + 1))); if (newap_gl_matchlist == NULL) { free(matchpfx); ap_gl_beep(); return 0; } ap_gl_matchlist = newap_gl_matchlist; - nalloced = ntoalloc; - for (i = nused; i <= nalloced; i++) + int nalloced = ntoalloc; + for (int i = nused; i <= nalloced; i++) ap_gl_matchlist[i] = NULL; ap_gl_completion_exact_match_extra_char = ' '; @@ -1642,7 +1622,7 @@ static int ap_gl_do_tab_completion(char *buf, int *loc, size_t bufsize, int tabt (char **)realloc((char *)ap_gl_matchlist, (size_t)(sizeof(char *) * (ntoalloc + 1))); if (newap_gl_matchlist == NULL) { /* not enough memory to expand list -- abort */ - for (i = 0; i < nused; i++) + for (int i = 0; i < nused; i++) free(ap_gl_matchlist[i]); free(ap_gl_matchlist); ap_gl_matchlist = NULL; @@ -1652,7 +1632,7 @@ static int ap_gl_do_tab_completion(char *buf, int *loc, size_t bufsize, int tabt } ap_gl_matchlist = newap_gl_matchlist; nalloced = ntoalloc; - for (i = nused; i <= nalloced; i++) + for (int i = nused; i <= nalloced; i++) ap_gl_matchlist[i] = NULL; } cp = ap_gl_completion_proc(matchpfx, nused); @@ -1667,13 +1647,12 @@ static int ap_gl_do_tab_completion(char *buf, int *loc, size_t bufsize, int tabt } /* We now have an array strings, whose last element is NULL. */ - strtoadd = NULL; - strtoadd1 = NULL; - amt = 0; + char *strtoadd = NULL; + char *strtoadd1 = NULL; - addquotes = (ap_gl_filename_quoting_desired > 0) || - ((ap_gl_filename_quoting_desired < 0) && - (ap_gl_completion_proc == ap_gl_local_filename_completion_proc)); + int addquotes = (ap_gl_filename_quoting_desired > 0) || + ((ap_gl_filename_quoting_desired < 0) && + (ap_gl_completion_proc == ap_gl_local_filename_completion_proc)); if (nused == 1) { /* Exactly one match. */ @@ -1681,10 +1660,10 @@ static int ap_gl_do_tab_completion(char *buf, int *loc, size_t bufsize, int tabt } else if ((nused > 1) && (mlen > 0)) { /* Find the greatest amount that matches. */ - glen = 1; - for (glen = 1;; glen++) { - allmatch = 1; - for (i = 1; i < nused; i++) { + size_t glen = 1; + for (;; glen++) { + int allmatch = 1; + for (int i = 1; i < nused; i++) { if (ap_gl_matchlist[0][glen] != ap_gl_matchlist[i][glen]) { allmatch = 0; break; @@ -1710,8 +1689,8 @@ static int ap_gl_do_tab_completion(char *buf, int *loc, size_t bufsize, int tabt *startp++ = (char)qmode; } } - startoff = (size_t)(startp - buf); - amt = strlen(strtoadd); + size_t startoff = (size_t)(startp - buf); + size_t amt = strlen(strtoadd); if ((amt + startoff + lenaftercursor) >= bufsize) amt = bufsize - (amt + startoff + lenaftercursor); memmove(curposp + amt - mlen, curposp, lenaftercursor + 1 /* NUL */); @@ -1746,7 +1725,7 @@ static int ap_gl_do_tab_completion(char *buf, int *loc, size_t bufsize, int tabt } /* Don't need this any more. */ - for (i = 0; i < nused; i++) + for (int i = 0; i < nused; i++) free(ap_gl_matchlist[i]); free(ap_gl_matchlist); ap_gl_matchlist = NULL; diff --git a/packages/seacas/libraries/chaco/eigen/scale_diag.c b/packages/seacas/libraries/chaco/eigen/scale_diag.c index e39d0e97a012..c66864b5f2b4 100644 --- a/packages/seacas/libraries/chaco/eigen/scale_diag.c +++ b/packages/seacas/libraries/chaco/eigen/scale_diag.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -14,12 +14,10 @@ void scale_diag(double *vec, /* the vector to scale */ double *diag /* vector to scale by */ ) { - int i; - if (diag != NULL) { vec = vec + beg; diag = diag + beg; - for (i = end - beg + 1; i; i--) { + for (int i = end - beg + 1; i; i--) { *vec++ *= *diag++; } } @@ -32,13 +30,10 @@ void scale_diag_float(float *vec, /* the vector to scale */ float *diag /* vector to scale by */ ) { - - int i; - if (diag != NULL) { vec = vec + beg; diag = diag + beg; - for (i = end - beg + 1; i; i--) { + for (int i = end - beg + 1; i; i--) { *vec++ *= *diag++; } } diff --git a/packages/seacas/libraries/chaco/eigen/sturmcnt.c b/packages/seacas/libraries/chaco/eigen/sturmcnt.c index 1e7357e5bab2..bb0a6f5931b1 100644 --- a/packages/seacas/libraries/chaco/eigen/sturmcnt.c +++ b/packages/seacas/libraries/chaco/eigen/sturmcnt.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -23,17 +23,7 @@ int sturmcnt(double *alpha, /* vector of Lanczos scalars */ ) { extern double DOUBLE_MAX; /* maximum double precision number to be used */ - int i; /* loop index */ int cnt; /* number of sign changes in the sequence */ - int sign; /* algebraic sign of current sequence value */ - int last_sign; /* algebraic sign of previous sequence value */ - double * pntr_p; /* for stepping through sequence array */ - double * pntr_p1; /* for stepping through sequence array one behind */ - double * pntr_p2; /* for stepping through sequence array two behind */ - double * pntr_alpha; /* for stepping through alpha array */ - double * pntr_beta; /* for stepping through beta array */ - double limit; /* the cut-off for re-scaling the recurrence */ - double scale; /* magnitude of entry in p recursion */ if (j == 1) { /* have to special case this one */ @@ -46,21 +36,21 @@ int sturmcnt(double *alpha, /* vector of Lanczos scalars */ } else { /* compute the Sturm sequence */ - limit = sqrt(DOUBLE_MAX); - p[0] = 1; - p[1] = alpha[1] - mu; - pntr_p = &p[2]; - pntr_p1 = &p[1]; - pntr_p2 = &p[0]; - pntr_alpha = &alpha[2]; - pntr_beta = &beta[1]; - for (i = 2; i <= j; i++) { + double limit = sqrt(DOUBLE_MAX); + p[0] = 1; + p[1] = alpha[1] - mu; + double *pntr_p = &p[2]; + double *pntr_p1 = &p[1]; + double *pntr_p2 = &p[0]; + double *pntr_alpha = &alpha[2]; + double *pntr_beta = &beta[1]; + for (int i = 2; i <= j; i++) { *pntr_p++ = (*pntr_alpha - mu) * (*pntr_p1) - (*pntr_beta) * (*pntr_beta) * (*pntr_p2); pntr_alpha++; pntr_beta++; pntr_p1++; pntr_p2++; - scale = fabs(*pntr_p1); + double scale = fabs(*pntr_p1); if (scale > limit) { *pntr_p1 /= scale; *pntr_p2 /= scale; @@ -69,16 +59,17 @@ int sturmcnt(double *alpha, /* vector of Lanczos scalars */ } /* count sign changes */ - cnt = 0; - last_sign = 1; - pntr_p = &p[1]; - for (i = j; i; i--) { + cnt = 0; + int last_sign = 1; + pntr_p = &p[1]; + for (int i = j; i; i--) { if (*pntr_p != *pntr_p || fabs(*pntr_p) > limit) { return (-1); /* ... re-scaling failed; bail out and return error code. Note (x != x) is TRUE iff x is NaN, so this check should be ok on non-IEEE floating point systems too. */ } + int sign; if (*pntr_p > 0) { sign = 1; } diff --git a/packages/seacas/libraries/chaco/util/seconds.c b/packages/seacas/libraries/chaco/util/seconds.c index d8c91be9da99..dd448a3845e4 100644 --- a/packages/seacas/libraries/chaco/util/seconds.c +++ b/packages/seacas/libraries/chaco/util/seconds.c @@ -10,7 +10,9 @@ #else #include #endif -#if !defined(__CYGWIN__) && !defined(_MSC_VER) + +#if defined(__unix__) || defined(__unix) || defined(unix) || \ + (defined(__APPLE__) && defined(__MACH__)) #include #endif diff --git a/packages/seacas/libraries/exoIIv2for32/src/exo_jack_32.c b/packages/seacas/libraries/exoIIv2for32/src/exo_jack_32.c index ccdd27d6185f..be5f6f9a778f 100644 --- a/packages/seacas/libraries/exoIIv2for32/src/exo_jack_32.c +++ b/packages/seacas/libraries/exoIIv2for32/src/exo_jack_32.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -80,23 +80,21 @@ static void ex_fcdcpy(char *fstring, /* output string to be blank-filled */ int fslen, /* length of output string */ char *sstring) { /* input string, null-terminated */ - int i, len; - if (sstring != NULL) { - len = strlen(sstring); + int len = strlen(sstring); if (len > fslen) { len = fslen; } - for (i = 0; i < len; i++) { + for (int i = 0; i < len; i++) { *(fstring + i) = *(sstring + i); } - for (i = len; i < fslen; i++) { + for (int i = len; i < fslen; i++) { *(fstring + i) = ' '; } } else { - for (i = 0; i < fslen; i++) { + for (int i = 0; i < fslen; i++) { *(fstring + i) = ' '; } } diff --git a/packages/seacas/libraries/exodus/Doxyfile b/packages/seacas/libraries/exodus/Doxyfile index d41e316b542f..7603d423df84 100644 --- a/packages/seacas/libraries/exodus/Doxyfile +++ b/packages/seacas/libraries/exodus/Doxyfile @@ -1,4 +1,4 @@ -# Doxyfile 1.9.0 +# Doxyfile 1.9.2 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. @@ -93,14 +93,6 @@ ALLOW_UNICODE_NAMES = NO OUTPUT_LANGUAGE = English -# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all generated output in the proper direction. -# Possible values are: None, LTR, RTL and Context. -# The default value is: None. - -OUTPUT_TEXT_DIRECTION = None - # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. @@ -249,16 +241,16 @@ TAB_SIZE = 1 # the documentation. An alias has the form: # name=value # For example adding -# "sideeffect=@par Side Effects:\n" +# "sideeffect=@par Side Effects:^^" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines (in the resulting output). You can put ^^ in the value part of an -# alias to insert a newline as if a physical newline was in the original file. -# When you need a literal { or } or , in the value part of an alias you have to -# escape them by means of a backslash (\), this can lead to conflicts with the -# commands \{ and \} for these it is advised to use the version @{ and @} or use -# a double escape (\\{ and \\}) +# "Side Effects:". Note that you cannot put \n's in the value part of an alias +# to insert newlines (in the resulting output). You can put ^^ in the value part +# of an alias to insert a newline as if a physical newline was in the original +# file. When you need a literal { or } or , in the value part of an alias you +# have to escape them by means of a backslash (\), this can lead to conflicts +# with the commands \{ and \} for these it is advised to use the version @{ and +# @} or use a double escape (\\{ and \\}) ALIASES = func{1}=\1 \ fparam{1}=\1 \ @@ -308,8 +300,8 @@ OPTIMIZE_OUTPUT_SLICE = NO # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by doxygen: IDL, Java, JavaScript, -# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, -# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, +# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: # FortranFree, unknown formatted Fortran: Fortran. In the later case the parser # tries to guess whether the code is fixed or free formatted code, this is the # default for Fortran type files). For instance to make doxygen treat .inc files @@ -462,7 +454,7 @@ LOOKUP_CACHE_SIZE = 0 # than 0 to get more control over the balance between CPU load and processing # speed. At this moment only the input processing can be done using multiple # threads. Since this is still an experimental feature the default is set to 1, -# which efficively disables parallel processing. Please report any issues you +# which effectively disables parallel processing. Please report any issues you # encounter. Generating dot graphs in parallel is controlled by the # DOT_NUM_THREADS setting. # Minimum value: 0, maximum value: 32, default value: 1. @@ -606,6 +598,12 @@ HIDE_SCOPE_NAMES = NO HIDE_COMPOUND_REFERENCE= NO +# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class +# will show which file needs to be included to use the class. +# The default value is: YES. + +SHOW_HEADERFILE = YES + # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. @@ -763,7 +761,8 @@ FILE_VERSION_FILTER = # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. +# will be used as the name of the layout file. See also section "Changing the +# layout of pages" for information. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE @@ -809,18 +808,26 @@ WARNINGS = YES WARN_IF_UNDOCUMENTED = NO # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. +# potential errors in the documentation, such as documenting some parameters in +# a documented function twice, or documenting parameters that don't exist or +# using markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES +# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete +# function parameter documentation. If set to NO, doxygen will accept that some +# parameters have no documentation without warning. +# The default value is: YES. + +WARN_IF_INCOMPLETE_DOC = YES + # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. If -# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# value. If set to NO, doxygen will only warn about wrong parameter +# documentation, but not about the absence of documentation. If EXTRACT_ALL is +# set to YES then this flag will automatically be disabled. See also +# WARN_IF_INCOMPLETE_DOC # The default value is: NO. WARN_NO_PARAMDOC = YES @@ -887,10 +894,10 @@ INPUT_ENCODING = UTF-8 # # If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, # *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), -# *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, *.vhdl, -# *.ucf, *.qsf and *.ice. +# *.hh, *.hxx, *.hpp, *.h++, *.l, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, +# *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C +# comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, +# *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.c \ *.f \ @@ -1224,7 +1231,7 @@ HTML_EXTRA_FILES = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see +# this color. Hue is specified as an angle on a color-wheel, see # https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. @@ -1234,7 +1241,7 @@ HTML_EXTRA_FILES = HTML_COLORSTYLE_HUE = 224 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A +# in the HTML output. For a value of 0 the output will use gray-scales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1341,8 +1348,12 @@ DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: -# https://www.microsoft.com/en-us/download/details.aspx?id=21138) on Windows. +# on Windows. In the beginning of 2021 Microsoft took the original page, with +# a.o. the download links, offline the HTML help workshop was already many years +# in maintenance mode). You can download the HTML help workshop from the web +# archives at Installation executable (see: +# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo +# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML @@ -1501,16 +1512,28 @@ DISABLE_INDEX = NO # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. +# further fine tune the look of the index (see "Fine-tuning the output"). As an +# example, the default style sheet generated by doxygen has an example that +# shows how to put an image at the root of the tree instead of the PROJECT_NAME. +# Since the tree basically has the same information as the tab index, you could +# consider setting DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = YES +# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the +# FULL_SIDEBAR option determines if the side bar is limited to only the treeview +# area (value NO) or if it should extend to the full height of the window (value +# YES). Setting this to YES gives a layout similar to +# https://docs.readthedocs.io with more room for contents, but less room for the +# project logo, title, and description. If either GENERATOR_TREEVIEW or +# DISABLE_INDEX is set to NO, this option has no effect. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FULL_SIDEBAR = NO + # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # @@ -1583,11 +1606,29 @@ FORMULA_MACROFILE = USE_MATHJAX = NO +# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. +# Note that the different versions of MathJax have different requirements with +# regards to the different settings, so it is possible that also other MathJax +# settings have to be changed when switching between the different MathJax +# versions. +# Possible values are: MathJax_2 and MathJax_3. +# The default value is: MathJax_2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_VERSION = MathJax_2 + # When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. +# the MathJax output. For more details about the output format see MathJax +# version 2 (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 +# (see: +# http://docs.mathjax.org/en/latest/web/components/output.html). # Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. +# compatibility. This is the name for Mathjax version 2, for MathJax version 3 +# this will be translated into chtml), NativeMML (i.e. MathML. Only supported +# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This +# is the name for Mathjax version 3, for MathJax version 2 this will be +# translated into HTML-CSS) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1600,15 +1641,21 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. -# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# MathJax from https://www.mathjax.org before deployment. The default value is: +# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 +# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = http://www.mathjax.org/mathjax # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example +# for MathJax version 2 (see https://docs.mathjax.org/en/v2.7-latest/tex.html +# #tex-and-latex-extensions): # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# For example for MathJax version 3 (see +# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): +# MATHJAX_EXTENSIONS = ams # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = @@ -1788,29 +1835,31 @@ PAPER_TYPE = letter EXTRA_PACKAGES = -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the -# generated LaTeX document. The header should contain everything until the first -# chapter. If it is left blank doxygen will generate a standard header. See -# section "Doxygen usage" for information on how to let doxygen write the -# default header to a separate file. +# The LATEX_HEADER tag can be used to specify a user-defined LaTeX header for +# the generated LaTeX document. The header should contain everything until the +# first chapter. If it is left blank doxygen will generate a standard header. It +# is highly recommended to start with a default header using +# doxygen -w latex new_header.tex new_footer.tex new_stylesheet.sty +# and then modify the file new_header.tex. See also section "Doxygen usage" for +# information on how to generate the default header that doxygen normally uses. # -# Note: Only use a user-defined header if you know what you are doing! The -# following commands have a special meaning inside the header: $title, -# $datetime, $date, $doxygenversion, $projectname, $projectnumber, -# $projectbrief, $projectlogo. Doxygen will replace $title with the empty -# string, for the replacement values of the other commands the user is referred -# to HTML_HEADER. +# Note: Only use a user-defined header if you know what you are doing! +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. The following +# commands have a special meaning inside the header (and footer): For a +# description of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HEADER = -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the -# generated LaTeX document. The footer should contain everything after the last -# chapter. If it is left blank doxygen will generate a standard footer. See +# The LATEX_FOOTER tag can be used to specify a user-defined LaTeX footer for +# the generated LaTeX document. The footer should contain everything after the +# last chapter. If it is left blank doxygen will generate a standard footer. See # LATEX_HEADER for more information on how to generate a default footer and what -# special commands can be used inside the footer. -# -# Note: Only use a user-defined footer if you know what you are doing! +# special commands can be used inside the footer. See also section "Doxygen +# usage" for information on how to generate the default footer that doxygen +# normally uses. Note: Only use a user-defined footer if you know what you are +# doing! # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_FOOTER = @@ -1855,8 +1904,7 @@ USE_PDFLATEX = YES # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode # command to the generated LaTeX files. This will instruct LaTeX to keep running -# if errors occur, instead of asking the user for help. This option is also used -# when generating formulas in HTML. +# if errors occur, instead of asking the user for help. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1869,16 +1917,6 @@ LATEX_BATCHMODE = NO LATEX_HIDE_INDICES = NO -# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source -# code with syntax highlighting in the LaTeX output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_SOURCE_CODE = NO - # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See # https://en.wikipedia.org/wiki/BibTeX and \cite for more info. @@ -1959,16 +1997,6 @@ RTF_STYLESHEET_FILE = RTF_EXTENSIONS_FILE = -# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code -# with syntax highlighting in the RTF output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_RTF is set to YES. - -RTF_SOURCE_CODE = NO - #--------------------------------------------------------------------------- # Configuration options related to the man page output #--------------------------------------------------------------------------- @@ -2065,15 +2093,6 @@ GENERATE_DOCBOOK = NO DOCBOOK_OUTPUT = docbook -# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the -# program listings (including syntax highlighting and cross-referencing -# information) to the DOCBOOK output. Note that enabling this will significantly -# increase the size of the DOCBOOK output. -# The default value is: NO. -# This tag requires that the tag GENERATE_DOCBOOK is set to YES. - -DOCBOOK_PROGRAMLISTING = NO - #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- @@ -2181,7 +2200,7 @@ INCLUDE_FILE_PATTERNS = # recursively expanded use the := operator instead of the = operator. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -PREDEFINED = F2C(name,NAME)=name +PREDEFINED = "F2C(name,NAME)=name" # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this # tag can be used to specify a list of macro names that should be expanded. The @@ -2576,8 +2595,8 @@ GENERATE_LEGEND = YES # If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate # files that are used to generate the various graphs. # -# Note: This setting is not only used for dot files but also for msc and -# plantuml temporary files. +# Note: This setting is not only used for dot files but also for msc temporary +# files. # The default value is: YES. DOT_CLEANUP = YES diff --git a/packages/seacas/libraries/exodus/customdoxygen.css b/packages/seacas/libraries/exodus/customdoxygen.css deleted file mode 100644 index 01d679db49fb..000000000000 --- a/packages/seacas/libraries/exodus/customdoxygen.css +++ /dev/null @@ -1,686 +0,0 @@ -/* The standard CSS for doxygen */ - -body, table, div, p, dl { - font-family: Lucida Grande, Verdana, Geneva, Arial, sans-serif; - font-size: 12px; -} - -/* @group Heading Levels */ - -h1 { - font-size: 150%; -} - -h2 { - font-size: 120%; -} - -h3 { - font-size: 100%; -} - -dt { - font-weight: bold; -} - -div.multicol { - -moz-column-gap: 1em; - -webkit-column-gap: 1em; - -moz-column-count: 3; - -webkit-column-count: 3; -} - -p.startli, p.startdd, p.starttd { - margin-top: 2px; -} - -p.endli { - margin-bottom: 0px; -} - -p.enddd { - margin-bottom: 4px; -} - -p.endtd { - margin-bottom: 2px; -} - -/* @end */ - -caption { - font-weight: bold; -} - -span.legend { - font-size: 70%; - text-align: center; -} - -h3.version { - font-size: 90%; - text-align: center; -} - -div.qindex, div.navtab{ - background-color: #EBEFF6; - border: 1px solid #A3B4D7; - text-align: center; - margin: 2px; - padding: 2px; -} - -div.qindex, div.navpath { - width: 100%; - line-height: 140%; -} - -div.navtab { - margin-right: 15px; -} - -/* @group Link Styling */ - -a { - color: #3D578C; - font-weight: normal; - text-decoration: none; -} - -.contents a:visited { - color: #4665A2; -} - -a:hover { - text-decoration: underline; -} - -a.qindex { - font-weight: bold; -} - -a.qindexHL { - font-weight: bold; - background-color: #9CAFD4; - color: #ffffff; - border: 1px double #869DCA; -} - -.contents a.qindexHL:visited { - color: #ffffff; -} - -a.el { - font-weight: bold; -} - -a.elRef { -} - -a.code { - color: #4665A2; -} - -a.codeRef { - color: #4665A2; -} - -/* @end */ - -dl.el { - margin-left: -1cm; -} - -.fragment { - font-family: monospace, fixed; - font-size: 105%; -} - -pre.fragment { - border: 1px solid #C4CFE5; - background-color: #EEEEFF; - padding: 4px 6px; - margin: 4px 8px 4px 8px; - overflow: auto; - word-wrap: break-word; - font-size: 9pt; - line-height: 125%; -} - -div.ah { - background-color: black; - font-weight: bold; - color: #ffffff; - margin-bottom: 3px; - margin-top: 3px; - padding: 0.2em; - border: solid thin #333; - border-radius: 0.5em; - -webkit-border-radius: .5em; - -moz-border-radius: .5em; - box-shadow: 2px 2px 3px #999; - -webkit-box-shadow: 2px 2px 3px #999; - -moz-box-shadow: rgba(0, 0, 0, 0.15) 2px 2px 2px; - background-image: -webkit-gradient(linear, left top, left bottom, from(#eee), to(#000),color-stop(0.3, #444)); - background-image: -moz-linear-gradient(center top, #eee 0%, #444 40%, #000); -} - -div.groupHeader { - margin-left: 16px; - margin-top: 12px; - font-weight: bold; -} - -div.groupText { - margin-left: 16px; - font-style: italic; -} - -body { - background: white; - color: black; - margin: 0; -} - -div.contents { - margin-top: 10px; - margin-left: 10px; - margin-right: 10px; -} - -td.indexkey { - background-color: #EBEFF6; - font-weight: bold; - border: 1px solid #C4CFE5; - margin: 2px 0px 2px 0; - padding: 2px 10px; -} - -td.indexvalue { - background-color: #EBEFF6; - border: 1px solid #C4CFE5; - padding: 2px 10px; - margin: 2px 0px; -} - -tr.memlist { - background-color: #EEF1F7; -} - -p.formulaDsp { - text-align: center; -} - -img.formulaDsp { - -} - -img.formulaInl { - vertical-align: middle; -} - -div.center { - text-align: center; - margin-top: 0px; - margin-bottom: 0px; - padding: 0px; -} - -div.center img { - border: 0px; -} - -address.footer { - text-align: right; - padding-right: 12px; -} - -img.footer { - border: 0px; - vertical-align: middle; -} - -/* @group Code Colorization */ - -span.keyword { - color: #008000 -} - -span.keywordtype { - color: #604020 -} - -span.keywordflow { - color: #e08000 -} - -span.comment { - color: #800000 -} - -span.preprocessor { - color: #806020 -} - -span.stringliteral { - color: #002080 -} - -span.charliteral { - color: #008080 -} - -span.vhdldigit { - color: #ff00ff -} - -span.vhdlchar { - color: #000000 -} - -span.vhdlkeyword { - color: #700070 -} - -span.vhdllogic { - color: #ff0000 -} - -/* @end */ - -/* -.search { - color: #003399; - font-weight: bold; -} - -form.search { - margin-bottom: 0px; - margin-top: 0px; -} - -input.search { - font-size: 75%; - color: #000080; - font-weight: normal; - background-color: #e8eef2; -} -*/ - -td.tiny { - font-size: 75%; -} - -.dirtab { - padding: 4px; - border-collapse: collapse; - border: 1px solid #A3B4D7; -} - -th.dirtab { - background: #EBEFF6; - font-weight: bold; -} - -hr { - height: 0px; - border: none; - border-top: 1px solid #4A6AAA; -} - -hr.footer { - height: 1px; -} - -/* @group Member Descriptions */ - -table.memberdecls { - border-spacing: 0px; - padding: 0px; -} - -.mdescLeft, .mdescRight, -.memItemLeft, .memItemRight, -.memTemplItemLeft, .memTemplItemRight, .memTemplParams { - background-color: #F9FAFC; - border: none; - margin: 4px; - padding: 1px 0 0 8px; -} - -.mdescLeft, .mdescRight { - padding: 0px 8px 4px 8px; - color: #555; -} - -.memItemLeft, .memItemRight, .memTemplParams { - border-top: 1px solid #C4CFE5; -} - -.memItemLeft, .memTemplItemLeft { - white-space: nowrap; -} - -.memTemplParams { - color: #4665A2; - white-space: nowrap; -} - -/* @end */ - -/* @group Member Details */ - -/* Styles for detailed member documentation */ - -.memtemplate { - font-size: 80%; - color: #4665A2; - font-weight: normal; - margin-left: 9px; -} - -.memnav { - background-color: #EBEFF6; - border: 1px solid #A3B4D7; - text-align: center; - margin: 2px; - margin-right: 15px; - padding: 2px; -} - -.memitem { - padding: 0; - margin-bottom: 10px; -} - -.memname { - white-space: nowrap; - font-weight: bold; - margin-left: 6px; -} - -.memproto { - border-top: 1px solid #A8B8D9; - border-left: 1px solid #A8B8D9; - border-right: 1px solid #A8B8D9; - padding: 6px 0px 6px 0px; - color: #253555; - font-weight: bold; - text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); - /* opera specific markup */ - box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); - border-top-right-radius: 8px; - border-top-left-radius: 8px; - /* firefox specific markup */ - -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; - -moz-border-radius-topright: 8px; - -moz-border-radius-topleft: 8px; - /* webkit specific markup */ - -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); - -webkit-border-top-right-radius: 8px; - -webkit-border-top-left-radius: 8px; - background-image:url('nav_f.png'); - background-repeat:repeat-x; - background-color: #E2E8F2; - -} - -.memdoc { - border-bottom: 1px solid #A8B8D9; - border-left: 1px solid #A8B8D9; - border-right: 1px solid #A8B8D9; - padding: 2px 5px; - background-color: #FBFCFD; - border-top-width: 0; - /* opera specific markup */ - border-bottom-left-radius: 8px; - border-bottom-right-radius: 8px; - box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); - /* firefox specific markup */ - -moz-border-radius-bottomleft: 8px; - -moz-border-radius-bottomright: 8px; - -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; - background-image: -moz-linear-gradient(center top, #FFFFFF 0%, #FFFFFF 60%, #F7F8FB 95%, #EEF1F7); - /* webkit specific markup */ - -webkit-border-bottom-left-radius: 8px; - -webkit-border-bottom-right-radius: 8px; - -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); - background-image: -webkit-gradient(linear,center top,center bottom,from(#FFFFFF), color-stop(0.6,#FFFFFF), color-stop(0.60,#FFFFFF), color-stop(0.95,#F7F8FB), to(#EEF1F7)); -} - -.paramkey { - text-align: right; -} - -.paramtype { - white-space: nowrap; -} - -.paramname { - color: #602020; - white-space: nowrap; -} -.paramname em { - font-style: normal; -} - -.params, .retval, .exception, .tparams { - border-spacing: 6px 2px; -} - -.params .paramname, .retval .paramname { - font-weight: bold; - vertical-align: top; -} - -.params .paramtype { - font-style: italic; - vertical-align: top; -} - -.params .paramdir { - font-family: "courier new",courier,monospace; - vertical-align: top; -} - - - - -/* @end */ - -/* @group Directory (tree) */ - -/* for the tree view */ - -.ftvtree { - font-family: sans-serif; - margin: 0px; -} - -/* these are for tree view when used as main index */ - -.directory { - font-size: 9pt; - font-weight: bold; - margin: 5px; -} - -.directory h3 { - margin: 0px; - margin-top: 1em; - font-size: 11pt; -} - -/* -The following two styles can be used to replace the root node title -with an image of your choice. Simply uncomment the next two styles, -specify the name of your image and be sure to set 'height' to the -proper pixel height of your image. -*/ - -/* -.directory h3.swap { - height: 61px; - background-repeat: no-repeat; - background-image: url("yourimage.gif"); -} -.directory h3.swap span { - display: none; -} -*/ - -.directory > h3 { - margin-top: 0; -} - -.directory p { - margin: 0px; - white-space: nowrap; -} - -.directory div { - display: none; - margin: 0px; -} - -.directory img { - vertical-align: -30%; -} - -/* these are for tree view when not used as main index */ - -.directory-alt { - font-size: 100%; - font-weight: bold; -} - -.directory-alt h3 { - margin: 0px; - margin-top: 1em; - font-size: 11pt; -} - -.directory-alt > h3 { - margin-top: 0; -} - -.directory-alt p { - margin: 0px; - white-space: nowrap; -} - -.directory-alt div { - display: none; - margin: 0px; -} - -.directory-alt img { - vertical-align: -30%; -} - -/* @end */ - -div.dynheader { - margin-top: 8px; -} - -address { - font-style: normal; - color: #2A3D61; -} - -table.doxtable { - border-collapse:collapse; -} - -table.doxtable td, table.doxtable th { - border: 1px solid #2D4068; - padding: 3px 7px 2px; -} - -table.doxtable th { - background-color: #374F7F; - color: #FFFFFF; - font-size: 110%; - padding-bottom: 4px; - padding-top: 5px; - text-align:left; -} - -.tabsearch { - top: 0px; - left: 10px; - height: 36px; - background-image: url('tab_b.png'); - z-index: 101; - overflow: hidden; - font-size: 13px; -} - -.navpath ul -{ - font-size: 11px; - background-image:url('tab_b.png'); - background-repeat:repeat-x; - height:30px; - line-height:30px; - color:#8AA0CC; - border:solid 1px #C2CDE4; - overflow:hidden; - margin:0px; - padding:0px; -} - -.navpath li -{ - list-style-type:none; - float:left; - padding-left:10px; - padding-right: 15px; - background-image:url('bc_s.png'); - background-repeat:no-repeat; - background-position:right; - color:#364D7C; -} - -.navpath a -{ - height:32px; - display:block; - text-decoration: none; - outline: none; -} - -.navpath a:hover -{ - color:#6884BD; -} - -div.summary -{ - float: right; - font-size: 8pt; - padding-right: 5px; - width: 50%; - text-align: right; -} - -div.summary a -{ - white-space: nowrap; -} - -div.header -{ - background-image:url('nav_h.png'); - background-repeat:repeat-x; - background-color: #F9FAFC; - margin: 0px; - border-bottom: 1px solid #C4CFE5; -} - -div.headertitle -{ - padding: 5px 5px 5px 10px; -} - diff --git a/packages/seacas/libraries/exodus/include/exodusII.h b/packages/seacas/libraries/exodus/include/exodusII.h index 5e8416ba9393..a2f536e543de 100644 --- a/packages/seacas/libraries/exodus/include/exodusII.h +++ b/packages/seacas/libraries/exodus/include/exodusII.h @@ -54,12 +54,12 @@ #endif /* EXODUS version number */ -#define EXODUS_VERSION "8.11" +#define EXODUS_VERSION "8.12" #define EXODUS_VERSION_MAJOR 8 -#define EXODUS_VERSION_MINOR 11 -#define EXODUS_RELEASE_DATE "March 15, 2021" +#define EXODUS_VERSION_MINOR 12 +#define EXODUS_RELEASE_DATE "August 3, 2021" -#define EX_API_VERS 8.11f +#define EX_API_VERS 8.12f #define EX_API_VERS_NODOT (100 * EXODUS_VERSION_MAJOR + EXODUS_VERSION_MINOR) #define EX_VERS EX_API_VERS @@ -520,8 +520,7 @@ EXODUS_EXPORT int ex_get_group_ids(int parent_id, int *num_groups, int *group_id EXODUS_EXPORT int ex_get_info(int exoid, char **info); EXODUS_EXPORT int ex_get_qa(int exoid, char *qa_record[][4]); - -EXODUS_EXPORT int ex_put_info(int exoid, int num_info, char *info[]); +EXODUS_EXPORT int ex_put_info(int exoid, int num_info, char *const info[]); EXODUS_EXPORT int ex_put_qa(int exoid, int num_qa_records, char *qa_record[][4]); @@ -586,8 +585,8 @@ EXODUS_EXPORT int ex_get_truth_table(int exoid, ex_entity_type obj_type, int num int *var_tab); EXODUS_EXPORT int ex_put_all_var_param(int exoid, int num_g, int num_n, int num_e, - int *elem_var_tab, int num_m, int *nset_var_tab, int num_s, - int *sset_var_tab); + const int *elem_var_tab, int num_m, const int *nset_var_tab, + int num_s, const int *sset_var_tab); EXODUS_EXPORT int ex_put_time(int exoid, int time_step, const void *time_value); @@ -597,7 +596,7 @@ EXODUS_EXPORT int ex_put_variable_name(int exoid, ex_entity_type obj_type, int v const char *var_name); EXODUS_EXPORT int ex_put_variable_names(int exoid, ex_entity_type obj_type, int num_vars, - char *var_names[]); + char *const var_names[]); EXODUS_EXPORT int ex_put_variable_param(int exoid, ex_entity_type obj_type, int num_vars); @@ -605,12 +604,12 @@ EXODUS_EXPORT int ex_put_reduction_variable_name(int exoid, ex_entity_type obj_t const char *var_name); EXODUS_EXPORT int ex_put_reduction_variable_names(int exoid, ex_entity_type obj_type, int num_vars, - char *var_names[]); + char *const var_names[]); EXODUS_EXPORT int ex_put_reduction_variable_param(int exoid, ex_entity_type obj_type, int num_vars); EXODUS_EXPORT int ex_put_truth_table(int exoid, ex_entity_type obj_type, int num_blk, int num_var, - int *var_tab); + const int *var_tab); /* (MODIFIED) Write All Results Variables Parameters */ EXODUS_EXPORT int ex_put_all_var_param_ext(int exoid, const ex_var_params *vp); @@ -659,10 +658,10 @@ EXODUS_EXPORT int ex_get_init_info(int exoid, /* NemesisI file ID */ int * num_proc_in_f, /* Number of procs in this file */ char *ftype); -EXODUS_EXPORT int ex_put_init_info(int exoid, /* NemesisI file ID */ - int num_proc, /* Number of processors */ - int num_proc_in_f, /* Number of procs in this file */ - char *ftype); +EXODUS_EXPORT int ex_put_init_info(int exoid, /* NemesisI file ID */ + int num_proc, /* Number of processors */ + int num_proc_in_f, /* Number of procs in this file */ + const char *ftype); EXODUS_EXPORT int ex_get_init_global(int exoid, /* NemesisI file ID */ void_int *num_nodes_g, /* Number of global FEM nodes */ @@ -704,14 +703,15 @@ EXODUS_EXPORT int ex_put_loadbal_param(int exoid, /* NemesisI file int processor /* Processor ID */ ); -EXODUS_EXPORT int ex_put_loadbal_param_cc(int exoid, /* NetCDF/Exodus file ID */ - void_int *num_int_nodes, /* Number of internal node IDs */ - void_int *num_bor_nodes, /* Number of border node IDs */ - void_int *num_ext_nodes, /* Number of external node IDs */ - void_int *num_int_elems, /* Number of internal elem IDs */ - void_int *num_bor_elems, /* Number of border elem IDs */ - void_int *num_node_cmaps, /* Number of nodal comm maps */ - void_int *num_elem_cmaps /* Number of elem comm maps */ +EXODUS_EXPORT int +ex_put_loadbal_param_cc(int exoid, /* NetCDF/Exodus file ID */ + const void_int *num_int_nodes, /* Number of internal node IDs */ + const void_int *num_bor_nodes, /* Number of border node IDs */ + const void_int *num_ext_nodes, /* Number of external node IDs */ + const void_int *num_int_elems, /* Number of internal elem IDs */ + const void_int *num_bor_elems, /* Number of border elem IDs */ + const void_int *num_node_cmaps, /* Number of nodal comm maps */ + const void_int *num_elem_cmaps /* Number of elem comm maps */ ); /* Utility function to replace strncpy, strcpy -- guarantee null termination */ @@ -780,11 +780,12 @@ EXODUS_EXPORT int ex_get_attr_param(int exoid, ex_entity_type obj_type, ex_entit int *num_attrs); EXODUS_EXPORT int ex_put_concat_elem_block(int exoid, const void_int *elem_blk_id, - char *elem_type[], const void_int *num_elem_this_blk, + char *const elem_type[], + const void_int *num_elem_this_blk, const void_int *num_nodes_per_elem, const void_int *num_attr_this_blk, int define_maps); -EXODUS_EXPORT int ex_put_coord_names(int exoid, char *coord_names[]); +EXODUS_EXPORT int ex_put_coord_names(int exoid, char *const coord_names[]); EXODUS_EXPORT int ex_put_coord(int exoid, const void *x_coor, const void *y_coor, const void *z_coor); @@ -811,14 +812,14 @@ EXODUS_EXPORT int ex_get_partial_id_map(int exoid, ex_entity_type map_type, void_int *map); EXODUS_EXPORT int ex_put_coordinate_frames(int exoid, int nframes, const void_int *cf_ids, - void *pt_coordinates, const char *tags); + const void *pt_coordinates, const char *tags); EXODUS_EXPORT int ex_put_map_param(int exoid, int num_node_maps, int num_elem_maps); EXODUS_EXPORT int ex_put_name(int exoid, ex_entity_type obj_type, ex_entity_id entity_id, const char *name); -EXODUS_EXPORT int ex_put_names(int exoid, ex_entity_type obj_type, char *names[]); +EXODUS_EXPORT int ex_put_names(int exoid, ex_entity_type obj_type, char *const names[]); EXODUS_EXPORT int ex_put_partial_one_attr(int exoid, ex_entity_type obj_type, ex_entity_id obj_id, int64_t start_num, int64_t num_ent, int attrib_index, @@ -934,13 +935,15 @@ EXODUS_EXPORT int ex_put_blobs(int exoid, size_t count, const struct ex_blob *bl EXODUS_EXPORT int ex_get_blobs(int exoid, struct ex_blob *blobs); /* Write arbitrary integer, double, or text attributes on an entity */ -EXODUS_EXPORT int ex_put_attribute(int exoid, ex_attribute attributes); -EXODUS_EXPORT int ex_put_attributes(int exoid, size_t attr_count, ex_attribute *attributes); +EXODUS_EXPORT int ex_put_attribute(int exoid, const ex_attribute attributes); +EXODUS_EXPORT int ex_put_attributes(int exoid, size_t attr_count, const ex_attribute *attributes); EXODUS_EXPORT int ex_put_double_attribute(int exoid, ex_entity_type obj_type, ex_entity_id id, - const char *atr_name, int num_values, double *values); + const char *atr_name, int num_values, + const double *values); EXODUS_EXPORT int ex_put_integer_attribute(int exoid, ex_entity_type obj_type, ex_entity_id id, - const char *atr_name, int num_values, void_int *values); + const char *atr_name, int num_values, + const void_int *values); EXODUS_EXPORT int ex_put_text_attribute(int exoid, ex_entity_type obj_type, ex_entity_id id, const char *atr_name, const char *value); @@ -1037,10 +1040,10 @@ EXODUS_EXPORT int ex_get_ns_param_global(int exoid, /**< NetCDF/Exodu ); EXODUS_EXPORT int -ex_put_ns_param_global(int exoid, /**< NemesisI file ID */ - void_int *global_ids, /**< Vector of global node-set IDs */ - void_int *node_cnts, /**< Vector of node counts in node-sets */ - void_int *df_cnts /**< Vector of dist factor counts in node-sets */ +ex_put_ns_param_global(int exoid, /**< NemesisI file ID */ + const void_int *global_ids, /**< Vector of global node-set IDs */ + const void_int *node_cnts, /**< Vector of node counts in node-sets */ + const void_int *df_cnts /**< Vector of dist factor counts in node-sets */ ); EXODUS_EXPORT int ex_get_ss_param_global(int exoid, /**< NetCDF/Exodus file ID */ @@ -1049,12 +1052,12 @@ EXODUS_EXPORT int ex_get_ss_param_global(int exoid, /**< NetCDF/Exodu void_int *df_cnts /**< Global dist. factor count */ ); -EXODUS_EXPORT int -ex_put_ss_param_global(int exoid, /**< NemesisI file ID */ - void_int *global_ids, /**< Vector of global side-set IDs */ - void_int *side_cnts, /**< Vector of element/side counts in each side set */ - void_int *df_cnts /**< Vector of dist. factor */ - /**< counts in each side set */ +EXODUS_EXPORT int ex_put_ss_param_global( + int exoid, /**< NemesisI file ID */ + const void_int *global_ids, /**< Vector of global side-set IDs */ + const void_int *side_cnts, /**< Vector of element/side counts in each side set */ + const void_int *df_cnts /**< Vector of dist. factor */ + /**< counts in each side set */ ); EXODUS_EXPORT int @@ -1064,9 +1067,9 @@ ex_get_eb_info_global(int exoid, /**< NemesisI file ID ); EXODUS_EXPORT int -ex_put_eb_info_global(int exoid, /**< NemesisI file ID */ - void_int *el_blk_ids, /**< Vector of global element IDs */ - void_int *el_blk_cnts /**< Vector of global element counts */ +ex_put_eb_info_global(int exoid, /**< NemesisI file ID */ + const void_int *el_blk_ids, /**< Vector of global element IDs */ + const void_int *el_blk_cnts /**< Vector of global element counts */ ); /*============================================================================= @@ -1087,11 +1090,12 @@ EXODUS_EXPORT int ex_get_processor_node_maps(int exoid, /**< NetCDF/Ex int processor /**< Processor IDs */ ); -EXODUS_EXPORT int ex_put_processor_node_maps(int exoid, /**< NetCDF/Exodus file ID */ - void_int *node_mapi, /**< Internal FEM node IDs */ - void_int *node_mapb, /**< Border FEM node IDs */ - void_int *node_mape, /**< External FEM node IDs */ - int proc_id /**< This processor ID */ +EXODUS_EXPORT int +ex_put_processor_node_maps(int exoid, /**< NetCDF/Exodus file ID */ + const void_int *node_mapi, /**< Internal FEM node IDs */ + const void_int *node_mapb, /**< Border FEM node IDs */ + const void_int *node_mape, /**< External FEM node IDs */ + int proc_id /**< This processor ID */ ); EXODUS_EXPORT int ex_get_processor_elem_maps(int exoid, /**< NetCDF/Exodus file ID */ @@ -1100,10 +1104,11 @@ EXODUS_EXPORT int ex_get_processor_elem_maps(int exoid, /**< NetCDF/Ex int processor /**< Processor ID */ ); -EXODUS_EXPORT int ex_put_processor_elem_maps(int exoid, /**< NetCDF/Exodus file ID */ - void_int *elem_mapi, /**< Internal FEM element IDs */ - void_int *elem_mapb, /**< Border FEM element IDs */ - int processor /**< This processor ID */ +EXODUS_EXPORT int +ex_put_processor_elem_maps(int exoid, /**< NetCDF/Exodus file ID */ + const void_int *elem_mapi, /**< Internal FEM element IDs */ + const void_int *elem_mapb, /**< Border FEM element IDs */ + int processor /**< This processor ID */ ); /*============================================================================= @@ -1119,22 +1124,23 @@ ex_get_cmap_params(int exoid, /**< NetCDF/Exodus file ID */ int processor /**< This processor ID */ ); -EXODUS_EXPORT int ex_put_cmap_params(int exoid, /**< NetCDF/Exodus file ID */ - void_int *node_cmap_ids, /**< Node map IDs */ - void_int *node_cmap_node_cnts, /**< Nodes in nodal comm */ - void_int *elem_cmap_ids, /**< Elem map IDs */ - void_int *elem_cmap_elem_cnts, /**< Elems in elemental comm */ - int64_t processor /**< This processor ID */ +EXODUS_EXPORT int +ex_put_cmap_params(int exoid, /**< NetCDF/Exodus file ID */ + const void_int *node_cmap_ids, /**< Node map IDs */ + const void_int *node_cmap_node_cnts, /**< Nodes in nodal comm */ + const void_int *elem_cmap_ids, /**< Elem map IDs */ + const void_int *elem_cmap_elem_cnts, /**< Elems in elemental comm */ + int64_t processor /**< This processor ID */ ); EXODUS_EXPORT int -ex_put_cmap_params_cc(int exoid, /**< NetCDF/Exodus file ID */ - void_int *node_cmap_ids, /**< Node map IDs */ - void_int *node_cmap_node_cnts, /**< Nodes in nodal comm */ - void_int *node_proc_ptrs, /**< Pointer into array for node maps */ - void_int *elem_cmap_ids, /**< Elem map IDs */ - void_int *elem_cmap_elem_cnts, /**< Elems in elemental comm */ - void_int *elem_proc_ptrs /**< Pointer into array for elem maps */ +ex_put_cmap_params_cc(int exoid, /**< NetCDF/Exodus file ID */ + const void_int *node_cmap_ids, /**< Node map IDs */ + const void_int *node_cmap_node_cnts, /**< Nodes in nodal comm */ + const void_int *node_proc_ptrs, /**< Pointer into array for node maps */ + const void_int *elem_cmap_ids, /**< Elem map IDs */ + const void_int *elem_cmap_elem_cnts, /**< Elems in elemental comm */ + const void_int *elem_proc_ptrs /**< Pointer into array for elem maps */ ); EXODUS_EXPORT int ex_get_node_cmap(int exoid, /**< NetCDF/Exodus file ID */ @@ -1144,21 +1150,21 @@ EXODUS_EXPORT int ex_get_node_cmap(int exoid, /**< NetCDF/Exodus fil int processor /**< This processor ID */ ); -EXODUS_EXPORT int ex_put_node_cmap(int exoid, /**< NetCDF/Exodus file ID */ - ex_entity_id map_id, /**< Nodal comm map ID */ - void_int * node_ids, /**< FEM node IDs */ - void_int * proc_ids, /**< Processor IDs */ - int processor /**< This processor ID */ +EXODUS_EXPORT int ex_put_node_cmap(int exoid, /**< NetCDF/Exodus file ID */ + ex_entity_id map_id, /**< Nodal comm map ID */ + const void_int *node_ids, /**< FEM node IDs */ + const void_int *proc_ids, /**< Processor IDs */ + int processor /**< This processor ID */ ); EXODUS_EXPORT int -ex_put_partial_node_cmap(int exoid, /**< NetCDF/Exodus file ID */ - ex_entity_id map_id, /**< Nodal comm map ID */ - int64_t start_entity_num, /**< Starting position to write to */ - int64_t num_entities, /**< Number of nodes to write */ - void_int * node_ids, /**< FEM node IDs */ - void_int * proc_ids, /**< Processor IDs */ - int processor /**< This processor ID */ +ex_put_partial_node_cmap(int exoid, /**< NetCDF/Exodus file ID */ + ex_entity_id map_id, /**< Nodal comm map ID */ + int64_t start_entity_num, /**< Starting position to write to */ + int64_t num_entities, /**< Number of nodes to write */ + const void_int *node_ids, /**< FEM node IDs */ + const void_int *proc_ids, /**< Processor IDs */ + int processor /**< This processor ID */ ); EXODUS_EXPORT int ex_get_elem_cmap(int exoid, /**< NetCDF/Exodus file ID */ @@ -1169,12 +1175,12 @@ EXODUS_EXPORT int ex_get_elem_cmap(int exoid, /**< NetCDF/Exodus fil int processor /**< This processor ID */ ); -EXODUS_EXPORT int ex_put_elem_cmap(int exoid, /**< NetCDF/Exodus file ID */ - ex_entity_id map_id, /**< Elemental comm map ID */ - void_int * elem_ids, /**< Vector of element IDs */ - void_int * side_ids, /**< Vector of side IDs */ - void_int * proc_ids, /**< Vector of processor IDs */ - int processor /**< This processor ID */ +EXODUS_EXPORT int ex_put_elem_cmap(int exoid, /**< NetCDF/Exodus file ID */ + ex_entity_id map_id, /**< Elemental comm map ID */ + const void_int *elem_ids, /**< Vector of element IDs */ + const void_int *side_ids, /**< Vector of side IDs */ + const void_int *proc_ids, /**< Vector of processor IDs */ + int processor /**< This processor ID */ ); /*! @} */ diff --git a/packages/seacas/libraries/exodus/include/exodusII_int.h b/packages/seacas/libraries/exodus/include/exodusII_int.h index b7db92d76ffd..0a41d8fe8c27 100644 --- a/packages/seacas/libraries/exodus/include/exodusII_int.h +++ b/packages/seacas/libraries/exodus/include/exodusII_int.h @@ -84,7 +84,7 @@ extern "C" { /* Used to map between root (file id) and group ids when using groups */ #define EX_FILE_ID_MASK (0xffff0000) /**< Must match FILE_ID_MASK in NetCDF nc4internal.h */ -#define EX_GRP_ID_MASK (0x0000ffff) /**< Must match GRP_ID_MASK in NetCDF nc4internal.h */ +#define EX_GRP_ID_MASK (0x0000ffff) /**< Must match GRP_ID_MASK in NetCDF nc4internal.h */ void ex__reset_error_status(void); @@ -221,11 +221,11 @@ EXODUS_EXPORT int indent; * fail when they encounter a blank in a name. * */ -#define ATT_TITLE "title" /**< the database title */ +#define ATT_TITLE "title" /**< the database title */ #define ATT_API_VERSION "api_version" /**< the EXODUS api vers number */ /*! the EXODUS api vers # used for db version 2.01 and earlier */ #define ATT_API_VERSION_BLANK "api version" -#define ATT_VERSION "version" /**< the EXODUS file vers number */ +#define ATT_VERSION "version" /**< the EXODUS file vers number */ #define ATT_FILESIZE "file_size" /**< 1=large, 0=normal */ /*! word size of floating point numbers in file */ #define ATT_FLT_WORDSIZE "floating_point_word_size" @@ -240,40 +240,40 @@ EXODUS_EXPORT int indent; #define ATT_LAST_WRITTEN_TIME "last_written_time" #define DIM_NUM_ASSEMBLY "num_assembly" /**< number of assemblies */ -#define DIM_NUM_BLOB "num_blob" /**< number of blobs */ -#define DIM_NUM_NODES "num_nodes" /**< number of nodes */ -#define DIM_NUM_DIM "num_dim" /**< number of dimensions; 2- or 3-d*/ -#define DIM_NUM_EDGE "num_edge" /**< number of edges (over all blks)*/ -#define DIM_NUM_FACE "num_face" /**< number of faces (over all blks)*/ -#define DIM_NUM_ELEM "num_elem" /**< number of elements */ -#define DIM_NUM_EL_BLK "num_el_blk" /**< number of element blocks */ -#define DIM_NUM_ED_BLK "num_ed_blk" /**< number of edge blocks */ -#define DIM_NUM_FA_BLK "num_fa_blk" /**< number of face blocks */ -#define VAR_COORD "coord" /**< nodal coordinates */ -#define VAR_COORD_X "coordx" /**< X-dimension coordinate */ -#define VAR_COORD_Y "coordy" /**< Y-dimension coordinate */ -#define VAR_COORD_Z "coordz" /**< Z-dimension coordinate */ -#define VAR_NAME_COOR "coor_names" /**< names of coordinates */ -#define VAR_NAME_EL_BLK "eb_names" /**< names of element blocks */ -#define VAR_NAME_NS "ns_names" /**< names of node sets */ -#define VAR_NAME_SS "ss_names" /**< names of side sets */ -#define VAR_NAME_EM "emap_names" /**< names of element maps */ -#define VAR_NAME_EDM "edmap_names" /**< names of edge maps */ -#define VAR_NAME_FAM "famap_names" /**< names of face maps */ -#define VAR_NAME_NM "nmap_names" /**< names of node maps */ -#define VAR_NAME_ED_BLK "ed_names" /**< names of edge blocks */ -#define VAR_NAME_FA_BLK "fa_names" /**< names of face blocks */ -#define VAR_NAME_ES "es_names" /**< names of edge sets */ -#define VAR_NAME_FS "fs_names" /**< names of face sets */ -#define VAR_NAME_ELS "els_names" /**< names of element sets */ -#define VAR_STAT_EL_BLK "eb_status" /**< element block status */ -#define VAR_STAT_ECONN "econn_status" /**< element block edge status */ -#define VAR_STAT_FCONN "fconn_status" /**< element block face status */ -#define VAR_STAT_ED_BLK "ed_status" /**< edge block status */ -#define VAR_STAT_FA_BLK "fa_status" /**< face block status */ -#define VAR_ID_EL_BLK "eb_prop1" /**< element block ids props */ -#define VAR_ID_ED_BLK "ed_prop1" /**< edge block ids props */ -#define VAR_ID_FA_BLK "fa_prop1" /**< face block ids props */ +#define DIM_NUM_BLOB "num_blob" /**< number of blobs */ +#define DIM_NUM_NODES "num_nodes" /**< number of nodes */ +#define DIM_NUM_DIM "num_dim" /**< number of dimensions; 2- or 3-d*/ +#define DIM_NUM_EDGE "num_edge" /**< number of edges (over all blks)*/ +#define DIM_NUM_FACE "num_face" /**< number of faces (over all blks)*/ +#define DIM_NUM_ELEM "num_elem" /**< number of elements */ +#define DIM_NUM_EL_BLK "num_el_blk" /**< number of element blocks */ +#define DIM_NUM_ED_BLK "num_ed_blk" /**< number of edge blocks */ +#define DIM_NUM_FA_BLK "num_fa_blk" /**< number of face blocks */ +#define VAR_COORD "coord" /**< nodal coordinates */ +#define VAR_COORD_X "coordx" /**< X-dimension coordinate */ +#define VAR_COORD_Y "coordy" /**< Y-dimension coordinate */ +#define VAR_COORD_Z "coordz" /**< Z-dimension coordinate */ +#define VAR_NAME_COOR "coor_names" /**< names of coordinates */ +#define VAR_NAME_EL_BLK "eb_names" /**< names of element blocks */ +#define VAR_NAME_NS "ns_names" /**< names of node sets */ +#define VAR_NAME_SS "ss_names" /**< names of side sets */ +#define VAR_NAME_EM "emap_names" /**< names of element maps */ +#define VAR_NAME_EDM "edmap_names" /**< names of edge maps */ +#define VAR_NAME_FAM "famap_names" /**< names of face maps */ +#define VAR_NAME_NM "nmap_names" /**< names of node maps */ +#define VAR_NAME_ED_BLK "ed_names" /**< names of edge blocks */ +#define VAR_NAME_FA_BLK "fa_names" /**< names of face blocks */ +#define VAR_NAME_ES "es_names" /**< names of edge sets */ +#define VAR_NAME_FS "fs_names" /**< names of face sets */ +#define VAR_NAME_ELS "els_names" /**< names of element sets */ +#define VAR_STAT_EL_BLK "eb_status" /**< element block status */ +#define VAR_STAT_ECONN "econn_status" /**< element block edge status */ +#define VAR_STAT_FCONN "fconn_status" /**< element block face status */ +#define VAR_STAT_ED_BLK "ed_status" /**< edge block status */ +#define VAR_STAT_FA_BLK "fa_status" /**< face block status */ +#define VAR_ID_EL_BLK "eb_prop1" /**< element block ids props */ +#define VAR_ID_ED_BLK "ed_prop1" /**< edge block ids props */ +#define VAR_ID_FA_BLK "fa_prop1" /**< face block ids props */ #define DIM_NUM_ENTITY_ASSEMBLY(num) ex__catstr("num_entity_assembly", num) #define VAR_ENTITY_ASSEMBLY(num) ex__catstr("assembly_entity", num) #define DIM_NUM_VALUES_BLOB(num) ex__catstr("num_values_blob", num) @@ -367,10 +367,10 @@ EXODUS_EXPORT int indent; /*! name attached to element block, node set, side set, element map, or map properties */ #define ATT_PROP_NAME "name" -#define VAR_MAP "elem_map" /**< element order map */ +#define VAR_MAP "elem_map" /**< element order map */ #define DIM_NUM_SS "num_side_sets" /**< number of side sets */ -#define VAR_SS_STAT "ss_status" /**< side set status */ -#define VAR_SS_IDS "ss_prop1" /**< side set id properties */ +#define VAR_SS_STAT "ss_status" /**< side set status */ +#define VAR_SS_IDS "ss_prop1" /**< side set id properties */ /*! number of sides in side set num*/ #define DIM_NUM_SIDE_SS(num) ex__catstr("num_side_ss", num) /*! number of distribution factors in side set num */ @@ -384,8 +384,8 @@ EXODUS_EXPORT int indent; /*! list of the numth property for all side sets */ #define VAR_SS_PROP(num) ex__catstr("ss_prop", num) #define DIM_NUM_ES "num_edge_sets" /**< number of edge sets */ -#define VAR_ES_STAT "es_status" /**< edge set status */ -#define VAR_ES_IDS "es_prop1" /**< edge set id properties */ +#define VAR_ES_STAT "es_status" /**< edge set status */ +#define VAR_ES_IDS "es_prop1" /**< edge set id properties */ /*! number of edges in edge set num*/ #define DIM_NUM_EDGE_ES(num) ex__catstr("num_edge_es", num) /*! number of distribution factors in edge set num */ @@ -399,8 +399,8 @@ EXODUS_EXPORT int indent; /*! list of the numth property for all edge sets */ #define VAR_ES_PROP(num) ex__catstr("es_prop", num) #define DIM_NUM_FS "num_face_sets" /**< number of face sets */ -#define VAR_FS_STAT "fs_status" /**< face set status */ -#define VAR_FS_IDS "fs_prop1" /**< face set id properties */ +#define VAR_FS_STAT "fs_status" /**< face set status */ +#define VAR_FS_IDS "fs_prop1" /**< face set id properties */ /*! number of faces in side set num*/ #define DIM_NUM_FACE_FS(num) ex__catstr("num_face_fs", num) /*! number of distribution factors in face set num */ @@ -419,7 +419,7 @@ EXODUS_EXPORT int indent; /*! number of distribution factors in element set num */ #define DIM_NUM_DF_ELS(num) ex__catstr("num_df_els", num) #define VAR_ELS_STAT "els_status" /**< elem set status */ -#define VAR_ELS_IDS "els_prop1" /**< elem set id properties */ +#define VAR_ELS_IDS "els_prop1" /**< elem set id properties */ /*! list of elements in elem set num */ #define VAR_ELEM_ELS(num) ex__catstr("elem_els", num) /*! list of distribution factors in elem set num */ @@ -432,59 +432,59 @@ EXODUS_EXPORT int indent; /*! number of distribution factors in node set num */ #define DIM_NUM_DF_NS(num) ex__catstr("num_df_ns", num) #define VAR_NS_STAT "ns_status" /**< node set status */ -#define VAR_NS_IDS "ns_prop1" /**< node set id properties */ +#define VAR_NS_IDS "ns_prop1" /**< node set id properties */ /*! list of nodes in node set num */ #define VAR_NODE_NS(num) ex__catstr("node_ns", num) /*! list of distribution factors in node set num */ #define VAR_FACT_NS(num) ex__catstr("dist_fact_ns", num) /*! list of the numth property for all node sets */ #define VAR_NS_PROP(num) ex__catstr("ns_prop", num) -#define DIM_NUM_QA "num_qa_rec" /**< number of QA records */ -#define VAR_QA_TITLE "qa_records" /**< QA records */ -#define DIM_NUM_INFO "num_info" /**< number of information records */ -#define VAR_INFO "info_records" /**< information records */ -#define VAR_WHOLE_TIME "time_whole" /**< simulation times for whole time steps */ +#define DIM_NUM_QA "num_qa_rec" /**< number of QA records */ +#define VAR_QA_TITLE "qa_records" /**< QA records */ +#define DIM_NUM_INFO "num_info" /**< number of information records */ +#define VAR_INFO "info_records" /**< information records */ +#define VAR_WHOLE_TIME "time_whole" /**< simulation times for whole time steps */ #define VAR_ASSEMBLY_TAB "assembly_var_tab" /**< assembly variable truth table */ -#define VAR_BLOB_TAB "blob_var_tab" /**< blob variable truth table */ -#define VAR_ELEM_TAB "elem_var_tab" /**< element variable truth table */ -#define VAR_EBLK_TAB "edge_var_tab" /**< edge variable truth table */ -#define VAR_FBLK_TAB "face_var_tab" /**< face variable truth table */ -#define VAR_ELSET_TAB "elset_var_tab" /**< elemset variable truth table */ -#define VAR_SSET_TAB "sset_var_tab" /**< sideset variable truth table */ -#define VAR_FSET_TAB "fset_var_tab" /**< faceset variable truth table */ -#define VAR_ESET_TAB "eset_var_tab" /**< edgeset variable truth table */ -#define VAR_NSET_TAB "nset_var_tab" /**< nodeset variable truth table */ -#define DIM_NUM_GLO_VAR "num_glo_var" /**< number of global variables */ -#define VAR_NAME_GLO_VAR "name_glo_var" /**< names of global variables */ -#define VAR_GLO_VAR "vals_glo_var" /**< values of global variables*/ -#define DIM_NUM_NOD_VAR "num_nod_var" /**< number of nodal variables */ -#define VAR_NAME_NOD_VAR "name_nod_var" /**< names of nodal variables */ -#define VAR_NOD_VAR "vals_nod_var" /**< values of nodal variables \deprecated */ +#define VAR_BLOB_TAB "blob_var_tab" /**< blob variable truth table */ +#define VAR_ELEM_TAB "elem_var_tab" /**< element variable truth table */ +#define VAR_EBLK_TAB "edge_var_tab" /**< edge variable truth table */ +#define VAR_FBLK_TAB "face_var_tab" /**< face variable truth table */ +#define VAR_ELSET_TAB "elset_var_tab" /**< elemset variable truth table */ +#define VAR_SSET_TAB "sset_var_tab" /**< sideset variable truth table */ +#define VAR_FSET_TAB "fset_var_tab" /**< faceset variable truth table */ +#define VAR_ESET_TAB "eset_var_tab" /**< edgeset variable truth table */ +#define VAR_NSET_TAB "nset_var_tab" /**< nodeset variable truth table */ +#define DIM_NUM_GLO_VAR "num_glo_var" /**< number of global variables */ +#define VAR_NAME_GLO_VAR "name_glo_var" /**< names of global variables */ +#define VAR_GLO_VAR "vals_glo_var" /**< values of global variables*/ +#define DIM_NUM_NOD_VAR "num_nod_var" /**< number of nodal variables */ +#define VAR_NAME_NOD_VAR "name_nod_var" /**< names of nodal variables */ +#define VAR_NOD_VAR "vals_nod_var" /**< values of nodal variables \deprecated */ /*! values of nodal variables */ #define VAR_NOD_VAR_NEW(num) ex__catstr("vals_nod_var", num) -#define DIM_NUM_ASSEMBLY_VAR "num_assembly_var" /**< number of assembly variables */ +#define DIM_NUM_ASSEMBLY_VAR "num_assembly_var" /**< number of assembly variables */ #define VAR_NAME_ASSEMBLY_VAR "name_assembly_var" /**< names of assembly variables*/ #define VAR_ASSEMBLY_VAR(num1, num2) ex__catstr2("vals_assembly_var", num1, "assembly", num2) -#define DIM_NUM_BLOB_VAR "num_blob_var" /**< number of blob variables */ +#define DIM_NUM_BLOB_VAR "num_blob_var" /**< number of blob variables */ #define VAR_NAME_BLOB_VAR "name_blob_var" /**< names of blob variables*/ #define VAR_BLOB_VAR(num1, num2) ex__catstr2("vals_blob_var", num1, "blob", num2) -#define DIM_NUM_ELE_VAR "num_elem_var" /**< number of element variables */ +#define DIM_NUM_ELE_VAR "num_elem_var" /**< number of element variables */ #define VAR_NAME_ELE_VAR "name_elem_var" /**< names of element variables*/ /*! values of element variable num1 in element block num2 */ #define VAR_ELEM_VAR(num1, num2) ex__catstr2("vals_elem_var", num1, "eb", num2) -#define DIM_NUM_EDG_VAR "num_edge_var" /**< number of edge variables */ +#define DIM_NUM_EDG_VAR "num_edge_var" /**< number of edge variables */ #define VAR_NAME_EDG_VAR "name_edge_var" /**< names of edge variables */ /*! values of edge variable num1 in edge block num2 */ #define VAR_EDGE_VAR(num1, num2) ex__catstr2("vals_edge_var", num1, "eb", num2) -#define DIM_NUM_FAC_VAR "num_face_var" /**< number of face variables */ +#define DIM_NUM_FAC_VAR "num_face_var" /**< number of face variables */ #define VAR_NAME_FAC_VAR "name_face_var" /**< names of face variables */ /*! values of face variable num1 in face block num2 */ #define VAR_FACE_VAR(num1, num2) ex__catstr2("vals_face_var", num1, "fb", num2) -#define DIM_NUM_NSET_VAR "num_nset_var" /**< number of nodeset variables */ +#define DIM_NUM_NSET_VAR "num_nset_var" /**< number of nodeset variables */ #define VAR_NAME_NSET_VAR "name_nset_var" /**< names of nodeset variables*/ /*! values of nodeset variable num1 in nodeset num2 */ #define VAR_NS_VAR(num1, num2) ex__catstr2("vals_nset_var", num1, "ns", num2) @@ -492,15 +492,15 @@ EXODUS_EXPORT int indent; /*! values of edgeset variable num1 in edgeset num2 */ #define VAR_NAME_ESET_VAR "name_eset_var" /**< names of edgeset variables*/ #define VAR_ES_VAR(num1, num2) ex__catstr2("vals_eset_var", num1, "es", num2) -#define DIM_NUM_FSET_VAR "num_fset_var" /**< number of faceset variables */ +#define DIM_NUM_FSET_VAR "num_fset_var" /**< number of faceset variables */ #define VAR_NAME_FSET_VAR "name_fset_var" /**< names of faceset variables*/ /*! values of faceset variable num1 in faceset num2 */ #define VAR_FS_VAR(num1, num2) ex__catstr2("vals_fset_var", num1, "fs", num2) -#define DIM_NUM_SSET_VAR "num_sset_var" /**< number of sideset variables */ +#define DIM_NUM_SSET_VAR "num_sset_var" /**< number of sideset variables */ #define VAR_NAME_SSET_VAR "name_sset_var" /**< names of sideset variables*/ /*! values of sideset variable num1 in sideset num2 */ #define VAR_SS_VAR(num1, num2) ex__catstr2("vals_sset_var", num1, "ss", num2) -#define DIM_NUM_ELSET_VAR "num_elset_var" /**< number of element set variables*/ +#define DIM_NUM_ELSET_VAR "num_elset_var" /**< number of element set variables*/ #define VAR_NAME_ELSET_VAR "name_elset_var" /**< names of elemset variables*/ /*! values of elemset variable num1 in elemset num2 */ #define VAR_ELS_VAR(num1, num2) ex__catstr2("vals_elset_var", num1, "es", num2) @@ -509,30 +509,30 @@ EXODUS_EXPORT int indent; * \defgroup ReductionVariables Variables controlling storage of reduction variables *@{ */ -#define DIM_NUM_ASSEMBLY_RED_VAR "num_assembly_red_var" /**< number of assembly variables */ +#define DIM_NUM_ASSEMBLY_RED_VAR "num_assembly_red_var" /**< number of assembly variables */ #define VAR_NAME_ASSEMBLY_RED_VAR "name_assembly_red_var" /**< names of assembly variables*/ #define VAR_ASSEMBLY_RED_VAR(num) ex__catstr("vals_red_var_assembly", num) -#define DIM_NUM_BLOB_RED_VAR "num_blob_red_var" /**< number of blob variables */ +#define DIM_NUM_BLOB_RED_VAR "num_blob_red_var" /**< number of blob variables */ #define VAR_NAME_BLOB_RED_VAR "name_blob_red_var" /**< names of blob variables*/ #define VAR_BLOB_RED_VAR(num) ex__catstr("vals_red_var_blob", num) -#define DIM_NUM_ELE_RED_VAR "num_elem_red_var" /**< number of element variables */ +#define DIM_NUM_ELE_RED_VAR "num_elem_red_var" /**< number of element variables */ #define VAR_NAME_ELE_RED_VAR "name_elem_red_var" /**< names of element variables*/ /*! values of element variable num in element block num */ #define VAR_ELEM_RED_VAR(num) ex__catstr("vals_red_var_eb", num) -#define DIM_NUM_EDG_RED_VAR "num_edge_red_var" /**< number of edge variables */ +#define DIM_NUM_EDG_RED_VAR "num_edge_red_var" /**< number of edge variables */ #define VAR_NAME_EDG_RED_VAR "name_edge_red_var" /**< names of edge variables */ /*! values of edge variable num in edge block num */ #define VAR_EDGE_RED_VAR(num) ex__catstr("vals_red_var_edb", num) -#define DIM_NUM_FAC_RED_VAR "num_face_red_var" /**< number of face variables */ +#define DIM_NUM_FAC_RED_VAR "num_face_red_var" /**< number of face variables */ #define VAR_NAME_FAC_RED_VAR "name_face_red_var" /**< names of face variables */ /*! values of face variable num in face block num */ #define VAR_FACE_RED_VAR(num) ex__catstr("vals_red_var_fb", num) -#define DIM_NUM_NSET_RED_VAR "num_nset_red_var" /**< number of nodeset variables */ +#define DIM_NUM_NSET_RED_VAR "num_nset_red_var" /**< number of nodeset variables */ #define VAR_NAME_NSET_RED_VAR "name_nset_red_var" /**< names of nodeset variables*/ /*! values of nodeset variable num in nodeset num */ #define VAR_NS_RED_VAR(num) ex__catstr("vals_red_var_nset", num) @@ -542,17 +542,17 @@ EXODUS_EXPORT int indent; #define VAR_NAME_ESET_RED_VAR "name_eset_red_var" /**< names of edgeset variables*/ #define VAR_ES_RED_VAR(num) ex__catstr("vals_red_var_eset", num) -#define DIM_NUM_FSET_RED_VAR "num_fset_red_var" /**< number of faceset variables */ +#define DIM_NUM_FSET_RED_VAR "num_fset_red_var" /**< number of faceset variables */ #define VAR_NAME_FSET_RED_VAR "name_fset_red_var" /**< names of faceset variables*/ /*! values of faceset variable num in faceset num */ #define VAR_FS_RED_VAR(num) ex__catstr("vals_red_var_fset", num) -#define DIM_NUM_SSET_RED_VAR "num_sset_red_var" /**< number of sideset variables */ +#define DIM_NUM_SSET_RED_VAR "num_sset_red_var" /**< number of sideset variables */ #define VAR_NAME_SSET_RED_VAR "name_sset_red_var" /**< names of sideset variables*/ /*! values of sideset variable num in sideset num */ #define VAR_SS_RED_VAR(num) ex__catstr("vals_red_var_sset", num) -#define DIM_NUM_ELSET_RED_VAR "num_elset_red_var" /**< number of element set variables*/ +#define DIM_NUM_ELSET_RED_VAR "num_elset_red_var" /**< number of element set variables*/ #define VAR_NAME_ELSET_RED_VAR "name_elset_red_var" /**< names of elemset variables*/ /*! values of elemset variable num in elemset num */ #define VAR_ELS_RED_VAR(num) ex__catstr("vals_red_var_elset", num) @@ -572,7 +572,7 @@ EXODUS_EXPORT int indent; #define VAR_FACE_NUM_MAP "face_num_map" /**< face numbering map */ #define VAR_EDGE_NUM_MAP "edge_num_map" /**< edge numbering map */ #define VAR_NODE_NUM_MAP "node_num_map" /**< node numbering map */ -#define DIM_NUM_EM "num_elem_maps" /**< number of element maps */ +#define DIM_NUM_EM "num_elem_maps" /**< number of element maps */ /*! the numth element map */ #define VAR_ELEM_MAP(num) ex__catstr("elem_map", num) /*! list of the numth property for all element maps */ @@ -835,7 +835,7 @@ EXODUS_EXPORT int ex__get_names(int exoid, int varid, size_t num_entity, char * ex_entity_type obj_type, const char *routine); EXODUS_EXPORT int ex__put_name(int exoid, int varid, size_t index, const char *name, ex_entity_type obj_type, const char *subtype, const char *routine); -EXODUS_EXPORT int ex__put_names(int exoid, int varid, size_t num_entity, char **names, +EXODUS_EXPORT int ex__put_names(int exoid, int varid, size_t num_entity, char *const *names, ex_entity_type obj_type, const char *subtype, const char *routine); EXODUS_EXPORT void ex__trim(char *name); EXODUS_EXPORT void ex__update_max_name_length(int exoid, int length); diff --git a/packages/seacas/libraries/exodus/include/polyhedra.md b/packages/seacas/libraries/exodus/include/polyhedra.md index 6f4d1a5e4829..b33c9f8b7caa 100644 --- a/packages/seacas/libraries/exodus/include/polyhedra.md +++ b/packages/seacas/libraries/exodus/include/polyhedra.md @@ -18,32 +18,30 @@ consisting of 3 elements is shown below. The three elements have the following geometry: -* Element 1: 5 faces. - * Face 1: triangle with nodes 5, 6, 8 - * Face 2: triangle with nodes 2, 1, 4 - * Face 3: quadrilateral with nodes 6, 2, 4, 8 - * Face 4: quadrilateral with nodes 8, 4, 1, 5 - * Face 5: quadrilateral with nodes 1, 2, 6, 5 - -* Element 2: 5 faces. - * Face 6: triangle with nodes 5, 8, 7 - * Face 7: triangle with nodes 1, 3, 4 - * Face 8: quadrilateral with nodes 7, 8, 4, 3 - * Face 9: quadrilateral with nodes 7, 3, 1, 5 - * Face 4: quadrilateral with nodes 8, 4, 1, 5 (shared with element 1) - -* Element 3: 7 faces. - * Face 8: quadrilateral with nodes 7, 8, 4, 3 (shared with element 2) - * Face 10: pentagonal with nodes 8, 4, 14, 10, 12 - * Face 11: pentagonal with nodes 7, 11, 9, 13, 3 - * Face 12: quadrilateral with nodes 7, 8, 12, 11 - * Face 13: quadrilateral with nodes 11, 12, 10, 9 - * Face 14: quadrilateral with nodes 9, 10, 14, 13 - * Face 15: quadrilateral with nodes 12, 14, 4, 3 +* Element 1: 5 faces. + * Face 1: triangle with nodes 5, 6, 8 + * Face 2: triangle with nodes 2, 1, 4 + * Face 3: quadrilateral with nodes 6, 2, 4, 8 + * Face 4: quadrilateral with nodes 8, 4, 1, 5 + * Face 5: quadrilateral with nodes 1, 2, 6, 5 +* Element 2: 5 faces. + * Face 6: triangle with nodes 5, 8, 7 + * Face 7: triangle with nodes 1, 3, 4 + * Face 8: quadrilateral with nodes 7, 8, 4, 3 + * Face 9: quadrilateral with nodes 7, 3, 1, 5 + * Face 4: quadrilateral with nodes 8, 4, 1, 5 (shared with element 1) +* Element 3: 7 faces. + * Face 8: quadrilateral with nodes 7, 8, 4, 3 (shared with element 2) + * Face 10: pentagonal with nodes 8, 4, 14, 10, 12 + * Face 11: pentagonal with nodes 7, 11, 9, 13, 3 + * Face 12: quadrilateral with nodes 7, 8, 12, 11 + * Face 13: quadrilateral with nodes 11, 12, 10, 9 + * Face 14: quadrilateral with nodes 9, 10, 14, 13 + * Face 15: quadrilateral with nodes 12, 14, 4, 3 The Exodus model is created via the following calls: -* Output the initial information. Since the model contains faces and +* Output the initial information. Since the model contains faces and a face block, the "extended" version of the `ex_put_init_ext()` call must be used: ~~~~C ex_init_params par; @@ -69,9 +67,8 @@ The Exodus model is created via the following calls: ex_put_init_ext (exoid, &par); ~~~~ -* Coordinate output is normal... - -* Define the face block. +* Coordinate output is normal... +* Define the face block. ~~~~C block_name = "face_block_1"; num_face_in_block[0] = 15; @@ -85,7 +82,7 @@ The Exodus model is created via the following calls: ex_put_name(exoid, EX_FACE_BLOCK, block_id, block_name); ~~~~ -* Output the face connectivity for "face_block_1". +* Output the face connectivity for "face_block_1". The data for the face connectivity is listed above; a portion is shown below... ~~~~C connect = (int *) calloc(num_total_nodes_per_blk[0], sizeof(int)); @@ -123,7 +120,7 @@ The Exodus model is created via the following calls: ex_put_conn (exoid, EX_FACE_BLOCK, block_id, connect, NULL, NULL); ~~~~ -* Output the number of nodes per face count for "face_block_1": +* Output the number of nodes per face count for "face_block_1": ~~~~C j = 0; nnpe[ 1] = 3; /* Face 1 */ @@ -145,7 +142,7 @@ The Exodus model is created via the following calls: ex_put_entity_count_per_polyhedra(exoid, EX_FACE_BLOCK, block_id, nnpe); ~~~~ -* The face block is now fully defined; now define the nfaced element +* The face block is now fully defined; now define the nfaced element block which uses these faces. ~~~~C block_name = "nfaced_1"; @@ -170,7 +167,7 @@ The Exodus model is created via the following calls: nfaced block is the total number of faces in the connectivity for all elements in the nfaced block. -* Write the face connectivity: +* Write the face connectivity: ~~~~C /* write element-face connectivity */ connect = (int *) calloc(num_total_faces_per_blk, sizeof(int)); @@ -200,7 +197,7 @@ The Exodus model is created via the following calls: ex_put_conn (exoid, EX_ELEM_BLOCK, block_id, NULL, NULL, connect); ~~~~ -* Output the number of faces per element count for "nfaced_1": +* Output the number of faces per element count for "nfaced_1": ~~~~C nnpe[1] = 5; /* Number of faces per element 1 */ nnpe[2] = 5; /* Number of faces per element 2 */ @@ -209,15 +206,15 @@ The Exodus model is created via the following calls: ex_put_entity_count_per_polyhedra(exoid, EX_ELEM_BLOCK, block_id, nnpe); ~~~~ -* That's all; the rest of the calls are the same as normal Exodus except: +* That's all; the rest of the calls are the same as normal Exodus except: - * There is a similar `ex_get_entity_count_per_polyhedra()` function for read. + * There is a similar `ex_get_entity_count_per_polyhedra()` function for read. - * The `ex_get_block()` functions return the total number of nodes or + * The `ex_get_block()` functions return the total number of nodes or faces for all faces or element for "nfaced" and "nsided" blocks and not the number per element -* An example read/write usage is shown in the +* An example read/write usage is shown in the [testwt-nfaced.c](../test/testwt-nfaced.c) and [testrd-nfaced](../test/testrd-nfaced.c) files. -* These changes are in Exodus version v4.93 and later. +* These changes are in Exodus version v4.93 and later. diff --git a/packages/seacas/libraries/exodus/src/ex_close.c b/packages/seacas/libraries/exodus/src/ex_close.c index 1e1de7f31389..865f0b00ba85 100644 --- a/packages/seacas/libraries/exodus/src/ex_close.c +++ b/packages/seacas/libraries/exodus/src/ex_close.c @@ -50,9 +50,6 @@ int ex_close(int exoid) int status; int status1; int status2; -#if NC_HAS_HDF5 - int parent_id = 0; -#endif EX_FUNC_ENTER(); @@ -70,6 +67,7 @@ int ex_close(int exoid) /* nc_inq_grp_parent() will return NC_ENOGRP error if exoid * refers to the root group (which is what we want) */ + int parent_id = 0; if ((status = nc_inq_grp_parent(exoid, &parent_id)) != NC_ENOGRP) { snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: file id %d does not refer to root group.", exoid); ex_err_fn(exoid, __func__, errmsg, EX_NOTROOTID); diff --git a/packages/seacas/libraries/exodus/src/ex_conv.c b/packages/seacas/libraries/exodus/src/ex_conv.c index 596172c17e41..4364e9c0f800 100644 --- a/packages/seacas/libraries/exodus/src/ex_conv.c +++ b/packages/seacas/libraries/exodus/src/ex_conv.c @@ -111,8 +111,7 @@ int ex__conv_init(int exoid, int *comp_wordsize, int *io_wordsize, int file_word int int64_status, bool is_parallel, bool is_hdf5, bool is_pnetcdf, bool is_write) { char errmsg[MAX_ERR_LENGTH]; - struct ex__file_item *new_file; - int filetype = 0; + struct ex__file_item *new_file = NULL; /*! ex__conv_init() initializes the floating point conversion process. * @@ -220,6 +219,7 @@ int ex__conv_init(int exoid, int *comp_wordsize, int *io_wordsize, int file_word * 3 -- netcdf4 classic (NC_FORMAT_NETCDF4_CLASSIC -1) */ + int filetype = 0; nc_inq_format(exoid, &filetype); if (!(new_file = malloc(sizeof(struct ex__file_item)))) { @@ -279,8 +279,6 @@ int ex__conv_init(int exoid, int *comp_wordsize, int *io_wordsize, int file_word */ void ex__conv_exit(int exoid) { - - char errmsg[MAX_ERR_LENGTH]; struct ex__file_item *file = file_list; struct ex__file_item *prev = NULL; @@ -295,6 +293,7 @@ void ex__conv_exit(int exoid) } if (!file) { + char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, "Warning: failure to clear file id %d - not in list.", exoid); ex_err(__func__, errmsg, -EX_BADFILEID); EX_FUNC_VOID(); @@ -389,10 +388,6 @@ int ex_set_int64_status(int exoid, int mode) | #EX_ALL_INT64_API | (#EX_MAPS_INT64_API \| #EX_IDS_INT64_API \| #EX_BULK_INT64_API \| #EX_INQ_INT64_API) | */ - - int api_mode = 0; - int db_mode = 0; - EX_FUNC_ENTER(); struct ex__file_item *file = ex__find_file_item(exoid); @@ -404,8 +399,8 @@ int ex_set_int64_status(int exoid, int mode) } /* Strip of all non-INT64_API values */ - api_mode = mode & EX_ALL_INT64_API; - db_mode = file->int64_status & EX_ALL_INT64_DB; + int api_mode = mode & EX_ALL_INT64_API; + int db_mode = file->int64_status & EX_ALL_INT64_DB; file->int64_status = api_mode | db_mode; EX_FUNC_LEAVE(file->int64_status); @@ -537,8 +532,7 @@ int ex__is_parallel(int exoid) int ex_set_parallel(int exoid, int is_parallel) { EX_FUNC_ENTER(); - int old_value = 0; - struct ex__file_item *file = ex__find_file_item(exoid); + struct ex__file_item *file = ex__find_file_item(exoid); if (!file) { char errmsg[MAX_ERR_LENGTH]; @@ -547,7 +541,7 @@ int ex_set_parallel(int exoid, int is_parallel) EX_FUNC_LEAVE(EX_FATAL); } - old_value = file->is_parallel; + int old_value = file->is_parallel; file->is_parallel = is_parallel; /* Stored as 1 for parallel, 0 for serial or file-per-processor */ EX_FUNC_LEAVE(old_value); diff --git a/packages/seacas/libraries/exodus/src/ex_copy.c b/packages/seacas/libraries/exodus/src/ex_copy.c index 9b691962dc5a..d1783b6e40c9 100644 --- a/packages/seacas/libraries/exodus/src/ex_copy.c +++ b/packages/seacas/libraries/exodus/src/ex_copy.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -31,12 +31,6 @@ } /*! \cond INTERNAL */ -struct ncdim -{ /* dimension */ - char name[MAX_VAR_NAME_LENGTH]; - size_t size; -}; - struct ncvar { /* variable */ char name[MAX_VAR_NAME_LENGTH]; @@ -48,11 +42,9 @@ struct ncvar struct ncatt { /* attribute */ - int var; char name[MAX_VAR_NAME_LENGTH]; nc_type type; size_t len; - void * val; }; static size_t type_size(nc_type type); @@ -89,10 +81,6 @@ static int is_non_mesh_variable(const char *var_name) /*! \cond INTERNAL */ static int ex_copy_internal(int in_exoid, int out_exoid, int mesh_only) { - int status; - int in_large; - char errmsg[MAX_ERR_LENGTH]; - EX_FUNC_ENTER(); if (ex__check_valid_file_id(in_exoid, __func__) != EX_NOERR) { EX_FUNC_LEAVE(EX_FATAL); @@ -105,13 +93,14 @@ static int ex_copy_internal(int in_exoid, int out_exoid, int mesh_only) * Get exodus_large_model setting on both input and output * databases so know how to handle coordinates. */ - in_large = ex_large_model(in_exoid); + int in_large = ex_large_model(in_exoid); /* * Get integer sizes for both input and output databases. * Currently they should both match or there will be an error. */ if (ex_int64_status(in_exoid) != ex_int64_status(out_exoid)) { + char errmsg[MAX_ERR_LENGTH]; snprintf_nowarn(errmsg, MAX_ERR_LENGTH, "ERROR: integer sizes do not match for input and output databases."); ex_err_fn(in_exoid, __func__, errmsg, EX_WRONGFILETYPE); @@ -131,7 +120,7 @@ static int ex_copy_internal(int in_exoid, int out_exoid, int mesh_only) EXCHECK(cpy_variables(in_exoid, out_exoid, in_large, mesh_only)); /* take the output file out of define mode */ - if ((status = ex__leavedef(out_exoid, __func__)) != NC_NOERR) { + if (ex__leavedef(out_exoid, __func__) != NC_NOERR) { EX_FUNC_LEAVE(EX_FATAL); } @@ -186,29 +175,28 @@ int ex_copy_transient(int in_exoid, int out_exoid) int cpy_variable_data(int in_exoid, int out_exoid, int in_large, int mesh_only) { int nvars; /* number of variables */ - int varid; /* variable id */ - int is_filtered = 0; - struct ncvar var; /* variable */ + struct ncvar var; /* variable */ EXCHECKI(nc_inq(in_exoid, NULL, &nvars, NULL, NULL)); - for (varid = 0; varid < nvars; varid++) { + for (int varid = 0; varid < nvars; varid++) { + bool is_filtered; EXCHECKI(nc_inq_var(in_exoid, varid, var.name, &var.type, &var.ndims, var.dims, &var.natts)); if ((strcmp(var.name, VAR_QA_TITLE) == 0) || (strcmp(var.name, VAR_INFO) == 0)) { - is_filtered = 1; + is_filtered = true; } else if (is_truth_table_variable(var.name)) { - is_filtered = 1; + is_filtered = true; } else if (mesh_only == 1 && (is_non_mesh_variable(var.name) || (strcmp(var.name, VAR_WHOLE_TIME) == 0))) { - is_filtered = 1; + is_filtered = true; } else if (mesh_only == 0 && (!is_non_mesh_variable(var.name) && strcmp(var.name, VAR_WHOLE_TIME) != 0)) { - is_filtered = 1; + is_filtered = true; } else { - is_filtered = 0; + is_filtered = false; } if (!is_filtered) { @@ -226,31 +214,30 @@ int cpy_variable_data(int in_exoid, int out_exoid, int in_large, int mesh_only) /*! \cond INTERNAL */ int cpy_variables(int in_exoid, int out_exoid, int in_large, int mesh_only) { - int nvars; /* number of variables */ - int recdimid; /* id of unlimited dimension */ - int varid; /* variable id */ - int var_out_id; /* variable id */ - int is_filtered = 0; - struct ncvar var; /* variable */ + int nvars; /* number of variables */ + int recdimid; /* id of unlimited dimension */ + struct ncvar var; /* variable */ EXCHECKI(nc_inq(in_exoid, NULL, &nvars, NULL, &recdimid)); - for (varid = 0; varid < nvars; varid++) { + for (int varid = 0; varid < nvars; varid++) { EXCHECKI(nc_inq_var(in_exoid, varid, var.name, &var.type, &var.ndims, var.dims, &var.natts)); + bool is_filtered; if ((strcmp(var.name, VAR_QA_TITLE) == 0) || (strcmp(var.name, VAR_INFO) == 0)) { - is_filtered = 1; + is_filtered = true; } else if (is_truth_table_variable(var.name)) { - is_filtered = 1; + is_filtered = true; } else if (mesh_only == 1 && is_non_mesh_variable(var.name)) { - is_filtered = 1; + is_filtered = true; } else { - is_filtered = 0; + is_filtered = false; } + int var_out_id; /* variable id */ if (!is_filtered) { if (strncmp(var.name, VAR_COORD, 5) == 0) { var_out_id = cpy_coord_def(in_exoid, out_exoid, recdimid, var.name, in_large); @@ -274,10 +261,8 @@ int cpy_dimension(int in_exoid, int out_exoid, int mesh_only) int recdimid; /* id of unlimited dimension */ int dimid; /* dimension id */ int dim_out_id; /* dimension id */ - int is_filtered = 0; char dim_nm[NC_MAX_NAME]; size_t dim_sz; - char errmsg[MAX_ERR_LENGTH]; EXCHECKI(nc_inq(in_exoid, &ndims, NULL, NULL, &recdimid)); for (dimid = 0; dimid < ndims; dimid++) { @@ -287,10 +272,11 @@ int cpy_dimension(int in_exoid, int out_exoid, int mesh_only) /* If the dimension isn't one we specifically don't want * to copy (ie, number of QA or INFO records) and it * hasn't been defined, copy it */ + bool is_filtered; if (strcmp(dim_nm, DIM_NUM_QA) == 0 || strcmp(dim_nm, DIM_NUM_INFO) == 0 || strcmp(dim_nm, DIM_N4) == 0 || strcmp(dim_nm, DIM_STR) == 0 || strcmp(dim_nm, DIM_LIN) == 0) { - is_filtered = 1; + is_filtered = true; } else if (mesh_only == 1 && ((strcmp(dim_nm, DIM_NUM_NOD_VAR) == 0) || (strcmp(dim_nm, DIM_NUM_EDG_VAR) == 0) || @@ -300,10 +286,10 @@ int cpy_dimension(int in_exoid, int out_exoid, int mesh_only) (strcmp(dim_nm, DIM_NUM_ELSET_VAR) == 0) || (strcmp(dim_nm, DIM_NUM_GLO_VAR) == 0) || (strcmp(dim_nm, DIM_NUM_ASSEMBLY_VAR) == 0) || (strcmp(dim_nm, DIM_NUM_BLOB_VAR) == 0) || (strstr(dim_nm, "_red_var") != NULL))) { - is_filtered = 1; + is_filtered = true; } else { - is_filtered = 0; + is_filtered = false; } if (!is_filtered) { @@ -318,6 +304,7 @@ int cpy_dimension(int in_exoid, int out_exoid, int mesh_only) status = nc_def_dim(out_exoid, dim_nm, NC_UNLIMITED, &dim_out_id); } if (status != NC_NOERR) { + char errmsg[MAX_ERR_LENGTH]; snprintf_nowarn(errmsg, MAX_ERR_LENGTH, "ERROR: failed to define %s dimension in file id %d", dim_nm, out_exoid); ex_err_fn(out_exoid, __func__, errmsg, status); @@ -343,6 +330,7 @@ int cpy_dimension(int in_exoid, int out_exoid, int mesh_only) /* Not found; set to default value of 32+1. */ if ((status = nc_def_dim(out_exoid, DIM_STR_NAME, 33, &dim_out_id)) != NC_NOERR) { + char errmsg[MAX_ERR_LENGTH]; snprintf_nowarn(errmsg, MAX_ERR_LENGTH, "ERROR: failed to define string name dimension in file id %d", out_exoid); ex_err_fn(out_exoid, __func__, errmsg, status); @@ -358,13 +346,12 @@ int cpy_global_att(int in_exoid, int out_exoid) { int status; int ngatts; - size_t i; struct ncatt att; /* attribute */ EXCHECKI(nc_inq(in_exoid, NULL, NULL, &ngatts, NULL)); /* copy global attributes */ - for (i = 0; i < (size_t)ngatts; i++) { + for (size_t i = 0; i < (size_t)ngatts; i++) { EXCHECKI(nc_inq_attname(in_exoid, NC_GLOBAL, i, att.name)); @@ -412,7 +399,6 @@ int cpy_att(int in_id, int out_id, int var_in_id, int var_out_id) then the global attributes are copied. Otherwise the variable's attributes are copied. */ - int idx; int nbr_att; if (var_in_id == NC_GLOBAL) { @@ -423,7 +409,7 @@ int cpy_att(int in_id, int out_id, int var_in_id, int var_out_id) } /* Get the attributes names, types, lengths, and values */ - for (idx = 0; idx < nbr_att; idx++) { + for (int idx = 0; idx < nbr_att; idx++) { char att_nm[MAX_VAR_NAME_LENGTH]; EXCHECKI(nc_inq_attname(in_id, var_in_id, idx, att_nm)); @@ -436,14 +422,6 @@ int cpy_att(int in_id, int out_id, int var_in_id, int var_out_id) /*! \internal */ int cpy_coord_def(int in_id, int out_id, int rec_dim_id, char *var_nm, int in_large) { - const char *routine = NULL; - size_t spatial_dim; - int nbr_dim; - int temp; - - int dim_out_id[2]; - int var_out_id = -1; - /* Handle easiest situation first: in_large matches out_large (1) */ if (in_large == 1) { return cpy_var_def(in_id, out_id, rec_dim_id, var_nm); /* OK */ @@ -453,6 +431,9 @@ int cpy_coord_def(int in_id, int out_id, int rec_dim_id, char *var_nm, int in_la option is that in_large == 0 and out_large == 1. Also will need the spatial dimension, so get that now. */ + int temp; + size_t spatial_dim; + const char *routine = NULL; ex__get_dimension(in_id, DIM_NUM_DIM, "dimension", &spatial_dim, &temp, routine); /* output file will have coordx, coordy, coordz (if 3d). See if @@ -471,12 +452,14 @@ int cpy_coord_def(int in_id, int out_id, int rec_dim_id, char *var_nm, int in_la } /* Get dimid of the num_nodes dimension in output file... */ + int dim_out_id[2]; EXCHECKI(nc_inq_dimid(out_id, DIM_NUM_NODES, &dim_out_id[0])); /* Define the variables in the output file */ /* Define according to the EXODUS file's IO_word_size */ - nbr_dim = 1; + int nbr_dim = 1; + int var_out_id = -1; EXCHECKI(nc_def_var(out_id, VAR_COORD_X, nc_flt_code(out_id), nbr_dim, dim_out_id, &var_out_id)); ex__compress_variable(out_id, var_out_id, 2); if (spatial_dim > 1) { @@ -503,7 +486,6 @@ int cpy_var_def(int in_id, int out_id, int rec_dim_id, char *var_nm) int status; int dim_in_id[NC_MAX_VAR_DIMS]; int dim_out_id[NC_MAX_VAR_DIMS]; - int idx; int nbr_dim; int var_in_id; int var_out_id; @@ -531,7 +513,7 @@ int cpy_var_def(int in_id, int out_id, int rec_dim_id, char *var_nm) EXCHECKI(nc_inq_vardimid(in_id, var_in_id, dim_in_id)); /* Get the dimension sizes and names */ - for (idx = 0; idx < nbr_dim; idx++) { + for (int idx = 0; idx < nbr_dim; idx++) { char dim_nm[NC_MAX_NAME]; size_t dim_sz; @@ -577,7 +559,6 @@ int cpy_var_val(int in_id, int out_id, char *var_nm) int dim_id_in[NC_MAX_VAR_DIMS]; int dim_id_out[NC_MAX_VAR_DIMS]; - int idx; int nbr_dim; int var_in_id; int var_out_id; @@ -604,7 +585,7 @@ int cpy_var_val(int in_id, int out_id, char *var_nm) EXCHECKF(nc_inq_vardimid(out_id, var_out_id, dim_id_out)); /* Get the dimension sizes and names from the input file */ - for (idx = 0; idx < nbr_dim; idx++) { + for (int idx = 0; idx < nbr_dim; idx++) { /* NB: For the unlimited dimension, ncdiminq() returns the maximum value used so far in writing data for that dimension. Thus if you read the dimension sizes from the output file, then @@ -720,15 +701,6 @@ int cpy_coord_val(int in_id, int out_id, char *var_nm, int in_large) * to an output netCDF file. */ - const char *routine = NULL; - int temp; - size_t i; - size_t spatial_dim, num_nodes; - size_t start[2], count[2]; - nc_type var_type_in, var_type_out; - - void *void_ptr = NULL; - /* Handle easiest situation first: in_large matches out_large (1) */ if (in_large == 1) { return cpy_var_val(in_id, out_id, var_nm); /* OK */ @@ -737,6 +709,9 @@ int cpy_coord_val(int in_id, int out_id, char *var_nm, int in_large) /* At this point, know that in_large == 0, so will need to copy a vector to multiple scalars. Also will need a couple dimensions, so get them now.*/ + const char *routine = NULL; + int temp; + size_t spatial_dim, num_nodes; ex__get_dimension(in_id, DIM_NUM_DIM, "dimension", &spatial_dim, &temp, routine); ex__get_dimension(in_id, DIM_NUM_NODES, "nodes", &num_nodes, &temp, routine); @@ -753,15 +728,18 @@ int cpy_coord_val(int in_id, int out_id, char *var_nm, int in_large) EXCHECKI(nc_inq_varid(out_id, VAR_COORD_Z, &var_out_id[2])); } + nc_type var_type_in, var_type_out; EXCHECKI(nc_inq_vartype(in_id, var_in_id, &var_type_in)); EXCHECKI(nc_inq_vartype(out_id, var_out_id[0], &var_type_out)); + void *void_ptr = NULL; if (num_nodes > 0) { void_ptr = malloc(num_nodes * type_size(var_type_in)); } /* Copy each component of the variable... */ - for (i = 0; i < spatial_dim; i++) { + size_t start[2], count[2]; + for (size_t i = 0; i < spatial_dim; i++) { start[0] = i; start[1] = 0; count[0] = 1; @@ -807,10 +785,9 @@ void update_structs(int out_exoid) /*! \internal */ void update_internal_structs(int out_exoid, ex_inquiry inqcode, struct ex__list_item **ctr_list) { - int64_t i; int64_t number = ex_inquire_int(out_exoid, inqcode); if (number > 0) { - for (i = 0; i < number; i++) { + for (int64_t i = 0; i < number; i++) { ex__inc_file_item(out_exoid, ctr_list); } } diff --git a/packages/seacas/libraries/exodus/src/ex_create.c b/packages/seacas/libraries/exodus/src/ex_create.c index 7589adc7a2a3..59b2ff055b55 100644 --- a/packages/seacas/libraries/exodus/src/ex_create.c +++ b/packages/seacas/libraries/exodus/src/ex_create.c @@ -152,6 +152,11 @@ int ex_create_int(const char *path, int cmode, int *comp_ws, int *io_ws, int run EX_FUNC_LEAVE(EX_FATAL); } +#if defined NC_NOATTCREORD + /* Disable attribute creation order tracking if available... */ + nc_mode |= NC_NOATTCREORD; +#endif + if ((status = nc_create(path, nc_mode, &exoid)) != NC_NOERR) { #if NC_HAS_HDF5 snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: file create failed for %s", path); diff --git a/packages/seacas/libraries/exodus/src/ex_create_par.c b/packages/seacas/libraries/exodus/src/ex_create_par.c index b24cbde6b6b0..527a9750ad5d 100644 --- a/packages/seacas/libraries/exodus/src/ex_create_par.c +++ b/packages/seacas/libraries/exodus/src/ex_create_par.c @@ -168,6 +168,11 @@ int ex_create_par_int(const char *path, int cmode, int *comp_ws, int *io_ws, MPI nc_mode = ex__handle_mode(my_mode, is_parallel, run_version); +#if defined NC_NOATTCREORD + /* Disable attribute creation order tracking if available... */ + nc_mode |= NC_NOATTCREORD; +#endif + if ((status = nc_create_par(path, nc_mode, comm, info, &exoid)) != NC_NOERR) { if (my_mode & EX_NETCDF4) { #if NC_HAS_PARALLEL4 diff --git a/packages/seacas/libraries/exodus/src/ex_get_attr_names.c b/packages/seacas/libraries/exodus/src/ex_get_attr_names.c index 3d11a6c73b93..ff2ceddc407f 100644 --- a/packages/seacas/libraries/exodus/src/ex_get_attr_names.c +++ b/packages/seacas/libraries/exodus/src/ex_get_attr_names.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -34,7 +34,6 @@ int ex_get_attr_names(int exoid, ex_entity_type obj_type, ex_entity_id obj_id, c { int status; int varid, numattrdim, obj_id_ndx; - size_t num_attr, i; char errmsg[MAX_ERR_LENGTH]; const char *dnumobjatt; const char *vattrbname; @@ -121,6 +120,7 @@ int ex_get_attr_names(int exoid, ex_entity_type obj_type, ex_entity_id obj_id, c EX_FUNC_LEAVE(EX_WARN); /* no attributes for this object */ } + size_t num_attr = 0; if ((status = nc_inq_dimlen(exoid, numattrdim, &num_attr)) != NC_NOERR) { snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to get number of attributes for %s %" PRId64 " in file id %d", @@ -147,7 +147,7 @@ int ex_get_attr_names(int exoid, ex_entity_type obj_type, ex_entity_id obj_id, c /* Names variable does not exist on the database; probably since this is an * older version of the database. Return an empty array... */ - for (i = 0; i < num_attr; i++) { + for (int i = 0; i < num_attr; i++) { names[i][0] = '\0'; } } diff --git a/packages/seacas/libraries/exodus/src/ex_get_cmap_params.c b/packages/seacas/libraries/exodus/src/ex_get_cmap_params.c index 4466fa2f1762..9646d7b73d6e 100644 --- a/packages/seacas/libraries/exodus/src/ex_get_cmap_params.c +++ b/packages/seacas/libraries/exodus/src/ex_get_cmap_params.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -38,10 +38,10 @@ int ex_get_cmap_params(int exoid, void_int *node_cmap_ids, void_int *node_cmap_node_cnts, void_int *elem_cmap_ids, void_int *elem_cmap_elem_cnts, int processor) { - size_t cnt, num_n_comm_maps, num_e_comm_maps, start[1], count[1]; - int64_t cmap_info_idx[2], cmap_data_idx[2]; - int nmstat; + size_t start[1], count[1]; int status, map_idx, varid, dimid; + int nmstat; + int64_t cmap_data_idx[2]; char errmsg[MAX_ERR_LENGTH]; /*-----------------------------Execution begins-----------------------------*/ @@ -58,6 +58,7 @@ int ex_get_cmap_params(int exoid, void_int *node_cmap_ids, void_int *node_cmap_n /*****************************************************************************/ /* get the cmap information variables index */ + int64_t cmap_info_idx[2]; if (ex_get_idx(exoid, VAR_N_COMM_INFO_IDX, cmap_info_idx, processor) == -1) { snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to find index variable, \"%s\", in file ID %d", VAR_N_COMM_INFO_IDX, exoid); @@ -82,7 +83,7 @@ int ex_get_cmap_params(int exoid, void_int *node_cmap_ids, void_int *node_cmap_n cmap_info_idx[1] = count[0]; } /* End "if (cmap_info_idx[1] == -1) */ - num_n_comm_maps = cmap_info_idx[1] - cmap_info_idx[0]; + size_t num_n_comm_maps = cmap_info_idx[1] - cmap_info_idx[0]; if (num_n_comm_maps > 0) { count[0] = num_n_comm_maps; @@ -124,7 +125,7 @@ int ex_get_cmap_params(int exoid, void_int *node_cmap_ids, void_int *node_cmap_n if (node_cmap_node_cnts != NULL) { /* Get the node counts in each of the nodal communication maps */ - for (cnt = 0; cnt < num_n_comm_maps; cnt++) { + for (size_t cnt = 0; cnt < num_n_comm_maps; cnt++) { int64_t cmap_id; if (ex_int64_status(exoid) & EX_IDS_INT64_API) { cmap_id = ((int64_t *)node_cmap_ids)[cnt]; @@ -242,7 +243,7 @@ int ex_get_cmap_params(int exoid, void_int *node_cmap_ids, void_int *node_cmap_n cmap_info_idx[1] = count[0]; } /* End "if (cmap_info_idx[1] == -1) */ - num_e_comm_maps = cmap_info_idx[1] - cmap_info_idx[0]; + size_t num_e_comm_maps = cmap_info_idx[1] - cmap_info_idx[0]; if (num_e_comm_maps > 0) { count[0] = num_e_comm_maps; @@ -285,7 +286,7 @@ int ex_get_cmap_params(int exoid, void_int *node_cmap_ids, void_int *node_cmap_n * Get the element counts in each of the elemental * communication maps */ - for (cnt = 0; cnt < num_e_comm_maps; cnt++) { + for (size_t cnt = 0; cnt < num_e_comm_maps; cnt++) { int64_t cmap_id; if (ex_int64_status(exoid) & EX_IDS_INT64_API) { cmap_id = ((int64_t *)elem_cmap_ids)[cnt]; diff --git a/packages/seacas/libraries/exodus/src/ex_get_coord.c b/packages/seacas/libraries/exodus/src/ex_get_coord.c index a3a91f7f80bb..e36abc6213e1 100644 --- a/packages/seacas/libraries/exodus/src/ex_get_coord.c +++ b/packages/seacas/libraries/exodus/src/ex_get_coord.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -75,7 +75,7 @@ int ex_get_coord(int exoid, void *x_coor, void *y_coor, void *z_coor) int coordidx, coordidy, coordidz; int numnoddim, ndimdim; - size_t num_nod, num_dim, start[2], count[2], i; + size_t num_nod, num_dim; char errmsg[MAX_ERR_LENGTH]; EX_FUNC_ENTER(); @@ -111,10 +111,12 @@ int ex_get_coord(int exoid, void *x_coor, void *y_coor, void *z_coor) EX_FUNC_LEAVE(EX_FATAL); } - for (i = 0; i < num_dim; i++) { - char *which = NULL; - start[0] = i; - start[1] = 0; + for (size_t i = 0; i < num_dim; i++) { + char * which = NULL; + size_t start[2]; + size_t count[2]; + start[0] = i; + start[1] = 0; count[0] = 1; count[1] = num_nod; @@ -188,7 +190,7 @@ int ex_get_coord(int exoid, void *x_coor, void *y_coor, void *z_coor) } /* write out the coordinates */ - for (i = 0; i < num_dim; i++) { + for (size_t i = 0; i < num_dim; i++) { void *coor = NULL; char *which = NULL; diff --git a/packages/seacas/libraries/exodus/src/ex_get_map.c b/packages/seacas/libraries/exodus/src/ex_get_map.c index 96ebafb6b87d..6badb4bc0dc2 100644 --- a/packages/seacas/libraries/exodus/src/ex_get_map.c +++ b/packages/seacas/libraries/exodus/src/ex_get_map.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -43,9 +43,7 @@ error = ex_get_map(exoid, elem_map); int ex_get_map(int exoid, void_int *elem_map) { - int numelemdim, mapid, status; - size_t num_elem, i; - char errmsg[MAX_ERR_LENGTH]; + int status; EX_FUNC_ENTER(); if (ex__check_valid_file_id(exoid, __func__) == EX_FATAL) { @@ -55,28 +53,32 @@ int ex_get_map(int exoid, void_int *elem_map) /* inquire id's of previously defined dimensions and variables */ /* See if file contains any elements...*/ + int numelemdim; if ((status = nc_inq_dimid(exoid, DIM_NUM_ELEM, &numelemdim)) != NC_NOERR) { EX_FUNC_LEAVE(EX_NOERR); } + size_t num_elem; if ((status = nc_inq_dimlen(exoid, numelemdim, &num_elem)) != NC_NOERR) { + char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to get number of elements in file id %d", exoid); ex_err_fn(exoid, __func__, errmsg, status); EX_FUNC_LEAVE(EX_FATAL); } + int mapid; if (nc_inq_varid(exoid, VAR_MAP, &mapid) != NC_NOERR) { /* generate default map of 1..n, where n is num_elem */ if (ex_int64_status(exoid) & EX_MAPS_INT64_API) { int64_t *lmap = (int64_t *)elem_map; - for (i = 0; i < num_elem; i++) { + for (size_t i = 0; i < num_elem; i++) { lmap[i] = i + 1; } } else { int *lmap = (int *)elem_map; - for (i = 0; i < num_elem; i++) { + for (size_t i = 0; i < num_elem; i++) { lmap[i] = i + 1; } } @@ -93,6 +95,7 @@ int ex_get_map(int exoid, void_int *elem_map) } if (status != NC_NOERR) { + char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to get element order map in file id %d", exoid); ex_err_fn(exoid, __func__, errmsg, status); EX_FUNC_LEAVE(EX_FATAL); diff --git a/packages/seacas/libraries/exodus/src/ex_int_get_block_param.c b/packages/seacas/libraries/exodus/src/ex_int_get_block_param.c index 7b33bc4b4108..1592ed34dfc3 100644 --- a/packages/seacas/libraries/exodus/src/ex_int_get_block_param.c +++ b/packages/seacas/libraries/exodus/src/ex_int_get_block_param.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -23,9 +23,6 @@ static int el_node_count_error(int exoid, struct ex__elem_blk_parm elem_blk_parm int ex__get_block_param(int exoid, ex_entity_id id, int ndim, struct ex__elem_blk_parm *elem_blk_parm) { - size_t m; - char errmsg[MAX_ERR_LENGTH]; - EX_FUNC_ENTER(); ex_block block; @@ -38,6 +35,7 @@ int ex__get_block_param(int exoid, ex_entity_id id, int ndim, /* read in an element block parameter */ if ((ex_get_block_param(exoid, &block)) != EX_NOERR) { + char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to get element block %" PRId64 " parameters in file id %d", block.id, exoid); @@ -50,6 +48,7 @@ int ex__get_block_param(int exoid, ex_entity_id id, int ndim, elem_blk_parm->num_attr = block.num_attribute; elem_blk_parm->elem_blk_id = block.id; + size_t m; for (m = 0; m < strlen(block.topology); m++) { elem_blk_parm->elem_type[m] = toupper(block.topology[m]); } diff --git a/packages/seacas/libraries/exodus/src/ex_ne_util.c b/packages/seacas/libraries/exodus/src/ex_ne_util.c index 8854d626fa28..c7bf00e031a8 100644 --- a/packages/seacas/libraries/exodus/src/ex_ne_util.c +++ b/packages/seacas/libraries/exodus/src/ex_ne_util.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -40,8 +40,7 @@ char *ne_ret_string; int ne__id_lkup(int exoid, const char *ne_var_name, int64_t *idx, ex_entity_id ne_var_id) { int status; - int varid, ndims, dimid[1], ret = -1; - nc_type var_type; + int varid, ret = -1; size_t length, start[1]; int64_t my_index, begin, end; long long id_val; @@ -58,6 +57,9 @@ int ne__id_lkup(int exoid, const char *ne_var_name, int64_t *idx, ex_entity_id n /* check if I need the length for this variable */ if (idx[1] == -1) { /* Get the dimension IDs for this variable */ + int ndims; + int dimid[1]; + nc_type var_type; if ((status = nc_inq_var(exoid, varid, (char *)0, &var_type, &ndims, dimid, (int *)0)) != NC_NOERR) { snprintf(errmsg, MAX_ERR_LENGTH, @@ -119,7 +121,7 @@ int ex__get_file_type(int exoid, char *ftype) EX_FUNC_ENTER(); - if ((status = nc_inq_varid(exoid, VAR_FILE_TYPE, &varid)) != NC_NOERR) { + if (nc_inq_varid(exoid, VAR_FILE_TYPE, &varid) != NC_NOERR) { /* If no file type is found, assume parallel */ ftype[0] = 'p'; diff --git a/packages/seacas/libraries/exodus/src/ex_put_all_var_param.c b/packages/seacas/libraries/exodus/src/ex_put_all_var_param.c index 5879804796f7..00a3e1919459 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_all_var_param.c +++ b/packages/seacas/libraries/exodus/src/ex_put_all_var_param.c @@ -42,8 +42,8 @@ * \param *sset_var_tab sideset variable truth table array */ -int ex_put_all_var_param(int exoid, int num_g, int num_n, int num_e, int *elem_var_tab, int num_m, - int *nset_var_tab, int num_s, int *sset_var_tab) +int ex_put_all_var_param(int exoid, int num_g, int num_n, int num_e, const int *elem_var_tab, + int num_m, const int *nset_var_tab, int num_s, const int *sset_var_tab) { ex_var_params vparam; @@ -54,15 +54,15 @@ int ex_put_all_var_param(int exoid, int num_g, int num_n, int num_e, int *elem_v vparam.num_face = 0; vparam.face_var_tab = 0; vparam.num_elem = num_e; - vparam.elem_var_tab = elem_var_tab; + vparam.elem_var_tab = (int *)elem_var_tab; vparam.num_nset = num_m; - vparam.nset_var_tab = nset_var_tab; + vparam.nset_var_tab = (int *)nset_var_tab; vparam.num_eset = 0; vparam.eset_var_tab = 0; vparam.num_fset = 0; vparam.fset_var_tab = 0; vparam.num_sset = num_s; - vparam.sset_var_tab = sset_var_tab; + vparam.sset_var_tab = (int *)sset_var_tab; vparam.num_elset = 0; vparam.elset_var_tab = 0; diff --git a/packages/seacas/libraries/exodus/src/ex_put_attribute.c b/packages/seacas/libraries/exodus/src/ex_put_attribute.c index 8ba6e29ecca2..0fb1680d2220 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_attribute.c +++ b/packages/seacas/libraries/exodus/src/ex_put_attribute.c @@ -111,7 +111,7 @@ static int ex__get_varid(int exoid, ex_entity_type obj_type, ex_entity_id id) /* define and output a double attribute */ int ex_put_double_attribute(int exoid, ex_entity_type obj_type, ex_entity_id id, - const char *atr_name, int num_values, double *values) + const char *atr_name, int num_values, const double *values) { int status; char errmsg[MAX_ERR_LENGTH]; @@ -155,7 +155,7 @@ int ex_put_double_attribute(int exoid, ex_entity_type obj_type, ex_entity_id id, /* define and output an integer attribute */ int ex_put_integer_attribute(int exoid, ex_entity_type obj_type, ex_entity_id id, - const char *atr_name, int num_values, void_int *values) + const char *atr_name, int num_values, const void_int *values) { int status; char errmsg[MAX_ERR_LENGTH]; @@ -273,7 +273,7 @@ int ex_put_attribute(int exoid, ex_attribute attribute) } /*! Define and output the specified attributes. */ -int ex_put_attributes(int exoid, size_t attr_count, ex_attribute *attr) +int ex_put_attributes(int exoid, size_t attr_count, const ex_attribute *attr) { for (size_t i = 0; i < attr_count; i++) { int status = ex_put_attribute(exoid, attr[i]); diff --git a/packages/seacas/libraries/exodus/src/ex_put_cmap_params.c b/packages/seacas/libraries/exodus/src/ex_put_cmap_params.c index 570751179060..a3993b7b94f3 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_cmap_params.c +++ b/packages/seacas/libraries/exodus/src/ex_put_cmap_params.c @@ -35,8 +35,9 @@ #include // for ex_err, etc #include // for EX_FATAL, ex__leavedef, etc -int ex_put_cmap_params(int exoid, void_int *node_cmap_ids, void_int *node_cmap_node_cnts, - void_int *elem_cmap_ids, void_int *elem_cmap_elem_cnts, int64_t processor) +int ex_put_cmap_params(int exoid, const void_int *node_cmap_ids, + const void_int *node_cmap_node_cnts, const void_int *elem_cmap_ids, + const void_int *elem_cmap_elem_cnts, int64_t processor) { size_t num_n_comm_maps, num_e_comm_maps; size_t ncnt_cmap, ecnt_cmap; diff --git a/packages/seacas/libraries/exodus/src/ex_put_cmap_params_cc.c b/packages/seacas/libraries/exodus/src/ex_put_cmap_params_cc.c index 607af5d6fccc..9e4363cfbe5f 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_cmap_params_cc.c +++ b/packages/seacas/libraries/exodus/src/ex_put_cmap_params_cc.c @@ -35,9 +35,10 @@ #include // for ex_err, etc #include // for EX_FATAL, ex__leavedef, etc -int ex_put_cmap_params_cc(int exoid, void_int *node_cmap_ids, void_int *node_cmap_node_cnts, - void_int *node_proc_ptrs, void_int *elem_cmap_ids, - void_int *elem_cmap_elem_cnts, void_int *elem_proc_ptrs) +int ex_put_cmap_params_cc(int exoid, const void_int *node_cmap_ids, + const void_int *node_cmap_node_cnts, const void_int *node_proc_ptrs, + const void_int *elem_cmap_ids, const void_int *elem_cmap_elem_cnts, + const void_int *elem_proc_ptrs) { size_t num_n_comm_maps, num_e_comm_maps, num_procs_in_file; int status, icm, n_varid[2], e_varid[2], iproc; diff --git a/packages/seacas/libraries/exodus/src/ex_put_concat_elem_block.c b/packages/seacas/libraries/exodus/src/ex_put_concat_elem_block.c index fb9d05164bc8..1f2cee9247dd 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_concat_elem_block.c +++ b/packages/seacas/libraries/exodus/src/ex_put_concat_elem_block.c @@ -33,7 +33,7 @@ * \param num_attr_this_blk number of attributes * \param define_maps if != 0, write maps, else don't */ -int ex_put_concat_elem_block(int exoid, const void_int *elem_blk_id, char *elem_type[], +int ex_put_concat_elem_block(int exoid, const void_int *elem_blk_id, char *const elem_type[], const void_int *num_elem_this_blk, const void_int *num_nodes_per_elem, const void_int *num_attr_this_blk, int define_maps) { diff --git a/packages/seacas/libraries/exodus/src/ex_put_conn.c b/packages/seacas/libraries/exodus/src/ex_put_conn.c index 523a555b6c0c..e53f74680bb4 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_conn.c +++ b/packages/seacas/libraries/exodus/src/ex_put_conn.c @@ -44,7 +44,8 @@ int ex_int_write_conn(int exoid, ex_entity_id blk_id, const char *type, int var_ int ex_put_conn(int exoid, ex_entity_type blk_type, ex_entity_id blk_id, const void_int *node_conn, const void_int *elem_edge_conn, const void_int *elem_face_conn) { - int connid = -1, blk_id_ndx, status; + int connid = -1; + int status; char errmsg[MAX_ERR_LENGTH]; EX_FUNC_ENTER(); @@ -52,7 +53,7 @@ int ex_put_conn(int exoid, ex_entity_type blk_type, ex_entity_id blk_id, const v EX_FUNC_LEAVE(EX_FATAL); } - blk_id_ndx = ex__id_lkup(exoid, blk_type, blk_id); + int blk_id_ndx = ex__id_lkup(exoid, blk_type, blk_id); if (blk_id_ndx <= 0) { ex_get_err(NULL, NULL, &status); diff --git a/packages/seacas/libraries/exodus/src/ex_put_coord_names.c b/packages/seacas/libraries/exodus/src/ex_put_coord_names.c index 2396b74304ca..4a299743a025 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_coord_names.c +++ b/packages/seacas/libraries/exodus/src/ex_put_coord_names.c @@ -43,7 +43,7 @@ error = ex_put_coord_names (exoid, coord_names); */ -int ex_put_coord_names(int exoid, char *coord_names[]) +int ex_put_coord_names(int exoid, char *const coord_names[]) { int status; int ndimdim, varid; diff --git a/packages/seacas/libraries/exodus/src/ex_put_coordinate_frames.c b/packages/seacas/libraries/exodus/src/ex_put_coordinate_frames.c index 820dc25c3a3c..3ee2a5ef61e4 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_coordinate_frames.c +++ b/packages/seacas/libraries/exodus/src/ex_put_coordinate_frames.c @@ -27,8 +27,8 @@ #include "exodusII.h" // for ex_err, EXERRVAL, etc #include "exodusII_int.h" // for EX_FATAL, EX_NOERR, etc -int ex_put_coordinate_frames(int exoid, int nframes, const void_int *cf_ids, void *pt_coordinates, - const char *tags) +int ex_put_coordinate_frames(int exoid, int nframes, const void_int *cf_ids, + const void *pt_coordinates, const char *tags) { int status; int dim, dim9; /* dimension id for nframes, nframes*9 */ diff --git a/packages/seacas/libraries/exodus/src/ex_put_eb_info_global.c b/packages/seacas/libraries/exodus/src/ex_put_eb_info_global.c index 26b2a2c839c7..7186ba2fc720 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_eb_info_global.c +++ b/packages/seacas/libraries/exodus/src/ex_put_eb_info_global.c @@ -26,7 +26,7 @@ #include #include -int ex_put_eb_info_global(int exoid, void_int *el_blk_ids, void_int *el_blk_cnts) +int ex_put_eb_info_global(int exoid, const void_int *el_blk_ids, const void_int *el_blk_cnts) { int varid, status; char errmsg[MAX_ERR_LENGTH]; diff --git a/packages/seacas/libraries/exodus/src/ex_put_elem_cmap.c b/packages/seacas/libraries/exodus/src/ex_put_elem_cmap.c index 2b43db5c46e9..91b015854aa2 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_elem_cmap.c +++ b/packages/seacas/libraries/exodus/src/ex_put_elem_cmap.c @@ -30,8 +30,8 @@ #include // for ex_err, etc #include // for EX_FATAL, DIM_ECNT_CMAP, etc -int ex_put_elem_cmap(int exoid, ex_entity_id map_id, void_int *elem_ids, void_int *side_ids, - void_int *proc_ids, int processor) +int ex_put_elem_cmap(int exoid, ex_entity_id map_id, const void_int *elem_ids, + const void_int *side_ids, const void_int *proc_ids, int processor) { int map_idx, varid, dimid, status; size_t start[1], count[1], ret_val; diff --git a/packages/seacas/libraries/exodus/src/ex_put_info.c b/packages/seacas/libraries/exodus/src/ex_put_info.c index dae2a8adfe7f..a0dda7ae0679 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_info.c +++ b/packages/seacas/libraries/exodus/src/ex_put_info.c @@ -70,7 +70,7 @@ error = ex_put_info(exoid, num_info, info); */ -int ex_put_info(int exoid, int num_info, char *info[]) +int ex_put_info(int exoid, int num_info, char *const info[]) { int status; int i, lindim, num_info_dim, dims[2], varid; diff --git a/packages/seacas/libraries/exodus/src/ex_put_init_ext.c b/packages/seacas/libraries/exodus/src/ex_put_init_ext.c index e4d7b757258a..60436a2fdde4 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_init_ext.c +++ b/packages/seacas/libraries/exodus/src/ex_put_init_ext.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -187,23 +187,22 @@ static int ex_write_map_params(int exoid, const char *map_name, const char *map_ static void invalidate_id_status(int exoid, const char *var_stat, const char *var_id, int count, int *ids) { - int i; - int id_var, stat_var; - if (count > 0) { if (var_id != 0) { - for (i = 0; i < count; i++) { + for (int i = 0; i < count; i++) { ids[i] = EX_INVALID_ID; } + int id_var; (void)nc_inq_varid(exoid, var_id, &id_var); (void)nc_put_var_int(exoid, id_var, ids); } if (var_stat != 0) { - for (i = 0; i < count; i++) { + for (int i = 0; i < count; i++) { ids[i] = 0; } + int stat_var; (void)nc_inq_varid(exoid, var_stat, &stat_var); (void)nc_put_var_int(exoid, stat_var, ids); } @@ -289,7 +288,7 @@ int ex_put_init_ext(int exoid, const ex_init_params *model) struct ex__file_item *file = ex__find_file_item(exoid); file->time_varid = temp; } - ex__compress_variable(exoid, temp, 2); + ex__compress_variable(exoid, temp, -2); /* Don't compress, but do set collective io */ if (model->num_dim > 0) { if ((status = nc_def_dim(exoid, DIM_NUM_DIM, model->num_dim, &numdimdim)) != NC_NOERR) { diff --git a/packages/seacas/libraries/exodus/src/ex_put_init_info.c b/packages/seacas/libraries/exodus/src/ex_put_init_info.c index 43c1ed9b5ad1..ded7e84de99f 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_init_info.c +++ b/packages/seacas/libraries/exodus/src/ex_put_init_info.c @@ -31,7 +31,7 @@ * \ingroup ModelDescription * \undoc */ -int ex_put_init_info(int exoid, int num_proc, int num_proc_in_f, char *ftype) +int ex_put_init_info(int exoid, int num_proc, int num_proc_in_f, const char *ftype) { int dimid, varid; int ltempsv; diff --git a/packages/seacas/libraries/exodus/src/ex_put_loadbal_param_cc.c b/packages/seacas/libraries/exodus/src/ex_put_loadbal_param_cc.c index b2e78d8a812a..9b7b2d837b2a 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_loadbal_param_cc.c +++ b/packages/seacas/libraries/exodus/src/ex_put_loadbal_param_cc.c @@ -46,10 +46,10 @@ * \ingroup ModelDescription * \undoc */ -int ex_put_loadbal_param_cc(int exoid, void_int *num_int_nodes, void_int *num_bor_nodes, - void_int *num_ext_nodes, void_int *num_int_elems, - void_int *num_bor_elems, void_int *num_node_cmaps, - void_int *num_elem_cmaps) +int ex_put_loadbal_param_cc(int exoid, const void_int *num_int_nodes, const void_int *num_bor_nodes, + const void_int *num_ext_nodes, const void_int *num_int_elems, + const void_int *num_bor_elems, const void_int *num_node_cmaps, + const void_int *num_elem_cmaps) { int status; int iproc, varid, dimid_npf, dimid[3]; diff --git a/packages/seacas/libraries/exodus/src/ex_put_names.c b/packages/seacas/libraries/exodus/src/ex_put_names.c index e5393a2c8c90..81df26487509 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_names.c +++ b/packages/seacas/libraries/exodus/src/ex_put_names.c @@ -34,7 +34,7 @@ * \param names ptr array of entity names */ -int ex_put_names(int exoid, ex_entity_type obj_type, char *names[]) +int ex_put_names(int exoid, ex_entity_type obj_type, char *const names[]) { int status; int varid; diff --git a/packages/seacas/libraries/exodus/src/ex_put_node_cmap.c b/packages/seacas/libraries/exodus/src/ex_put_node_cmap.c index 3551f479d32f..5e8adf211599 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_node_cmap.c +++ b/packages/seacas/libraries/exodus/src/ex_put_node_cmap.c @@ -30,8 +30,8 @@ #include // for ex_err, etc #include // for EX_FATAL, DIM_NCNT_CMAP, etc -int ex_put_node_cmap(int exoid, ex_entity_id map_id, void_int *node_ids, void_int *proc_ids, - int processor) +int ex_put_node_cmap(int exoid, ex_entity_id map_id, const void_int *node_ids, + const void_int *proc_ids, int processor) { int map_idx, varid, dimid, status; size_t start[1], count[1], ret_val; diff --git a/packages/seacas/libraries/exodus/src/ex_put_ns_param_global.c b/packages/seacas/libraries/exodus/src/ex_put_ns_param_global.c index 8cb690febbc8..843ce96cc16e 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_ns_param_global.c +++ b/packages/seacas/libraries/exodus/src/ex_put_ns_param_global.c @@ -29,7 +29,8 @@ #include #include -int ex_put_ns_param_global(int exoid, void_int *global_ids, void_int *node_cnts, void_int *df_cnts) +int ex_put_ns_param_global(int exoid, const void_int *global_ids, const void_int *node_cnts, + const void_int *df_cnts) { int varid, status; diff --git a/packages/seacas/libraries/exodus/src/ex_put_num_map.c b/packages/seacas/libraries/exodus/src/ex_put_num_map.c index 6f55f0de5341..fcf91eb2e94f 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_num_map.c +++ b/packages/seacas/libraries/exodus/src/ex_put_num_map.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -37,25 +37,19 @@ int ex_put_num_map(int exoid, ex_entity_type map_type, ex_entity_id map_id, const void_int *map) { - int dimid, varid; - int map_int_type; - size_t start[1]; - int ldum; - int num_maps; - size_t num_entries; - int cur_num_maps; - char errmsg[MAX_ERR_LENGTH]; - const char *dnumentries; - const char *dnummaps; - const char *vmapids; - const char *vmap; - int status; + size_t start[1]; + char errmsg[MAX_ERR_LENGTH]; + int status; EX_FUNC_ENTER(); if (ex__check_valid_file_id(exoid, __func__) == EX_FATAL) { EX_FUNC_LEAVE(EX_FATAL); } + const char *dnumentries; + const char *dnummaps; + const char *vmapids; + switch (map_type) { case EX_NODE_MAP: dnumentries = DIM_NUM_NODES; @@ -85,6 +79,7 @@ int ex_put_num_map(int exoid, ex_entity_type map_type, ex_entity_id map_id, cons } /* Make sure the file contains entries */ + int dimid; if (nc_inq_dimid(exoid, dnumentries, &dimid) != NC_NOERR) { EX_FUNC_LEAVE(EX_NOERR); } @@ -107,20 +102,21 @@ int ex_put_num_map(int exoid, ex_entity_type map_type, ex_entity_id map_id, cons } /* Get number of maps initialized for this file */ + size_t num_entries; if ((status = nc_inq_dimlen(exoid, dimid, &num_entries)) != NC_NOERR) { snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to get number of %ss in file id %d", ex_name_of_object(map_type), exoid); ex_err_fn(exoid, __func__, errmsg, status); EX_FUNC_LEAVE(EX_FATAL); } - num_maps = num_entries; + int num_maps = num_entries; /* Keep track of the total number of maps defined using a counter stored in a linked list keyed by exoid. NOTE: ex__get_file_item is used to find the number of maps for a specific file and returns that value. */ - cur_num_maps = ex__get_file_item(exoid, ex__get_counter_list(map_type)); + int cur_num_maps = ex__get_file_item(exoid, ex__get_counter_list(map_type)); if (cur_num_maps >= num_maps) { snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: exceeded number of %ss (%d) specified in file id %d", ex_name_of_object(map_type), num_maps, exoid); @@ -135,6 +131,7 @@ int ex_put_num_map(int exoid, ex_entity_type map_type, ex_entity_id map_id, cons /* write out information to previously defined variable */ /* first get id of variable */ + int varid; if ((status = nc_inq_varid(exoid, vmapids, &varid)) == -1) { snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to locate %s ids in file id %d", ex_name_of_object(map_type), exoid); @@ -145,7 +142,7 @@ int ex_put_num_map(int exoid, ex_entity_type map_type, ex_entity_id map_id, cons /* then, write out map id */ start[0] = cur_num_maps; - ldum = (int)map_id; + int ldum = (int)map_id; if ((status = nc_put_var1_int(exoid, varid, start, &ldum)) != NC_NOERR) { snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to store %s id %" PRId64 " in file id %d", ex_name_of_object(map_type), map_id, exoid); @@ -153,6 +150,7 @@ int ex_put_num_map(int exoid, ex_entity_type map_type, ex_entity_id map_id, cons EX_FUNC_LEAVE(EX_FATAL); } + const char *vmap; switch (map_type) { case EX_NODE_MAP: vmap = VAR_NODE_MAP(cur_num_maps + 1); break; case EX_EDGE_MAP: vmap = VAR_EDGE_MAP(cur_num_maps + 1); break; @@ -185,7 +183,7 @@ int ex_put_num_map(int exoid, ex_entity_type map_type, ex_entity_id map_id, cons } /* Check type to be used for maps... */ - map_int_type = NC_INT; + int map_int_type = NC_INT; if (ex_int64_status(exoid) & EX_MAPS_INT64_DB) { map_int_type = NC_INT64; } diff --git a/packages/seacas/libraries/exodus/src/ex_put_partial_node_cmap.c b/packages/seacas/libraries/exodus/src/ex_put_partial_node_cmap.c index d4367c92d168..079e2220a464 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_partial_node_cmap.c +++ b/packages/seacas/libraries/exodus/src/ex_put_partial_node_cmap.c @@ -40,8 +40,8 @@ */ int ex_put_partial_node_cmap(int exoid, ex_entity_id map_id, int64_t start_entity_num, - int64_t num_entities, void_int *node_ids, void_int *proc_ids, - int processor) + int64_t num_entities, const void_int *node_ids, + const void_int *proc_ids, int processor) { int map_idx, varid, dimid, status; size_t start[1], count[1], ret_val; diff --git a/packages/seacas/libraries/exodus/src/ex_put_partial_num_map.c b/packages/seacas/libraries/exodus/src/ex_put_partial_num_map.c index 0374797ef4fd..82689c0fd1c9 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_partial_num_map.c +++ b/packages/seacas/libraries/exodus/src/ex_put_partial_num_map.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -35,7 +35,7 @@ int ex_put_partial_num_map(int exoid, ex_entity_type map_type, ex_entity_id map_ int64_t ent_start, int64_t ent_count, const void_int *map) { int status; - int dimid, varid, map_ndx, map_exists; + int dimid, varid, map_exists; size_t start[1]; size_t num_maps, num_mobj, count[1]; int cur_num_maps; @@ -129,7 +129,7 @@ int ex_put_partial_num_map(int exoid, ex_entity_type map_type, ex_entity_id map_ cur_num_maps = ex__inc_file_item(exoid, ex__get_counter_list(map_type)); } else { - map_ndx = ex__id_lkup(exoid, map_type, map_id); + int map_ndx = ex__id_lkup(exoid, map_type, map_id); cur_num_maps = map_ndx - 1; } diff --git a/packages/seacas/libraries/exodus/src/ex_put_processor_elem_maps.c b/packages/seacas/libraries/exodus/src/ex_put_processor_elem_maps.c index 8d170ed4461d..5e3ba6520984 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_processor_elem_maps.c +++ b/packages/seacas/libraries/exodus/src/ex_put_processor_elem_maps.c @@ -26,7 +26,8 @@ #include // for ex_err, etc #include // for EX_FATAL, DIM_NUM_BOR_ELEMS, etc -int ex_put_processor_elem_maps(int exoid, void_int *elem_mapi, void_int *elem_mapb, int processor) +int ex_put_processor_elem_maps(int exoid, const void_int *elem_mapi, const void_int *elem_mapb, + int processor) { char ftype[2]; int status, dimid, varid; diff --git a/packages/seacas/libraries/exodus/src/ex_put_processor_node_maps.c b/packages/seacas/libraries/exodus/src/ex_put_processor_node_maps.c index 0fa313742c6b..cbb76d724a0e 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_processor_node_maps.c +++ b/packages/seacas/libraries/exodus/src/ex_put_processor_node_maps.c @@ -26,8 +26,8 @@ #include // for ex_err, etc #include // for EX_FATAL, DIM_NUM_BOR_NODES, etc -int ex_put_processor_node_maps(int exoid, void_int *node_mapi, void_int *node_mapb, - void_int *node_mape, int proc_id) +int ex_put_processor_node_maps(int exoid, const void_int *node_mapi, const void_int *node_mapb, + const void_int *node_mape, int proc_id) { int status, varid, dimid; char ftype[2]; diff --git a/packages/seacas/libraries/exodus/src/ex_put_reduction_variable_names.c b/packages/seacas/libraries/exodus/src/ex_put_reduction_variable_names.c index 318981a8ea1f..4c8827fa180e 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_reduction_variable_names.c +++ b/packages/seacas/libraries/exodus/src/ex_put_reduction_variable_names.c @@ -119,7 +119,7 @@ error = ex_put_variable_names (exoid, EX_NODAL, num_nod_vars, var_names); */ int ex_put_reduction_variable_names(int exoid, ex_entity_type obj_type, int num_vars, - char *var_names[]) + char *const var_names[]) { int varid, status; char errmsg[MAX_ERR_LENGTH]; diff --git a/packages/seacas/libraries/exodus/src/ex_put_ss_param_global.c b/packages/seacas/libraries/exodus/src/ex_put_ss_param_global.c index ab5bd6860427..378484cae0c6 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_ss_param_global.c +++ b/packages/seacas/libraries/exodus/src/ex_put_ss_param_global.c @@ -29,7 +29,8 @@ #include #include -int ex_put_ss_param_global(int exoid, void_int *global_ids, void_int *side_cnts, void_int *df_cnts) +int ex_put_ss_param_global(int exoid, const void_int *global_ids, const void_int *side_cnts, + const void_int *df_cnts) { int varid; diff --git a/packages/seacas/libraries/exodus/src/ex_put_truth_table.c b/packages/seacas/libraries/exodus/src/ex_put_truth_table.c index 9bb4d45de6f1..78e861bf3645 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_truth_table.c +++ b/packages/seacas/libraries/exodus/src/ex_put_truth_table.c @@ -64,7 +64,8 @@ error = ex_put_truth_table(exoid, EX_ELEM_BLOCK, num_elem_blk, num_ele_vars, ~~~ */ -int ex_put_truth_table(int exoid, ex_entity_type obj_type, int num_blk, int num_var, int *var_tab) +int ex_put_truth_table(int exoid, ex_entity_type obj_type, int num_blk, int num_var, + const int *var_tab) { int numelblkdim, numelvardim, timedim, dims[2], varid; char * sta_type = NULL; diff --git a/packages/seacas/libraries/exodus/src/ex_put_variable_names.c b/packages/seacas/libraries/exodus/src/ex_put_variable_names.c index c085241df562..2897b890bd04 100644 --- a/packages/seacas/libraries/exodus/src/ex_put_variable_names.c +++ b/packages/seacas/libraries/exodus/src/ex_put_variable_names.c @@ -122,7 +122,7 @@ error = ex_put_variable_names (exoid, EX_NODAL, num_nod_vars, var_names); */ -int ex_put_variable_names(int exoid, ex_entity_type obj_type, int num_vars, char *var_names[]) +int ex_put_variable_names(int exoid, ex_entity_type obj_type, int num_vars, char *const var_names[]) { int varid = 0; int status = 0; diff --git a/packages/seacas/libraries/exodus/src/ex_utils.c b/packages/seacas/libraries/exodus/src/ex_utils.c index 0255a11e3d8e..076483508550 100644 --- a/packages/seacas/libraries/exodus/src/ex_utils.c +++ b/packages/seacas/libraries/exodus/src/ex_utils.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -181,8 +181,6 @@ int ex__check_file_type(const char *path, int *type) /* Get the 4-byte magic from the beginning of the file. */ { FILE *fp; - int i; - if (!(fp = fopen(path, "r"))) { char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: Could not open file '%s', error = %s.", path, @@ -190,7 +188,7 @@ int ex__check_file_type(const char *path, int *type) ex_err(__func__, errmsg, EX_WRONGFILETYPE); EX_FUNC_LEAVE(EX_FATAL); } - i = fread(magic, MAGIC_NUMBER_LEN, 1, fp); + int i = fread(magic, MAGIC_NUMBER_LEN, 1, fp); magic[MAGIC_NUMBER_LEN] = '\0'; fclose(fp); if (i != 1) { @@ -264,16 +262,15 @@ int ex_set_max_name_length(int exoid, int length) */ void ex__update_max_name_length(int exoid, int length) { - int status; - int db_length = 0; - int rootid = exoid & EX_FILE_ID_MASK; - EX_FUNC_ENTER(); if (ex__check_valid_file_id(exoid, __func__) == EX_FATAL) { EX_FUNC_VOID(); } /* Get current value of the maximum_name_length attribute... */ + int status; + int db_length = 0; + int rootid = exoid & EX_FILE_ID_MASK; if ((status = nc_get_att_int(rootid, NC_GLOBAL, ATT_MAX_NAME_LENGTH, &db_length)) != NC_NOERR) { char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, @@ -301,27 +298,19 @@ void ex__update_max_name_length(int exoid, int length) \internal \undoc */ -int ex__put_names(int exoid, int varid, size_t num_names, char **names, ex_entity_type obj_type, - const char *subtype, const char *routine) +int ex__put_names(int exoid, int varid, size_t num_names, char *const *names, + ex_entity_type obj_type, const char *subtype, const char *routine) { - size_t i; - int status; - char errmsg[MAX_ERR_LENGTH]; - int max_name_len = 0; - size_t name_length; - size_t length; - char * int_names = NULL; - size_t idx = 0; - int found_name = 0; - EX_FUNC_ENTER(); if (ex__check_valid_file_id(exoid, __func__) == EX_FATAL) { EX_FUNC_LEAVE(EX_FATAL); } /* inquire previously defined dimensions */ - name_length = ex_inquire_int(exoid, EX_INQ_DB_MAX_ALLOWED_NAME_LENGTH) + 1; + size_t name_length = ex_inquire_int(exoid, EX_INQ_DB_MAX_ALLOWED_NAME_LENGTH) + 1; + char *int_names = NULL; if (!(int_names = calloc(num_names * name_length, 1))) { + char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to allocate memory for internal int_names " "array in file id %d", @@ -330,11 +319,14 @@ int ex__put_names(int exoid, int varid, size_t num_names, char **names, ex_entit EX_FUNC_LEAVE(EX_FATAL); } - for (i = 0; i < num_names; i++) { + size_t idx = 0; + int max_name_len = 0; + int found_name = 0; + for (size_t i = 0; i < num_names; i++) { if (names != NULL && *names != NULL && *names[i] != '\0') { found_name = 1; ex_copy_string(&int_names[idx], names[i], name_length); - length = strlen(names[i]) + 1; + size_t length = strlen(names[i]) + 1; if (length > name_length) { fprintf(stderr, "Warning: The %s %s name '%s' is too long.\n\tIt will " @@ -351,8 +343,10 @@ int ex__put_names(int exoid, int varid, size_t num_names, char **names, ex_entit idx += name_length; } + int status; if ((status = nc_put_var_text(exoid, varid, int_names)) != NC_NOERR) { free(int_names); + char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to store %s names in file id %d", ex_name_of_object(obj_type), exoid); ex_err_fn(exoid, __func__, errmsg, status); @@ -376,22 +370,18 @@ int ex__put_names(int exoid, int varid, size_t num_names, char **names, ex_entit int ex__put_name(int exoid, int varid, size_t index, const char *name, ex_entity_type obj_type, const char *subtype, const char *routine) { - int status; - size_t start[2], count[2]; - char errmsg[MAX_ERR_LENGTH]; - size_t name_length; - if (ex__check_valid_file_id(exoid, __func__) == EX_FATAL) { EX_FUNC_LEAVE(EX_FATAL); } /* inquire previously defined dimensions */ - name_length = ex_inquire_int(exoid, EX_INQ_DB_MAX_ALLOWED_NAME_LENGTH) + 1; + size_t name_length = ex_inquire_int(exoid, EX_INQ_DB_MAX_ALLOWED_NAME_LENGTH) + 1; if (name != NULL && *name != '\0') { - int too_long = 0; - start[0] = index; - start[1] = 0; + size_t start[2], count[2]; + int too_long = 0; + start[0] = index; + start[1] = 0; count[0] = 1; count[1] = strlen(name) + 1; @@ -406,7 +396,9 @@ int ex__put_name(int exoid, int varid, size_t index, const char *name, ex_entity too_long = 1; } + int status; if ((status = nc_put_vara_text(exoid, varid, start, count, name)) != NC_NOERR) { + char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to store %s name in file id %d", ex_name_of_object(obj_type), exoid); ex_err_fn(exoid, __func__, errmsg, status); @@ -432,9 +424,6 @@ int ex__put_name(int exoid, int varid, size_t index, const char *name, ex_entity int ex__get_names(int exoid, int varid, size_t num_names, char **names, ex_entity_type obj_type, const char *routine) { - size_t i; - int status; - /* Query size of names on file * Use the smaller of the size on file or user-specified length */ @@ -442,8 +431,8 @@ int ex__get_names(int exoid, int varid, size_t num_names, char **names, ex_entit int api_name_size = ex_inquire_int(exoid, EX_INQ_MAX_READ_NAME_LENGTH); int name_size = db_name_size < api_name_size ? db_name_size : api_name_size; - for (i = 0; i < num_names; i++) { - status = ex__get_name(exoid, varid, i, names[i], name_size, obj_type, routine); + for (int i = 0; i < num_names; i++) { + int status = ex__get_name(exoid, varid, i, names[i], name_size, obj_type, routine); if (status != NC_NOERR) { return (status); } @@ -458,21 +447,17 @@ int ex__get_names(int exoid, int varid, size_t num_names, char **names, ex_entit int ex__get_name(int exoid, int varid, size_t index, char *name, int name_size, ex_entity_type obj_type, const char *routine) { - size_t start[2], count[2]; - int status; - char errmsg[MAX_ERR_LENGTH]; - int api_name_size = 0; - - api_name_size = ex_inquire_int(exoid, EX_INQ_MAX_READ_NAME_LENGTH); /* read the name */ + size_t start[2], count[2]; start[0] = index; count[0] = 1; start[1] = 0; count[1] = name_size + 1; - status = nc_get_vara_text(exoid, varid, start, count, name); + int status = nc_get_vara_text(exoid, varid, start, count, name); if (status != NC_NOERR) { + char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to get %s name at index %d from file id %d [Called from %s]", ex_name_of_object(obj_type), (int)index, exoid, routine); @@ -480,6 +465,7 @@ int ex__get_name(int exoid, int varid, size_t index, char *name, int name_size, return (EX_FATAL); } + int api_name_size = ex_inquire_int(exoid, EX_INQ_MAX_READ_NAME_LENGTH); name[api_name_size] = '\0'; ex__trim(name); @@ -494,19 +480,16 @@ void ex__trim(char *name) { /* Thread-safe, reentrant */ /* Trim trailing spaces... */ - size_t size; - char * end; - if (name == NULL) { return; } - size = strlen(name); + size_t size = strlen(name); if (size == 0) { return; } - end = name + size - 1; + char *end = name + size - 1; while (end >= name && isspace(*end)) { end--; } @@ -772,16 +755,15 @@ char *ex__name_of_map(ex_entity_type map_type, int map_index) */ int ex__id_lkup(int exoid, ex_entity_type id_type, ex_entity_id num) { - char * id_table; - char * id_dim; - char * stat_table; + char * id_table = NULL; + char * id_dim = NULL; + char * stat_table = NULL; int varid, dimid; - size_t dim_len, j; + size_t dim_len; int64_t i; int64_t *id_vals = NULL; int * stat_vals = NULL; - static bool filled = false; static bool sequential = false; struct ex__obj_stats *tmp_stats; int status; @@ -942,8 +924,8 @@ int ex__id_lkup(int exoid, ex_entity_type id_type, ex_entity_id num) } /* check if values in stored arrays are filled with non-zeroes */ - filled = true; - sequential = true; + bool filled = true; + sequential = true; for (i = 0; i < dim_len; i++) { if (id_vals[i] != i + 1) { sequential = false; @@ -1019,7 +1001,7 @@ int ex__id_lkup(int exoid, ex_entity_type id_type, ex_entity_id num) } } else { - for (j = 0; j < dim_len; j++) { + for (size_t j = 0; j < dim_len; j++) { stat_vals[j] = 1; } } @@ -1065,9 +1047,7 @@ int ex__id_lkup(int exoid, ex_entity_type id_type, ex_entity_id num) struct ex__obj_stats *ex__get_stat_ptr(int exoid, struct ex__obj_stats **obj_ptr) { - struct ex__obj_stats *tmp_ptr; - - tmp_ptr = *obj_ptr; + struct ex__obj_stats *tmp_ptr = *obj_ptr; while (tmp_ptr) { if ((tmp_ptr)->exoid == exoid) { @@ -1105,10 +1085,8 @@ struct ex__obj_stats *ex__get_stat_ptr(int exoid, struct ex__obj_stats **obj_ptr void ex__rm_stat_ptr(int exoid, struct ex__obj_stats **obj_ptr) { - struct ex__obj_stats *last_head_list_ptr, *tmp_ptr; - - tmp_ptr = *obj_ptr; - last_head_list_ptr = *obj_ptr; /* save last head pointer */ + struct ex__obj_stats *tmp_ptr = *obj_ptr; + struct ex__obj_stats *last_head_list_ptr = *obj_ptr; /* save last head pointer */ while (tmp_ptr) /* Walk linked list of file ids/vals */ { @@ -1388,11 +1366,9 @@ int ex__get_cpu_ws(void) { return (sizeof(float)); } static void ex_swap(int v[], int64_t i, int64_t j) { /* Thread-safe, reentrant */ - int temp; - - temp = v[i]; - v[i] = v[j]; - v[j] = temp; + int temp = v[i]; + v[i] = v[j]; + v[j] = temp; } /*! @@ -1402,11 +1378,9 @@ static void ex_swap(int v[], int64_t i, int64_t j) static void ex_swap64(int64_t v[], int64_t i, int64_t j) { /* Thread-safe, reentrant */ - int64_t temp; - - temp = v[i]; - v[i] = v[j]; - v[j] = temp; + int64_t temp = v[i]; + v[i] = v[j]; + v[j] = temp; } #define EX_QSORT_CUTOFF 12 @@ -1418,8 +1392,7 @@ static void ex_swap64(int64_t v[], int64_t i, int64_t j) static int ex_int_median3(int v[], int iv[], int64_t left, int64_t right) { /* Thread-safe, reentrant */ - int64_t center; - center = (left + right) / 2; + int64_t center = (left + right) / 2; if (v[iv[left]] > v[iv[center]]) { ex_swap(iv, left, center); @@ -1442,8 +1415,7 @@ static int ex_int_median3(int v[], int iv[], int64_t left, int64_t right) static int64_t ex_int_median3_64(int64_t v[], int64_t iv[], int64_t left, int64_t right) { /* Thread-safe, reentrant */ - int64_t center; - center = (left + right) / 2; + int64_t center = (left + right) / 2; if (v[iv[left]] > v[iv[center]]) { ex_swap64(iv, left, center); @@ -1466,13 +1438,10 @@ static int64_t ex_int_median3_64(int64_t v[], int64_t iv[], int64_t left, int64_ static void ex_int_iqsort(int v[], int iv[], int left, int right) { /* Thread-safe, reentrant */ - int pivot; - int i, j; - if (left + EX_QSORT_CUTOFF <= right) { - pivot = ex_int_median3(v, iv, left, right); - i = left; - j = right - 1; + int pivot = ex_int_median3(v, iv, left, right); + int i = left; + int j = right - 1; for (;;) { while (v[iv[++i]] < v[pivot]) { @@ -1502,13 +1471,10 @@ static void ex_int_iqsort(int v[], int iv[], int left, int right) static void ex_int_iqsort64(int64_t v[], int64_t iv[], int64_t left, int64_t right) { /* Thread-safe, reentrant */ - int64_t pivot; - int64_t i, j; - if (left + EX_QSORT_CUTOFF <= right) { - pivot = ex_int_median3_64(v, iv, left, right); - i = left; - j = right - 1; + int64_t pivot = ex_int_median3_64(v, iv, left, right); + int64_t i = left; + int64_t j = right - 1; for (;;) { while (v[iv[++i]] < v[pivot]) { @@ -1538,13 +1504,9 @@ static void ex_int_iqsort64(int64_t v[], int64_t iv[], int64_t left, int64_t rig static void ex_int_iisort(int v[], int iv[], int N) { /* Thread-safe, reentrant */ - int i, j; - int ndx = 0; - int small; - int tmp; - - small = v[iv[0]]; - for (i = 1; i < N; i++) { + int ndx = 0; + int small = v[iv[0]]; + for (int i = 1; i < N; i++) { if (v[iv[i]] < small) { small = v[iv[i]]; ndx = i; @@ -1553,8 +1515,9 @@ static void ex_int_iisort(int v[], int iv[], int N) /* Put smallest value in slot 0 */ ex_swap(iv, 0, ndx); - for (i = 1; i < N; i++) { - tmp = iv[i]; + for (int i = 1; i < N; i++) { + int tmp = iv[i]; + int j; for (j = i; v[tmp] < v[iv[j - 1]]; j--) { iv[j] = iv[j - 1]; } @@ -1569,13 +1532,9 @@ static void ex_int_iisort(int v[], int iv[], int N) static void ex_int_iisort64(int64_t v[], int64_t iv[], int64_t N) { /* Thread-safe, reentrant */ - int64_t i, j; - int64_t ndx = 0; - int64_t small; - int64_t tmp; - - small = v[iv[0]]; - for (i = 1; i < N; i++) { + int64_t ndx = 0; + int64_t small = v[iv[0]]; + for (int64_t i = 1; i < N; i++) { if (v[iv[i]] < small) { small = v[iv[i]]; ndx = i; @@ -1584,8 +1543,9 @@ static void ex_int_iisort64(int64_t v[], int64_t iv[], int64_t N) /* Put smallest value in slot 0 */ ex_swap64(iv, 0, ndx); - for (i = 1; i < N; i++) { - tmp = iv[i]; + for (int64_t i = 1; i < N; i++) { + int64_t tmp = iv[i]; + int64_t j; for (j = i; v[tmp] < v[iv[j - 1]]; j--) { iv[j] = iv[j - 1]; } @@ -1677,8 +1637,7 @@ int ex_large_model(int exoid) int ex__get_dimension(int exoid, const char *DIMENSION, const char *label, size_t *count, int *dimid, const char *routine) { - char errmsg[MAX_ERR_LENGTH]; - int status; + int status; *count = 0; *dimid = -1; @@ -1686,11 +1645,13 @@ int ex__get_dimension(int exoid, const char *DIMENSION, const char *label, size_ if ((status = nc_inq_dimid(exoid, DIMENSION, dimid)) != NC_NOERR) { if (routine != NULL) { if (status == NC_EBADDIM) { + char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, "Warning: no dimension defining '%s' found in file id %d", label, exoid); ex_err_fn(exoid, __func__, errmsg, status); } else { + char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to locate dimension defining number of '%s' in file id %d", label, exoid); @@ -1702,6 +1663,7 @@ int ex__get_dimension(int exoid, const char *DIMENSION, const char *label, size_ if ((status = nc_inq_dimlen(exoid, *dimid, count)) != NC_NOERR) { if (routine != NULL) { + char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to get length of dimension defining number of '%s' in file id %d", label, exoid); @@ -1735,7 +1697,11 @@ void ex__set_compact_storage(int exoid, int varid) #endif } -/* type = 1 for integer, 2 for real, 3 for character */ +/* + * type = 1 for integer, 2 for real, 3 for character + * If type < 0, then don't compress, but do set collective on parallel + */ + /*! \internal \undoc @@ -1753,7 +1719,7 @@ void ex__compress_variable(int exoid, int varid, int type) } else { /* Compression only supported on HDF5 (NetCDF-4) files; Do not try to compress character data */ - if (type != 3 && file->is_hdf5) { + if ((type == 1 || type == 2) && file->is_hdf5) { if (file->compression_algorithm == EX_COMPRESS_GZIP) { int deflate_level = file->compression_level; int compress = 1; @@ -1782,13 +1748,12 @@ void ex__compress_variable(int exoid, int varid, int type) ex_err_fn(exoid, __func__, errmsg, EX_BADPARAM); #endif } - + } #if defined(PARALLEL_AWARE_EXODUS) - if (file->is_parallel && file->is_hdf5) { - nc_var_par_access(exoid, varid, NC_COLLECTIVE); - } -#endif + if (file->is_parallel) { + nc_var_par_access(exoid, varid, NC_COLLECTIVE); } +#endif } #endif } @@ -1799,10 +1764,10 @@ void ex__compress_variable(int exoid, int varid, int type) */ int ex__leavedef(int exoid, const char *call_rout) { - char errmsg[MAX_ERR_LENGTH]; - int status; + int status; if ((status = nc_enddef(exoid)) != NC_NOERR) { + char errmsg[MAX_ERR_LENGTH]; snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to complete definition for file id %d", exoid); ex_err_fn(exoid, call_rout, errmsg, status); diff --git a/packages/seacas/libraries/exodus/test/ReadEdgeFace.c b/packages/seacas/libraries/exodus/test/ReadEdgeFace.c index 22b4654b755c..99fdccda52bb 100644 --- a/packages/seacas/libraries/exodus/test/ReadEdgeFace.c +++ b/packages/seacas/libraries/exodus/test/ReadEdgeFace.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -59,31 +59,21 @@ int obj_sizeinq[] = {EX_INQ_EDGE, int cReadEdgeFace(int argc, char **argv) { - int exoid; - int appWordSize = 8; - int diskWordSize = 8; - float exoVersion; - int itmp[5]; - int * ids; - int nids; - int obj; - int i, j; - int num_timesteps; - int ti; - char ** obj_names; - char ** var_names; - int have_var_names; - int num_vars; /* number of variables per object */ - int num_entries; /* number of values per variable per object */ - double * entry_vals; /* variable values for each entry of an object */ - ex_init_params modelParams; - - exoid = ex_open(EX_TEST_FILENAME, EX_READ, &appWordSize, &diskWordSize, &exoVersion); + int nids; + char **var_names; + int num_vars; /* number of variables per object */ + int num_entries; /* number of values per variable per object */ + + int appWordSize = 8; + int diskWordSize = 8; + float exoVersion; + int exoid = ex_open(EX_TEST_FILENAME, EX_READ, &appWordSize, &diskWordSize, &exoVersion); if (exoid <= 0) { fprintf(stderr, "Unable to open \"%s\" for reading.\n", EX_TEST_FILENAME); return 1; } + ex_init_params modelParams; EXCHECK(ex_get_init_ext(exoid, &modelParams), "Unable to read database parameters.\n"); fprintf(stdout, @@ -112,12 +102,12 @@ int cReadEdgeFace(int argc, char **argv) modelParams.num_elem_sets, modelParams.num_node_maps, modelParams.num_edge_maps, modelParams.num_face_maps, modelParams.num_elem_maps); - num_timesteps = ex_inquire_int(exoid, EX_INQ_TIME); + int num_timesteps = ex_inquire_int(exoid, EX_INQ_TIME); /* *** NEW API *** */ - for (i = 0; i < sizeof(obj_types) / sizeof(obj_types[0]); ++i) { - int *truth_tab = 0; - have_var_names = 0; + for (int i = 0; i < sizeof(obj_types) / sizeof(obj_types[0]); ++i) { + int *truth_tab = 0; + int have_var_names = 0; EXCHECK(ex_inquire(exoid, obj_sizes[i], &nids, 0, 0), "Object ID list size could not be determined.\n"); @@ -129,9 +119,9 @@ int cReadEdgeFace(int argc, char **argv) fprintf(stdout, "=== %ss: %d\n", obj_typenames[i], nids); - ids = (int *)malloc(nids * sizeof(int)); - obj_names = (char **)malloc(nids * sizeof(char *)); - for (obj = 0; obj < nids; ++obj) { + int * ids = (int *)malloc(nids * sizeof(int)); + char **obj_names = (char **)malloc(nids * sizeof(char *)); + for (int obj = 0; obj < nids; ++obj) { obj_names[obj] = (char *)malloc((MAX_STR_LENGTH + 1) * sizeof(char)); } @@ -139,7 +129,6 @@ int cReadEdgeFace(int argc, char **argv) EXCHECK(ex_get_names(exoid, obj_types[i], obj_names), "Could not read object ids.\n"); if ((OBJECT_IS_BLOCK(i)) || (OBJECT_IS_SET(i))) { - int *tp; EXCHECK(ex_get_variable_param(exoid, obj_types[i], &num_vars), "Could not read number of variables.\n"); @@ -147,10 +136,10 @@ int cReadEdgeFace(int argc, char **argv) truth_tab = (int *)malloc(num_vars * nids * sizeof(int)); EXCHECK(ex_get_truth_table(exoid, obj_types[i], nids, num_vars, truth_tab), "Could not read truth table.\n"); - tp = truth_tab; + int *tp = truth_tab; fprintf(stdout, "Truth:"); - for (obj = 0; obj < nids; ++obj) { - for (j = 0; j < num_vars; ++j, ++tp) { + for (int obj = 0; obj < nids; ++obj) { + for (int j = 0; j < num_vars; ++j, ++tp) { fprintf(stdout, " %d", *tp); } fprintf(stdout, "\n "); @@ -158,7 +147,7 @@ int cReadEdgeFace(int argc, char **argv) fprintf(stdout, "\n"); var_names = (char **)malloc(num_vars * sizeof(char *)); - for (j = 0; j < num_vars; ++j) { + for (int j = 0; j < num_vars; ++j) { var_names[j] = (char *)malloc((MAX_STR_LENGTH + 1) * sizeof(char)); } @@ -172,7 +161,7 @@ int cReadEdgeFace(int argc, char **argv) var_names = 0; } - for (obj = 0; obj < nids; ++obj) { + for (int obj = 0; obj < nids; ++obj) { if (obj_names[obj]) { fprintf(stdout, "%s %3d (%s): ", obj_typenames[i], ids[obj], obj_names[obj]); } @@ -187,6 +176,7 @@ int cReadEdgeFace(int argc, char **argv) int ele; int ctr; int num_attrs; + int itmp[4]; if (obj_types[i] == EX_ELEM_BLOCK) { EXCHECK(ex_get_block(exoid, obj_types[i], ids[obj], 0, itmp, itmp + 1, itmp + 2, itmp + 3, &num_attrs), @@ -236,7 +226,7 @@ int cReadEdgeFace(int argc, char **argv) double *attr; attr = (double *)malloc(num_entries * num_attrs * sizeof(double)); attr_names = (char **)malloc(num_attrs * sizeof(char *)); - for (j = 0; j < num_attrs; ++j) { + for (int j = 0; j < num_attrs; ++j) { attr_names[j] = (char *)malloc((MAX_STR_LENGTH + 1) * sizeof(char)); } @@ -246,20 +236,19 @@ int cReadEdgeFace(int argc, char **argv) "Could not read attribute values.\n"); fprintf(stdout, "\n Attributes:\n ID "); - for (j = 0; j < num_attrs; ++j) { + for (int j = 0; j < num_attrs; ++j) { fprintf(stdout, " %s", attr_names[j]); } fprintf(stdout, "\n"); - for (j = 0; j < num_entries; ++j) { - int k; + for (int j = 0; j < num_entries; ++j) { fprintf(stdout, " %2d ", j + 1); - for (k = 0; k < num_attrs; ++k) { + for (int k = 0; k < num_attrs; ++k) { fprintf(stdout, " %4.1f", attr[j * num_attrs + k]); } fprintf(stdout, "\n"); } - for (j = 0; j < num_attrs; ++j) { + for (int j = 0; j < num_attrs; ++j) { free(attr_names[j]); } free(attr_names); @@ -267,46 +256,42 @@ int cReadEdgeFace(int argc, char **argv) } } else if (OBJECT_IS_SET(i)) { - int num_df; - int * set_entry; - int * set_extra; - double *set_df; + int num_df; EXCHECK(ex_get_set_param(exoid, obj_types[i], ids[obj], &num_entries, &num_df), "Could not read set parameters.\n"); - set_entry = (int *)malloc(num_entries * sizeof(int)); - set_extra = (obj_types[i] != EX_NODE_SET && obj_types[i] != EX_ELEM_SET) - ? (int *)malloc(num_entries * sizeof(int)) - : 0; + int *set_entry = (int *)malloc(num_entries * sizeof(int)); + int *set_extra = (obj_types[i] != EX_NODE_SET && obj_types[i] != EX_ELEM_SET) + ? (int *)malloc(num_entries * sizeof(int)) + : 0; EXCHECK(ex_get_set(exoid, obj_types[i], ids[obj], set_entry, set_extra), "Could not read set.\n"); fprintf(stdout, "Entries: %3d Distribution factors: %3d\n", num_entries, num_df); if (set_extra) { - for (j = 0; j < num_entries; ++j) { + for (int j = 0; j < num_entries; ++j) { fprintf(stdout, " %2d %2d\n", set_entry[j], set_extra[j]); } } else { - for (j = 0; j < num_entries; ++j) { + for (int j = 0; j < num_entries; ++j) { fprintf(stdout, " %2d\n", set_entry[j]); } } free(set_entry); free(set_extra); - set_df = num_df ? (double *)malloc(num_df * sizeof(double)) : 0; + double *set_df = num_df ? (double *)malloc(num_df * sizeof(double)) : 0; if (set_df) { EXCHECK(ex_get_set_dist_fact(exoid, obj_types[i], ids[obj], set_df), "Could not read set distribution factors.\n"); fprintf(stdout, "\n Distribution factors:\n"); - for (j = 0; j < num_df; ++j) { + for (int j = 0; j < num_df; ++j) { fprintf(stdout, " %4.1f\n", set_df[j]); } free(set_df); } } else { /* object is map */ - int *map; switch (obj_types[i]) { case EX_NODE_MAP: num_entries = modelParams.num_nodes; break; case EX_EDGE_MAP: num_entries = modelParams.num_edge; break; @@ -316,9 +301,9 @@ int cReadEdgeFace(int argc, char **argv) } if (num_entries) { fprintf(stdout, "Entries: %3d\n :", num_entries); - map = (int *)malloc(num_entries * sizeof(int)); + int *map = (int *)malloc(num_entries * sizeof(int)); EXCHECK(ex_get_num_map(exoid, obj_types[i], ids[obj], map), "Could not read map.\n"); - for (j = 0; j < num_entries; ++j) { + for (int j = 0; j < num_entries; ++j) { fprintf(stdout, " %d", map[j]); } free(map); @@ -332,20 +317,19 @@ int cReadEdgeFace(int argc, char **argv) /* Read results variables */ if (((OBJECT_IS_BLOCK(i)) || (OBJECT_IS_SET(i))) && num_vars && num_timesteps > 0) { /* Print out all the time values to exercise get_var */ - entry_vals = (double *)malloc(num_entries * sizeof(double)); - for (j = 0; j < num_vars; ++j) { - int k; + double *entry_vals = (double *)malloc(num_entries * sizeof(double)); + for (int j = 0; j < num_vars; ++j) { if (!truth_tab[num_vars * obj + j]) { continue; } fprintf(stdout, " Variable: %s", var_names[j]); - for (ti = 1; ti <= num_timesteps; ++ti) { + for (int ti = 1; ti <= num_timesteps; ++ti) { EXCHECK(ex_get_var(exoid, ti, obj_types[i], 1 + j, ids[obj], num_entries, entry_vals), "Could not read variable values.\n"); fprintf(stdout, "\n @t%d ", ti); - for (k = 0; k < num_entries; ++k) { + for (int k = 0; k < num_entries; ++k) { fprintf(stdout, " %4.1f", entry_vals[k]); } } @@ -358,13 +342,14 @@ int cReadEdgeFace(int argc, char **argv) if (((OBJECT_IS_BLOCK(i)) || (OBJECT_IS_SET(i))) && num_vars && num_timesteps > 0) { /* Print out one element's time values to exercise get_var_time */ - entry_vals = (double *)malloc(num_timesteps * sizeof(double)); + double *entry_vals = (double *)malloc(num_timesteps * sizeof(double)); + int itmp[2]; EXCHECK(ex_inquire(exoid, obj_sizeinq[i], itmp, 0, 0), "Inquire failed.\n"); itmp[1] = 11; while (itmp[1] > itmp[0]) { itmp[1] /= 2; } - for (j = 0; j < num_vars; ++j) { + for (int j = 0; j < num_vars; ++j) { /* FIXME: This works for the dataset created by CreateEdgeFace, but not for any dataset in * general since * NULL truth table entries may mean the referenced elements don't have variable values. @@ -372,7 +357,7 @@ int cReadEdgeFace(int argc, char **argv) EXCHECK(ex_get_var_time(exoid, obj_types[i], j + 1, itmp[1], 1, num_timesteps, entry_vals), "Could not read variable over time.\n"); fprintf(stdout, " Variable over time: %s Entry: %3d ", var_names[j], itmp[1]); - for (ti = 1; ti <= num_timesteps; ++ti) { + for (int ti = 1; ti <= num_timesteps; ++ti) { fprintf(stdout, " @t%d: %4.1f", ti, entry_vals[ti - 1]); } fprintf(stdout, "\n"); @@ -381,7 +366,7 @@ int cReadEdgeFace(int argc, char **argv) } if (var_names) { - for (j = 0; j < num_vars; ++j) { + for (int j = 0; j < num_vars; ++j) { free(var_names[j]); } free(var_names); @@ -389,7 +374,7 @@ int cReadEdgeFace(int argc, char **argv) free(truth_tab); free(ids); - for (obj = 0; obj < nids; ++obj) { + for (int obj = 0; obj < nids; ++obj) { free(obj_names[obj]); } free(obj_names); diff --git a/packages/seacas/libraries/exodus/test/test-compress.dmp b/packages/seacas/libraries/exodus/test/test-compress.dmp index e5e0f6d36078..ebb02ec215f2 100644 --- a/packages/seacas/libraries/exodus/test/test-compress.dmp +++ b/packages/seacas/libraries/exodus/test/test-compress.dmp @@ -1,4 +1,3 @@ - time_whole:_DeflateLevel = 1 ; coordx:_DeflateLevel = 1 ; coordy:_DeflateLevel = 1 ; coordz:_DeflateLevel = 1 ; diff --git a/packages/seacas/libraries/exodus/test/testwt-assembly.c b/packages/seacas/libraries/exodus/test/testwt-assembly.c index a12f89bb9f85..3a01429061a2 100644 --- a/packages/seacas/libraries/exodus/test/testwt-assembly.c +++ b/packages/seacas/libraries/exodus/test/testwt-assembly.c @@ -40,7 +40,7 @@ #define EXCHECK(funcall) \ do { \ - error = (funcall); \ + int error = (funcall); \ printf("after %s, error = %d\n", TOSTRING(funcall), error); \ if (error != EX_NOERR) { \ fprintf(stderr, "Error calling %s\n", TOSTRING(funcall)); \ @@ -51,39 +51,29 @@ int main(int argc, char **argv) { - int exoid, num_dim, num_nodes, num_elem, num_elem_blk; - int num_node_sets, num_side_sets, num_assembly; - int error; - int CPU_word_size, IO_word_size; - - char *title = "This is a test"; - ex_opts(EX_VERBOSE); /* Specify compute and i/o word size */ - - CPU_word_size = 8; - IO_word_size = 8; + int CPU_word_size = 8; + int IO_word_size = 8; /* create EXODUS II file */ - - exoid = ex_create("test-assembly.exo", /* filename path */ - EX_CLOBBER, /* create mode */ - &CPU_word_size, /* CPU double word size in bytes */ - &IO_word_size); /* I/O double word size in bytes */ + int exoid = ex_create("test-assembly.exo", /* filename path */ + EX_CLOBBER, /* create mode */ + &CPU_word_size, /* CPU double word size in bytes */ + &IO_word_size); /* I/O double word size in bytes */ printf("after ex_create for test.exo, exoid = %d\n", exoid); printf(" cpu word size: %d io word size: %d\n", CPU_word_size, IO_word_size); /* initialize file with parameters */ + int num_dim = 3; + int num_nodes = 1; + int num_elem = 7; + int num_node_sets = 0; + int num_side_sets = 0; + int num_elem_blk = 7; + int num_assembly = 4; { - num_dim = 3; - num_nodes = 1; - num_elem = 7; - num_elem_blk = 7; - num_node_sets = 0; - num_side_sets = 0; - num_assembly = 4; - ex_init_params par = {.num_dim = num_dim, .num_nodes = num_nodes, .num_elem = num_elem, @@ -92,6 +82,7 @@ int main(int argc, char **argv) .num_side_sets = num_side_sets, .num_assembly = num_assembly}; + char *title = "This is a test"; ex_copy_string(par.title, title, MAX_LINE_LENGTH + 1); EXCHECK(ex_put_init_ext(exoid, &par)); } diff --git a/packages/seacas/libraries/exodus/test/testwt-groups.c b/packages/seacas/libraries/exodus/test/testwt-groups.c index b7df5cac40a3..874685c26a75 100644 --- a/packages/seacas/libraries/exodus/test/testwt-groups.c +++ b/packages/seacas/libraries/exodus/test/testwt-groups.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -35,65 +35,54 @@ int main(int argc, char **argv) { - int exoid, rootid, num_dim, num_nodes, num_elem, num_elem_blk; - int num_elem_in_block[10], num_nodes_per_elem[10]; - int num_face_in_sset[10], num_nodes_in_nset[10]; - int num_node_sets, num_side_sets, error; - int i, j, k, m, *elem_map, *connect; - int node_list[100], elem_list[100], side_list[100]; - int ebids[10], ssids[10], nsids[10]; - int num_qa_rec, num_info; - int num_glo_vars, num_nod_vars, num_ele_vars, num_sset_vars, num_nset_vars; - int *truth_tab; - int whole_time_step, num_time_steps; - int CPU_word_size, IO_word_size; - int prop_array[2]; - - float *glob_var_vals, *nodal_var_vals, *elem_var_vals; - float *sset_var_vals, *nset_var_vals; - float time_value; - float x[100], y[100], z[100]; - float attrib[1], dist_fact[100]; - char * coord_names[3], *qa_record[2][4], *info[3], *var_names[3]; - char * block_names[10], *nset_names[10], *sset_names[10]; - char * prop_names[2], *attrib_names[2]; - char * title = "This is a test"; - int group, num_groups = 2; - char * group_names[2]; + int num_elem_in_block[10], num_nodes_per_elem[10]; + int num_face_in_sset[10], num_nodes_in_nset[10]; + int node_list[100], elem_list[100], side_list[100]; + int ebids[10], ssids[10], nsids[10]; + int prop_array[2]; + + float x[100], y[100], z[100]; + float attrib[1], dist_fact[100]; + char *coord_names[3], *qa_record[2][4], *info[3], *var_names[3]; + char *block_names[10], *nset_names[10], *sset_names[10]; + char *prop_names[2], *attrib_names[2]; + char *title = "This is a test"; + char *group_names[2]; ex_opts(EX_VERBOSE | EX_ABORT); /* Specify compute and i/o word size */ - CPU_word_size = 0; /* sizeof(float) */ - IO_word_size = 4; /* (4 bytes) */ + int CPU_word_size = 0; /* sizeof(float) */ + int IO_word_size = 4; /* (4 bytes) */ /* create EXODUS II file */ - rootid = ex_create("test-groups.exo", /* filename path */ - EX_CLOBBER | EX_NETCDF4 | EX_NOCLASSIC, /* create mode */ - &CPU_word_size, /* CPU float word size in bytes */ - &IO_word_size); /* I/O float word size in bytes */ + int rootid = ex_create("test-groups.exo", /* filename path */ + EX_CLOBBER | EX_NETCDF4 | EX_NOCLASSIC, /* create mode */ + &CPU_word_size, /* CPU float word size in bytes */ + &IO_word_size); /* I/O float word size in bytes */ printf("after ex_create for test-groups.exo, rootid = %d\n", rootid); printf(" cpu word size: %d io word size: %d\n", CPU_word_size, IO_word_size); + int num_groups = 2; group_names[0] = "Group_0"; group_names[1] = "Group_1"; - for (group = 0; group < num_groups; group++) { + for (int group = 0; group < num_groups; group++) { /* Now create a group using netdf direct call... */ - exoid = ex_create_group(rootid, group_names[group]); + int exoid = ex_create_group(rootid, group_names[group]); printf("after ex_create_group for group 'Group1' in 'test-groups.exo', exoid = %d\n", exoid); /* initialize file with parameters */ - num_dim = 3; - num_nodes = 33; - num_elem = 7; - num_elem_blk = 7; - num_node_sets = 2; - num_side_sets = 5; + int num_dim = 3; + int num_nodes = 33; + int num_elem = 7; + int num_elem_blk = 7; + int num_node_sets = 2; + int num_side_sets = 5; - error = ex_put_init(exoid, title, num_dim, num_nodes, num_elem, num_elem_blk, num_node_sets, - num_side_sets); + int error = ex_put_init(exoid, title, num_dim, num_nodes, num_elem, num_elem_blk, num_node_sets, + num_side_sets); printf("after ex_put_init, error = %d\n", error); @@ -270,9 +259,9 @@ int main(int argc, char **argv) /* write element order map */ - elem_map = (int *)calloc(num_elem, sizeof(int)); + int *elem_map = (int *)calloc(num_elem, sizeof(int)); - for (i = 1; i <= num_elem; i++) { + for (int i = 1; i <= num_elem; i++) { elem_map[i - 1] = i; } @@ -391,7 +380,7 @@ int main(int argc, char **argv) } /* Write element block names */ - for (i = 0; i < 7; i++) { + for (int i = 0; i < 7; i++) { error = ex_put_name(exoid, EX_ELEM_BLOCK, ebids[i], block_names[i]); printf("after ex_put_names, error = %d\n", error); } @@ -465,11 +454,11 @@ int main(int argc, char **argv) /* write element connectivity */ - connect = (int *)calloc(8, sizeof(int)); - connect[0] = 1; - connect[1] = 2; - connect[2] = 3; - connect[3] = 4; + int *connect = (int *)calloc(8, sizeof(int)); + connect[0] = 1; + connect[1] = 2; + connect[2] = 3; + connect[3] = 4; error = ex_put_conn(exoid, EX_ELEM_BLOCK, ebids[0], connect); printf("after ex_put_elem_conn, error = %d\n", error); @@ -634,7 +623,7 @@ int main(int argc, char **argv) } attrib_names[0] = "THICKNESS"; - for (i = 0; i < 7; i++) { + for (int i = 0; i < 7; i++) { error = ex_put_attr_names(exoid, EX_ELEM_BLOCK, ebids[i], attrib_names); printf("after ex_put_elem_attr_names, error = %d\n", error); if (error) { @@ -1006,7 +995,7 @@ int main(int argc, char **argv) /* write QA records; test empty and just blank-filled records */ - num_qa_rec = 2; + int num_qa_rec = 2; qa_record[0][0] = "TESTWT"; qa_record[0][1] = "testwt"; @@ -1027,7 +1016,7 @@ int main(int argc, char **argv) /* write information records; test empty and just blank-filled records */ - num_info = 3; + int num_info = 3; info[0] = "This is the first information record."; info[1] = ""; @@ -1043,7 +1032,7 @@ int main(int argc, char **argv) /* write results variables parameters and names */ - num_glo_vars = 1; + int num_glo_vars = 1; var_names[0] = "glo_vars"; @@ -1061,7 +1050,7 @@ int main(int argc, char **argv) exit(-1); } - num_nod_vars = 2; + int num_nod_vars = 2; /* 12345678901234567890123456789012 */ var_names[0] = "node_variable_a_very_long_name_0"; var_names[1] = "nod_var1"; @@ -1080,7 +1069,7 @@ int main(int argc, char **argv) exit(-1); } - num_ele_vars = 3; + int num_ele_vars = 3; /* 0 1 2 3 */ /* 12345678901234567890123456789012 */ var_names[0] = "this_variable_name_is_short"; @@ -1101,9 +1090,8 @@ int main(int argc, char **argv) exit(-1); } + int num_nset_vars = 3; { - num_nset_vars = 3; - var_names[0] = "ns_var0"; var_names[1] = "ns_var1"; var_names[2] = "ns_var2"; @@ -1123,9 +1111,8 @@ int main(int argc, char **argv) } } + int num_sset_vars = 3; { - num_sset_vars = 3; - var_names[0] = "ss_var0"; var_names[1] = "ss_var1"; var_names[2] = "ss_var2"; @@ -1147,12 +1134,14 @@ int main(int argc, char **argv) /* write element variable truth table */ - truth_tab = (int *)calloc((num_elem_blk * num_ele_vars), sizeof(int)); + int *truth_tab = (int *)calloc((num_elem_blk * num_ele_vars), sizeof(int)); - k = 0; - for (i = 0; i < num_elem_blk; i++) { - for (j = 0; j < num_ele_vars; j++) { - truth_tab[k++] = 1; + { + int k = 0; + for (int i = 0; i < num_elem_blk; i++) { + for (int j = 0; j < num_ele_vars; j++) { + truth_tab[k++] = 1; + } } } @@ -1172,17 +1161,17 @@ int main(int argc, char **argv) * obviously the analysis code will populate these arrays */ - whole_time_step = 1; - num_time_steps = 10; + int whole_time_step = 1; + int num_time_steps = 10; - glob_var_vals = (float *)calloc(num_glo_vars, CPU_word_size); - nodal_var_vals = (float *)calloc(num_nodes, CPU_word_size); - elem_var_vals = (float *)calloc(4, CPU_word_size); - sset_var_vals = (float *)calloc(10, CPU_word_size); - nset_var_vals = (float *)calloc(10, CPU_word_size); + float *glob_var_vals = (float *)calloc(num_glo_vars, CPU_word_size); + float *nodal_var_vals = (float *)calloc(num_nodes, CPU_word_size); + float *elem_var_vals = (float *)calloc(4, CPU_word_size); + float *sset_var_vals = (float *)calloc(10, CPU_word_size); + float *nset_var_vals = (float *)calloc(10, CPU_word_size); - for (i = 0; i < num_time_steps; i++) { - time_value = (float)(i + 1) / 100.; + for (int i = 0; i < num_time_steps; i++) { + float time_value = (float)(i + 1) / 100.; /* write time value */ @@ -1196,7 +1185,7 @@ int main(int argc, char **argv) /* write global variables */ - for (j = 0; j < num_glo_vars; j++) { + for (int j = 0; j < num_glo_vars; j++) { glob_var_vals[j] = (float)(j + 2) * time_value; } @@ -1210,8 +1199,8 @@ int main(int argc, char **argv) /* write nodal variables */ - for (k = 1; k <= num_nod_vars; k++) { - for (j = 0; j < num_nodes; j++) { + for (int k = 1; k <= num_nod_vars; k++) { + for (int j = 0; j < num_nodes; j++) { nodal_var_vals[j] = (float)k + ((float)(j + 1) * time_value); } @@ -1225,9 +1214,9 @@ int main(int argc, char **argv) /* write element variables */ - for (k = 1; k <= num_ele_vars; k++) { - for (j = 0; j < num_elem_blk; j++) { - for (m = 0; m < num_elem_in_block[j]; m++) { + for (int k = 1; k <= num_ele_vars; k++) { + for (int j = 0; j < num_elem_blk; j++) { + for (int m = 0; m < num_elem_in_block[j]; m++) { elem_var_vals[m] = (float)(k + 1) + (float)(j + 2) + ((float)(m + 1) * time_value); /* printf("elem_var_vals[%d]: %f\n",m,elem_var_vals[m]); */ } @@ -1243,9 +1232,9 @@ int main(int argc, char **argv) /* write sideset variables */ - for (k = 1; k <= num_sset_vars; k++) { - for (j = 0; j < num_side_sets; j++) { - for (m = 0; m < num_face_in_sset[j]; m++) { + for (int k = 1; k <= num_sset_vars; k++) { + for (int j = 0; j < num_side_sets; j++) { + for (int m = 0; m < num_face_in_sset[j]; m++) { sset_var_vals[m] = (float)(k + 2) + (float)(j + 3) + ((float)(m + 1) * time_value); /* printf("sset_var_vals[%d]: %f\n",m,sset_var_vals[m]); */ } @@ -1261,9 +1250,9 @@ int main(int argc, char **argv) /* write nodeset variables */ - for (k = 1; k <= num_nset_vars; k++) { - for (j = 0; j < num_node_sets; j++) { - for (m = 0; m < num_nodes_in_nset[j]; m++) { + for (int k = 1; k <= num_nset_vars; k++) { + for (int j = 0; j < num_node_sets; j++) { + for (int m = 0; m < num_nodes_in_nset[j]; m++) { nset_var_vals[m] = (float)(k + 3) + (float)(j + 4) + ((float)(m + 1) * time_value); /* printf("nset_var_vals[%d]: %f\n",m,nset_var_vals[m]); */ } diff --git a/packages/seacas/libraries/exodus/test/testwt-localization.C b/packages/seacas/libraries/exodus/test/testwt-localization.C index 63322c1dfee0..8973c7dd62db 100644 --- a/packages/seacas/libraries/exodus/test/testwt-localization.C +++ b/packages/seacas/libraries/exodus/test/testwt-localization.C @@ -139,7 +139,7 @@ int main() coord_names[1] = "ycoor"; coord_names[2] = "zcoor"; - EXCHECK(ex_put_coord_names(exoid, (char **)coord_names)); + EXCHECK(ex_put_coord_names(exoid, (char *const *)coord_names)); std::vector blocks(num_elem_blk); @@ -204,7 +204,7 @@ int main() info[1] = ""; info[2] = " "; - EXCHECK(ex_put_info(exoid, num_info, (char **)info)); + EXCHECK(ex_put_info(exoid, num_info, (char *const *)info)); /* write results variables parameters and names */ const int num_glo_vars = 1; @@ -213,7 +213,7 @@ int main() var_names[0] = "glo_vars"; EXCHECK(ex_put_variable_param(exoid, EX_GLOBAL, num_glo_vars)); - EXCHECK(ex_put_variable_names(exoid, EX_GLOBAL, num_glo_vars, (char **)var_names)); + EXCHECK(ex_put_variable_names(exoid, EX_GLOBAL, num_glo_vars, (char *const *)var_names)); const int num_nod_vars = 2; /* 12345678901234567890123456789012 */ @@ -221,7 +221,7 @@ int main() var_names[1] = "nod_var1"; EXCHECK(ex_put_variable_param(exoid, EX_NODAL, num_nod_vars)); - EXCHECK(ex_put_variable_names(exoid, EX_NODAL, num_nod_vars, (char **)var_names)); + EXCHECK(ex_put_variable_names(exoid, EX_NODAL, num_nod_vars, (char *const *)var_names)); const int num_ele_vars = 3; /* 0 1 2 3 */ @@ -231,7 +231,7 @@ int main() var_names[2] = "this_variable_name_is_tooooo_long"; EXCHECK(ex_put_variable_param(exoid, EX_ELEM_BLOCK, num_ele_vars)); - EXCHECK(ex_put_variable_names(exoid, EX_ELEM_BLOCK, num_ele_vars, (char **)var_names)); + EXCHECK(ex_put_variable_names(exoid, EX_ELEM_BLOCK, num_ele_vars, (char *const *)var_names)); // for each time step, write the analysis results; // the code below fills the arrays glob_var_vals, diff --git a/packages/seacas/libraries/exodus/test/testwt-nface-nside.c b/packages/seacas/libraries/exodus/test/testwt-nface-nside.c index e92487784c8e..2e5ec3d90586 100644 --- a/packages/seacas/libraries/exodus/test/testwt-nface-nside.c +++ b/packages/seacas/libraries/exodus/test/testwt-nface-nside.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -19,38 +19,30 @@ int main(int argc, char **argv) { - int exoid, num_dim, num_nodes, num_elem, num_elem_blk; - int num_elem_in_block[10], num_total_nodes_per_blk[10]; - int num_face_in_block[10], num_total_faces_per_blk[10]; - int num_node_sets, error; - int i, j, k, m, *connect; - int bids[10], nnpe[10]; - int num_qa_rec, num_info; - int num_glo_vars, num_nod_vars, num_ele_vars; - int *truth_tab; - int whole_time_step, num_time_steps; - int CPU_word_size, IO_word_size; - - float *glob_var_vals, *nodal_var_vals, *elem_var_vals; - float time_value; - float x[100], y[100], z[100]; - char * coord_names[3], *qa_record[2][4], *info[3], *var_names[3]; - char * block_names[10], *nset_names[10], *sset_names[10]; - char * prop_names[2], *attrib_names[2]; - char * title = "This is a test"; + int num_elem_in_block[10], num_total_nodes_per_blk[10]; + int num_face_in_block[10], num_total_faces_per_blk[10]; + int error; + int i, j; + int bids[10], nnpe[10]; + + float x[100], y[100], z[100]; + char *coord_names[3], *qa_record[2][4], *info[3], *var_names[3]; + char *block_names[10], *nset_names[10], *sset_names[10]; + char *prop_names[2], *attrib_names[2]; + char *title = "This is a test"; ex_opts(EX_VERBOSE | EX_ABORT); /* Specify compute and i/o word size */ - CPU_word_size = 0; /* sizeof(float) */ - IO_word_size = 4; /* (4 bytes) */ + int CPU_word_size = 0; /* sizeof(float) */ + int IO_word_size = 4; /* (4 bytes) */ /* create EXODUS II file */ - exoid = ex_create("test-nfaced.exo", /* filename path */ - EX_CLOBBER, /* create mode */ - &CPU_word_size, /* CPU float word size in bytes */ - &IO_word_size); /* I/O float word size in bytes */ + int exoid = ex_create("test-nfaced.exo", /* filename path */ + EX_CLOBBER, /* create mode */ + &CPU_word_size, /* CPU float word size in bytes */ + &IO_word_size); /* I/O float word size in bytes */ printf("after ex_create for test.exo, exoid = %d\n", exoid); printf(" cpu word size: %d io word size: %d\n", CPU_word_size, IO_word_size); @@ -58,11 +50,11 @@ int main(int argc, char **argv) { ex_init_params par; - num_dim = 3; - num_nodes = 14; - num_elem = 1; - num_elem_blk = 1; - num_node_sets = 0; + int num_dim = 3; + int num_nodes = 14; + int num_elem = 1; + int num_elem_blk = 1; + int num_node_sets = 0; ex_copy_string(par.title, title, MAX_LINE_LENGTH + 1); par.num_dim = num_dim; @@ -174,7 +166,7 @@ int main(int argc, char **argv) /* write face connectivity */ - connect = (int *)calloc(num_total_nodes_per_blk[0], sizeof(int)); + int *connect = (int *)calloc(num_total_nodes_per_blk[0], sizeof(int)); i = 0; j = 0; @@ -325,7 +317,7 @@ int main(int argc, char **argv) } /* write QA records; test empty and just blank-filled records */ - num_qa_rec = 2; + int num_qa_rec = 2; qa_record[0][0] = "TESTWT-NFACED"; qa_record[0][1] = "testwt-nfaced"; @@ -345,7 +337,7 @@ int main(int argc, char **argv) } /* write information records; test empty and just blank-filled records */ - num_info = 3; + int num_info = 3; info[0] = "This is the first information record."; info[1] = ""; @@ -360,7 +352,7 @@ int main(int argc, char **argv) } /* write results variables parameters and names */ - num_glo_vars = 1; + int num_glo_vars = 1; var_names[0] = "glo_vars"; @@ -378,7 +370,7 @@ int main(int argc, char **argv) exit(-1); } - num_nod_vars = 2; + int num_nod_vars = 2; /* 12345678901234567890123456789012 */ var_names[0] = "node_variable_a_very_long_name_0"; var_names[1] = EX_NODE_SET; @@ -397,7 +389,7 @@ int main(int argc, char **argv) exit(-1); } - num_ele_vars = 3; + int num_ele_vars = 3; var_names[0] = "ele_var0"; var_names[1] = "ele_var1"; @@ -418,9 +410,9 @@ int main(int argc, char **argv) } /* write element variable truth table */ - truth_tab = (int *)calloc((num_elem_blk * num_ele_vars), sizeof(int)); + int *truth_tab = (int *)calloc((num_elem_blk * num_ele_vars), sizeof(int)); - k = 0; + int k = 0; for (i = 0; i < num_elem_blk; i++) { for (j = 0; j < num_ele_vars; j++) { truth_tab[k++] = 1; @@ -443,15 +435,15 @@ int main(int argc, char **argv) * obviously the analysis code will populate these arrays */ - whole_time_step = 1; - num_time_steps = 10; + int whole_time_step = 1; + int num_time_steps = 10; - glob_var_vals = (float *)calloc(num_glo_vars, CPU_word_size); - nodal_var_vals = (float *)calloc(num_nodes, CPU_word_size); - elem_var_vals = (float *)calloc(8, CPU_word_size); + float *glob_var_vals = (float *)calloc(num_glo_vars, CPU_word_size); + float *nodal_var_vals = (float *)calloc(num_nodes, CPU_word_size); + float *elem_var_vals = (float *)calloc(8, CPU_word_size); for (i = 0; i < num_time_steps; i++) { - time_value = (float)(i + 1) / 100.; + float time_value = (float)(i + 1) / 100.; /* write time value */ error = ex_put_time(exoid, whole_time_step, &time_value); @@ -492,7 +484,7 @@ int main(int argc, char **argv) /* write element variables */ for (k = 1; k <= num_ele_vars; k++) { for (j = 0; j < num_elem_blk; j++) { - for (m = 0; m < num_elem_in_block[j]; m++) { + for (int m = 0; m < num_elem_in_block[j]; m++) { elem_var_vals[m] = (float)(k + 1) + (float)(j + 2) + ((float)(m + 1) * time_value); /* printf("elem_var_vals[%d]: %f\n",m,elem_var_vals[m]); */ } diff --git a/packages/seacas/libraries/exodus/test/testwt.c b/packages/seacas/libraries/exodus/test/testwt.c index 12b70f209850..1e8b0f64dc8e 100644 --- a/packages/seacas/libraries/exodus/test/testwt.c +++ b/packages/seacas/libraries/exodus/test/testwt.c @@ -38,28 +38,16 @@ int main(int argc, char **argv) { - int exoid, num_dim, num_nodes, num_elem, num_elem_blk; - int num_face_in_sset[10], num_nodes_in_nset[10]; - int num_node_sets, num_side_sets, error; - int i, j, k, m, *elem_map, *connect; - int node_list[100], elem_list[100], side_list[100]; - int ssids[10], nsids[10]; - int num_qa_rec, num_info; - int num_glo_vars, num_nod_vars, num_ele_vars, num_sset_vars, num_nset_vars; - int *truth_tab; - int whole_time_step, num_time_steps; - int CPU_word_size, IO_word_size; - int prop_array[2]; - - float *glob_var_vals, *nodal_var_vals, *elem_var_vals; - float *sset_var_vals, *nset_var_vals; - float time_value; - float x[100], y[100], z[100]; - float attrib[1], dist_fact[100]; - char * coord_names[3], *qa_record[2][4], *info[3], *var_names[3]; - char * block_names[10], *nset_names[10], *sset_names[10]; - char * prop_names[2], *attrib_names[2]; - char * title = "This is a test"; + int num_face_in_sset[10], num_nodes_in_nset[10]; + int node_list[100], elem_list[100], side_list[100]; + int ssids[10], nsids[10]; + + float x[100], y[100], z[100]; + float attrib[1], dist_fact[100]; + char *coord_names[3], *qa_record[2][4], *info[3], *var_names[3]; + char *block_names[10], *nset_names[10], *sset_names[10]; + char *prop_names[2], *attrib_names[2]; + char *title = "This is a test"; struct ex_block blocks[10]; @@ -67,29 +55,28 @@ int main(int argc, char **argv) /* Specify compute and i/o word size */ - CPU_word_size = 0; /* sizeof(float) */ - IO_word_size = 4; /* (4 bytes) */ + int CPU_word_size = 0; /* sizeof(float) */ + int IO_word_size = 4; /* (4 bytes) */ /* create EXODUS II file */ - - exoid = ex_create("test.exo", /* filename path */ - EX_CLOBBER, /* create mode */ - &CPU_word_size, /* CPU float word size in bytes */ - &IO_word_size); /* I/O float word size in bytes */ + int exoid = ex_create("test.exo", /* filename path */ + EX_CLOBBER, /* create mode */ + &CPU_word_size, /* CPU float word size in bytes */ + &IO_word_size); /* I/O float word size in bytes */ printf("after ex_create for test.exo, exoid = %d\n", exoid); printf(" cpu word size: %d io word size: %d\n", CPU_word_size, IO_word_size); /* initialize file with parameters */ - num_dim = 3; - num_nodes = 33; - num_elem = 7; - num_elem_blk = 7; - num_node_sets = 2; - num_side_sets = 5; + int num_dim = 3; + int num_nodes = 33; + int num_elem = 7; + int num_elem_blk = 7; + int num_node_sets = 2; + int num_side_sets = 5; - error = ex_put_init(exoid, title, num_dim, num_nodes, num_elem, num_elem_blk, num_node_sets, - num_side_sets); + int error = ex_put_init(exoid, title, num_dim, num_nodes, num_elem, num_elem_blk, num_node_sets, + num_side_sets); printf("after ex_put_init, error = %d\n", error); @@ -266,9 +253,8 @@ int main(int argc, char **argv) /* write element order map */ - elem_map = (int *)calloc(num_elem, sizeof(int)); - - for (i = 1; i <= num_elem; i++) { + int *elem_map = (int *)calloc(num_elem, sizeof(int)); + for (int i = 1; i <= num_elem; i++) { elem_map[i - 1] = i; } @@ -283,7 +269,7 @@ int main(int argc, char **argv) free(elem_map); /* write element block parameters */ - for (i = 0; i < 10; i++) { + for (int i = 0; i < 10; i++) { blocks[i].type = EX_ELEM_BLOCK; blocks[i].id = 0; blocks[i].num_entry = 0; @@ -356,7 +342,7 @@ int main(int argc, char **argv) } /* Write element block names */ - for (i = 0; i < num_elem_blk; i++) { + for (int i = 0; i < num_elem_blk; i++) { error = ex_put_name(exoid, EX_ELEM_BLOCK, blocks[i].id, block_names[i]); printf("after ex_put_names, error = %d\n", error); } @@ -430,11 +416,11 @@ int main(int argc, char **argv) /* write element connectivity */ - connect = (int *)calloc(8, sizeof(int)); - connect[0] = 1; - connect[1] = 2; - connect[2] = 3; - connect[3] = 4; + int *connect = (int *)calloc(8, sizeof(int)); + connect[0] = 1; + connect[1] = 2; + connect[2] = 3; + connect[3] = 4; error = ex_put_conn(exoid, EX_ELEM_BLOCK, blocks[0].id, connect, NULL, NULL); printf("after ex_put_elem_conn, error = %d\n", error); @@ -598,7 +584,7 @@ int main(int argc, char **argv) } attrib_names[0] = "THICKNESS"; - for (i = 0; i < num_elem_blk; i++) { + for (int i = 0; i < num_elem_blk; i++) { error = ex_put_attr_names(exoid, EX_ELEM_BLOCK, blocks[i].id, attrib_names); printf("after ex_put_elem_attr_names, error = %d\n", error); if (error) { @@ -706,6 +692,7 @@ int main(int argc, char **argv) exit(-1); } + int prop_array[2]; prop_array[0] = 1000; prop_array[1] = 2000; @@ -970,7 +957,7 @@ int main(int argc, char **argv) /* write QA records; test empty and just blank-filled records */ - num_qa_rec = 2; + int num_qa_rec = 2; qa_record[0][0] = "TESTWT"; qa_record[0][1] = "testwt"; @@ -991,7 +978,7 @@ int main(int argc, char **argv) /* write information records; test empty and just blank-filled records */ - num_info = 3; + int num_info = 3; info[0] = "This is the first information record."; info[1] = ""; @@ -1007,7 +994,7 @@ int main(int argc, char **argv) /* write results variables parameters and names */ - num_glo_vars = 1; + int num_glo_vars = 1; var_names[0] = "glo_vars"; @@ -1025,7 +1012,7 @@ int main(int argc, char **argv) exit(-1); } - num_nod_vars = 2; + int num_nod_vars = 2; /* 12345678901234567890123456789012 */ var_names[0] = "node_variable_a_very_long_name_0"; var_names[1] = "nod_var1"; @@ -1044,7 +1031,7 @@ int main(int argc, char **argv) exit(-1); } - num_ele_vars = 3; + int num_ele_vars = 3; /* 0 1 2 3 */ /* 12345678901234567890123456789012 */ var_names[0] = "this_variable_name_is_short"; @@ -1065,9 +1052,8 @@ int main(int argc, char **argv) exit(-1); } + int num_nset_vars = 3; { - num_nset_vars = 3; - var_names[0] = "ns_var0"; var_names[1] = "ns_var1"; var_names[2] = "ns_var2"; @@ -1087,9 +1073,8 @@ int main(int argc, char **argv) } } + int num_sset_vars = 3; { - num_sset_vars = 3; - var_names[0] = "ss_var0"; var_names[1] = "ss_var1"; var_names[2] = "ss_var2"; @@ -1110,12 +1095,11 @@ int main(int argc, char **argv) } /* write element variable truth table */ + int *truth_tab = (int *)calloc((num_elem_blk * num_ele_vars), sizeof(int)); - truth_tab = (int *)calloc((num_elem_blk * num_ele_vars), sizeof(int)); - - k = 0; - for (i = 0; i < num_elem_blk; i++) { - for (j = 0; j < num_ele_vars; j++) { + int k = 0; + for (int i = 0; i < num_elem_blk; i++) { + for (int j = 0; j < num_ele_vars; j++) { truth_tab[k++] = 1; } } @@ -1136,17 +1120,17 @@ int main(int argc, char **argv) * obviously the analysis code will populate these arrays */ - whole_time_step = 1; - num_time_steps = 10; + int whole_time_step = 1; + int num_time_steps = 10; - glob_var_vals = (float *)calloc(num_glo_vars, CPU_word_size); - nodal_var_vals = (float *)calloc(num_nodes, CPU_word_size); - elem_var_vals = (float *)calloc(4, CPU_word_size); - sset_var_vals = (float *)calloc(10, CPU_word_size); - nset_var_vals = (float *)calloc(10, CPU_word_size); + float *glob_var_vals = (float *)calloc(num_glo_vars, CPU_word_size); + float *nodal_var_vals = (float *)calloc(num_nodes, CPU_word_size); + float *elem_var_vals = (float *)calloc(4, CPU_word_size); + float *sset_var_vals = (float *)calloc(10, CPU_word_size); + float *nset_var_vals = (float *)calloc(10, CPU_word_size); - for (i = 0; i < num_time_steps; i++) { - time_value = (float)(i + 1) / 100.; + for (int i = 0; i < num_time_steps; i++) { + float time_value = (float)(i + 1) / 100.; /* write time value */ @@ -1160,7 +1144,7 @@ int main(int argc, char **argv) /* write global variables */ - for (j = 0; j < num_glo_vars; j++) { + for (int j = 0; j < num_glo_vars; j++) { glob_var_vals[j] = (float)(j + 2) * time_value; } @@ -1174,8 +1158,8 @@ int main(int argc, char **argv) /* write nodal variables */ - for (k = 1; k <= num_nod_vars; k++) { - for (j = 0; j < num_nodes; j++) { + for (int k = 1; k <= num_nod_vars; k++) { + for (int j = 0; j < num_nodes; j++) { nodal_var_vals[j] = (float)k + ((float)(j + 1) * time_value); } @@ -1189,9 +1173,9 @@ int main(int argc, char **argv) /* write element variables */ - for (k = 1; k <= num_ele_vars; k++) { - for (j = 0; j < num_elem_blk; j++) { - for (m = 0; m < blocks[j].num_entry; m++) { + for (int k = 1; k <= num_ele_vars; k++) { + for (int j = 0; j < num_elem_blk; j++) { + for (int m = 0; m < blocks[j].num_entry; m++) { elem_var_vals[m] = (float)(k + 1) + (float)(j + 2) + ((float)(m + 1) * time_value); /* printf("elem_var_vals[%d]: %f\n",m,elem_var_vals[m]); */ } @@ -1207,9 +1191,9 @@ int main(int argc, char **argv) /* write sideset variables */ - for (k = 1; k <= num_sset_vars; k++) { - for (j = 0; j < num_side_sets; j++) { - for (m = 0; m < num_face_in_sset[j]; m++) { + for (int k = 1; k <= num_sset_vars; k++) { + for (int j = 0; j < num_side_sets; j++) { + for (int m = 0; m < num_face_in_sset[j]; m++) { sset_var_vals[m] = (float)(k + 2) + (float)(j + 3) + ((float)(m + 1) * time_value); /* printf("sset_var_vals[%d]: %f\n",m,sset_var_vals[m]); */ } @@ -1225,9 +1209,9 @@ int main(int argc, char **argv) /* write nodeset variables */ - for (k = 1; k <= num_nset_vars; k++) { - for (j = 0; j < num_node_sets; j++) { - for (m = 0; m < num_nodes_in_nset[j]; m++) { + for (int k = 1; k <= num_nset_vars; k++) { + for (int j = 0; j < num_node_sets; j++) { + for (int m = 0; m < num_nodes_in_nset[j]; m++) { nset_var_vals[m] = (float)(k + 3) + (float)(j + 4) + ((float)(m + 1) * time_value); /* printf("nset_var_vals[%d]: %f\n",m,nset_var_vals[m]); */ } diff --git a/packages/seacas/libraries/exodus_for/src/exo_jack.c b/packages/seacas/libraries/exodus_for/src/exo_jack.c index afb96214acae..75a60222ba18 100644 --- a/packages/seacas/libraries/exodus_for/src/exo_jack.c +++ b/packages/seacas/libraries/exodus_for/src/exo_jack.c @@ -1,5 +1,5 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -80,23 +80,21 @@ static void ex_fcdcpy(char *fstring, /* output string to be blank-filled */ int fslen, /* length of output string */ char *sstring) { /* input string, null-terminated */ - int i, len; - if (sstring != NULL) { - len = strlen(sstring); + int len = strlen(sstring); if (len > fslen) { len = fslen; } - for (i = 0; i < len; i++) { + for (int i = 0; i < len; i++) { *(fstring + i) = *(sstring + i); } - for (i = len; i < fslen; i++) { + for (int i = len; i < fslen; i++) { *(fstring + i) = ' '; } } else { - for (i = 0; i < fslen; i++) { + for (int i = 0; i < fslen; i++) { *(fstring + i) = ' '; } } @@ -263,14 +261,12 @@ void F2C(exgini, EXGINI)(int *idexo, char *title, void_int *num_dim, void_int *n void_int *num_elem, void_int *num_elem_blk, void_int *num_node_sets, void_int *num_side_sets, int *ierr, int titlelen) { - int slen; - char *name; - - *ierr = 0; - slen = MAX_LINE_LENGTH; /* max line size */ + *ierr = 0; + int slen = MAX_LINE_LENGTH; /* max line size */ if (titlelen != MAX_LINE_LENGTH) { slen = titlelen; } + char *name; if (!(name = malloc((slen + 1) * sizeof(char)))) { *ierr = EX_MEMFAIL; return; @@ -294,15 +290,13 @@ void F2C(expqa, EXPQA)(int *idexo, int *num_qa_records, char *qa_record, int *ie char errmsg[MAX_ERR_LENGTH]; char **sptr; /* internal string pointer array for malloc * use */ - int i, ii, iii, slen, alen; + *ierr = 0; /* default no error */ - *ierr = 0; /* default no error */ - - slen = MAX_STR_LENGTH; /* max str size */ + int slen = MAX_STR_LENGTH; /* max str size */ if (qa_recordlen != MAX_STR_LENGTH) { slen = qa_recordlen; } - alen = 4; /* qa records are 4 strings deep */ + int alen = 4; /* qa records are 4 strings deep */ /* Allocate space for the name ptr array */ if (!(sptr = malloc(((*num_qa_records) * alen + 1) * sizeof(char *)))) { @@ -314,9 +308,9 @@ void F2C(expqa, EXPQA)(int *idexo, int *num_qa_records, char *qa_record, int *ie * ptr into str ptr array, and Copy Fortran qa records to staging * space */ - iii = 0; /* offset counter */ - for (i = 0; i < *num_qa_records; i++) { - for (ii = 0; ii < alen; ii++) { + int iii = 0; /* offset counter */ + for (int i = 0; i < *num_qa_records; i++) { + for (int ii = 0; ii < alen; ii++) { *(sptr + iii) = malloc((slen + 1) * sizeof(char)); if (*(sptr + iii) == 0) { free(sptr); /* free up array ptr space */ @@ -339,8 +333,8 @@ void F2C(expqa, EXPQA)(int *idexo, int *num_qa_records, char *qa_record, int *ie /* Free up the space we used */ iii = 0; - for (i = 0; i < *num_qa_records; i++) { - for (ii = 0; ii < alen; ii++) { + for (int i = 0; i < *num_qa_records; i++) { + for (int ii = 0; ii < alen; ii++) { free(*(sptr + iii)); /* First free up string space */ iii++; } @@ -354,7 +348,6 @@ void F2C(expqa, EXPQA)(int *idexo, int *num_qa_records, char *qa_record, int *ie */ void F2C(exgqa, EXGQA)(int *idexo, char *qa_record, int *ierr, int qa_recordlen) { - int num_qa_records; char **sptr; /* internal string pointer array for malloc * use */ int i, ii, iii, slen, alen; @@ -368,7 +361,7 @@ void F2C(exgqa, EXGQA)(int *idexo, char *qa_record, int *ierr, int qa_recordlen) alen = 4; /* qa records are 4 strings deep */ /* do Exodus C call to find out how many qa records are avail */ - num_qa_records = ex_inquire_int(*idexo, EX_INQ_QA); + int num_qa_records = ex_inquire_int(*idexo, EX_INQ_QA); if (num_qa_records < 0) { *ierr = EX_FATAL; return; diff --git a/packages/seacas/libraries/ioss/Doxyfile b/packages/seacas/libraries/ioss/Doxyfile index 318d781f032d..09d0cb240484 100644 --- a/packages/seacas/libraries/ioss/Doxyfile +++ b/packages/seacas/libraries/ioss/Doxyfile @@ -1,4 +1,4 @@ -# Doxyfile 1.9.0 +# Doxyfile 1.9.2 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. @@ -93,14 +93,6 @@ ALLOW_UNICODE_NAMES = NO OUTPUT_LANGUAGE = English -# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all generated output in the proper direction. -# Possible values are: None, LTR, RTL and Context. -# The default value is: None. - -OUTPUT_TEXT_DIRECTION = None - # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. @@ -248,16 +240,16 @@ TAB_SIZE = 1 # the documentation. An alias has the form: # name=value # For example adding -# "sideeffect=@par Side Effects:\n" +# "sideeffect=@par Side Effects:^^" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines (in the resulting output). You can put ^^ in the value part of an -# alias to insert a newline as if a physical newline was in the original file. -# When you need a literal { or } or , in the value part of an alias you have to -# escape them by means of a backslash (\), this can lead to conflicts with the -# commands \{ and \} for these it is advised to use the version @{ and @} or use -# a double escape (\\{ and \\}) +# "Side Effects:". Note that you cannot put \n's in the value part of an alias +# to insert newlines (in the resulting output). You can put ^^ in the value part +# of an alias to insert a newline as if a physical newline was in the original +# file. When you need a literal { or } or , in the value part of an alias you +# have to escape them by means of a backslash (\), this can lead to conflicts +# with the commands \{ and \} for these it is advised to use the version @{ and +# @} or use a double escape (\\{ and \\}) ALIASES = func{1}=\1 \ fparam{1}=\1 \ @@ -265,8 +257,8 @@ ALIASES = func{1}=\1 \ "comment{1}=/* \1 */" \ "co{1}=/* \1 */" \ "undoc=\xrefitem undoc \"Undocumented\" \"Undocumented\"" \ - rst="\code" \ - endrst="\endcode" + rst=\code \ + endrst=\endcode # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources # only. Doxygen will then generate output that is more tailored for C. For @@ -309,8 +301,8 @@ OPTIMIZE_OUTPUT_SLICE = NO # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by doxygen: IDL, Java, JavaScript, -# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, -# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, +# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: # FortranFree, unknown formatted Fortran: Fortran. In the later case the parser # tries to guess whether the code is fixed or free formatted code, this is the # default for Fortran type files). For instance to make doxygen treat .inc files @@ -463,7 +455,7 @@ LOOKUP_CACHE_SIZE = 0 # than 0 to get more control over the balance between CPU load and processing # speed. At this moment only the input processing can be done using multiple # threads. Since this is still an experimental feature the default is set to 1, -# which efficively disables parallel processing. Please report any issues you +# which effectively disables parallel processing. Please report any issues you # encounter. Generating dot graphs in parallel is controlled by the # DOT_NUM_THREADS setting. # Minimum value: 0, maximum value: 32, default value: 1. @@ -607,6 +599,12 @@ HIDE_SCOPE_NAMES = NO HIDE_COMPOUND_REFERENCE= NO +# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class +# will show which file needs to be included to use the class. +# The default value is: YES. + +SHOW_HEADERFILE = YES + # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. @@ -764,7 +762,8 @@ FILE_VERSION_FILTER = # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. +# will be used as the name of the layout file. See also section "Changing the +# layout of pages" for information. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE @@ -810,18 +809,26 @@ WARNINGS = YES WARN_IF_UNDOCUMENTED = NO # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. +# potential errors in the documentation, such as documenting some parameters in +# a documented function twice, or documenting parameters that don't exist or +# using markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES +# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete +# function parameter documentation. If set to NO, doxygen will accept that some +# parameters have no documentation without warning. +# The default value is: YES. + +WARN_IF_INCOMPLETE_DOC = YES + # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. If -# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# value. If set to NO, doxygen will only warn about wrong parameter +# documentation, but not about the absence of documentation. If EXTRACT_ALL is +# set to YES then this flag will automatically be disabled. See also +# WARN_IF_INCOMPLETE_DOC # The default value is: NO. WARN_NO_PARAMDOC = YES @@ -885,10 +892,10 @@ INPUT_ENCODING = UTF-8 # # If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, # *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), -# *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, *.vhdl, -# *.ucf, *.qsf and *.ice. +# *.hh, *.hxx, *.hpp, *.h++, *.l, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, +# *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C +# comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, +# *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.C \ *.h @@ -1214,7 +1221,7 @@ HTML_EXTRA_FILES = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see +# this color. Hue is specified as an angle on a color-wheel, see # https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. @@ -1224,7 +1231,7 @@ HTML_EXTRA_FILES = HTML_COLORSTYLE_HUE = 224 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A +# in the HTML output. For a value of 0 the output will use gray-scales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1331,8 +1338,12 @@ DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: -# https://www.microsoft.com/en-us/download/details.aspx?id=21138) on Windows. +# on Windows. In the beginning of 2021 Microsoft took the original page, with +# a.o. the download links, offline the HTML help workshop was already many years +# in maintenance mode). You can download the HTML help workshop from the web +# archives at Installation executable (see: +# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo +# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML @@ -1491,16 +1502,28 @@ DISABLE_INDEX = NO # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. +# further fine tune the look of the index (see "Fine-tuning the output"). As an +# example, the default style sheet generated by doxygen has an example that +# shows how to put an image at the root of the tree instead of the PROJECT_NAME. +# Since the tree basically has the same information as the tab index, you could +# consider setting DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = YES +# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the +# FULL_SIDEBAR option determines if the side bar is limited to only the treeview +# area (value NO) or if it should extend to the full height of the window (value +# YES). Setting this to YES gives a layout similar to +# https://docs.readthedocs.io with more room for contents, but less room for the +# project logo, title, and description. If either GENERATOR_TREEVIEW or +# DISABLE_INDEX is set to NO, this option has no effect. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FULL_SIDEBAR = NO + # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # @@ -1573,11 +1596,29 @@ FORMULA_MACROFILE = USE_MATHJAX = NO +# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. +# Note that the different versions of MathJax have different requirements with +# regards to the different settings, so it is possible that also other MathJax +# settings have to be changed when switching between the different MathJax +# versions. +# Possible values are: MathJax_2 and MathJax_3. +# The default value is: MathJax_2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_VERSION = MathJax_2 + # When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. +# the MathJax output. For more details about the output format see MathJax +# version 2 (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 +# (see: +# http://docs.mathjax.org/en/latest/web/components/output.html). # Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. +# compatibility. This is the name for Mathjax version 2, for MathJax version 3 +# this will be translated into chtml), NativeMML (i.e. MathML. Only supported +# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This +# is the name for Mathjax version 3, for MathJax version 2 this will be +# translated into HTML-CSS) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1590,15 +1631,21 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. -# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# MathJax from https://www.mathjax.org before deployment. The default value is: +# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 +# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = http://www.mathjax.org/mathjax # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example +# for MathJax version 2 (see https://docs.mathjax.org/en/v2.7-latest/tex.html +# #tex-and-latex-extensions): # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# For example for MathJax version 3 (see +# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): +# MATHJAX_EXTENSIONS = ams # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = @@ -1778,29 +1825,31 @@ PAPER_TYPE = letter EXTRA_PACKAGES = -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the -# generated LaTeX document. The header should contain everything until the first -# chapter. If it is left blank doxygen will generate a standard header. See -# section "Doxygen usage" for information on how to let doxygen write the -# default header to a separate file. +# The LATEX_HEADER tag can be used to specify a user-defined LaTeX header for +# the generated LaTeX document. The header should contain everything until the +# first chapter. If it is left blank doxygen will generate a standard header. It +# is highly recommended to start with a default header using +# doxygen -w latex new_header.tex new_footer.tex new_stylesheet.sty +# and then modify the file new_header.tex. See also section "Doxygen usage" for +# information on how to generate the default header that doxygen normally uses. # -# Note: Only use a user-defined header if you know what you are doing! The -# following commands have a special meaning inside the header: $title, -# $datetime, $date, $doxygenversion, $projectname, $projectnumber, -# $projectbrief, $projectlogo. Doxygen will replace $title with the empty -# string, for the replacement values of the other commands the user is referred -# to HTML_HEADER. +# Note: Only use a user-defined header if you know what you are doing! +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. The following +# commands have a special meaning inside the header (and footer): For a +# description of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HEADER = -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the -# generated LaTeX document. The footer should contain everything after the last -# chapter. If it is left blank doxygen will generate a standard footer. See +# The LATEX_FOOTER tag can be used to specify a user-defined LaTeX footer for +# the generated LaTeX document. The footer should contain everything after the +# last chapter. If it is left blank doxygen will generate a standard footer. See # LATEX_HEADER for more information on how to generate a default footer and what -# special commands can be used inside the footer. -# -# Note: Only use a user-defined footer if you know what you are doing! +# special commands can be used inside the footer. See also section "Doxygen +# usage" for information on how to generate the default footer that doxygen +# normally uses. Note: Only use a user-defined footer if you know what you are +# doing! # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_FOOTER = @@ -1845,8 +1894,7 @@ USE_PDFLATEX = YES # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode # command to the generated LaTeX files. This will instruct LaTeX to keep running -# if errors occur, instead of asking the user for help. This option is also used -# when generating formulas in HTML. +# if errors occur, instead of asking the user for help. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1859,16 +1907,6 @@ LATEX_BATCHMODE = NO LATEX_HIDE_INDICES = NO -# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source -# code with syntax highlighting in the LaTeX output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_SOURCE_CODE = NO - # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See # https://en.wikipedia.org/wiki/BibTeX and \cite for more info. @@ -1949,16 +1987,6 @@ RTF_STYLESHEET_FILE = RTF_EXTENSIONS_FILE = -# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code -# with syntax highlighting in the RTF output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_RTF is set to YES. - -RTF_SOURCE_CODE = NO - #--------------------------------------------------------------------------- # Configuration options related to the man page output #--------------------------------------------------------------------------- @@ -2055,15 +2083,6 @@ GENERATE_DOCBOOK = NO DOCBOOK_OUTPUT = docbook -# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the -# program listings (including syntax highlighting and cross-referencing -# information) to the DOCBOOK output. Note that enabling this will significantly -# increase the size of the DOCBOOK output. -# The default value is: NO. -# This tag requires that the tag GENERATE_DOCBOOK is set to YES. - -DOCBOOK_PROGRAMLISTING = NO - #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- @@ -2568,8 +2587,8 @@ GENERATE_LEGEND = YES # If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate # files that are used to generate the various graphs. # -# Note: This setting is not only used for dot files but also for msc and -# plantuml temporary files. +# Note: This setting is not only used for dot files but also for msc temporary +# files. # The default value is: YES. DOT_CLEANUP = YES diff --git a/packages/seacas/libraries/ioss/cmake/Dependencies.cmake b/packages/seacas/libraries/ioss/cmake/Dependencies.cmake index 567fc6a99e48..7f6fb0face31 100644 --- a/packages/seacas/libraries/ioss/cmake/Dependencies.cmake +++ b/packages/seacas/libraries/ioss/cmake/Dependencies.cmake @@ -1,12 +1,12 @@ if(CMAKE_PROJECT_NAME STREQUAL "SEACASProj") TRIBITS_PACKAGE_DEFINE_DEPENDENCIES( LIB_OPTIONAL_PACKAGES SEACASExodus Zoltan - LIB_OPTIONAL_TPLS HDF5 Pamgen CGNS ParMETIS Faodel Cereal DLlib Pthread ADIOS2 gtest Kokkos DataWarp fmt + LIB_OPTIONAL_TPLS HDF5 Pamgen CGNS ParMETIS Faodel Cereal DLlib Pthread ADIOS2 Catalyst2 GTest Kokkos DataWarp fmt ) else() TRIBITS_PACKAGE_DEFINE_DEPENDENCIES( LIB_OPTIONAL_PACKAGES SEACASExodus Pamgen Zoltan Kokkos - LIB_OPTIONAL_TPLS HDF5 CGNS ParMETIS Faodel Cereal DLlib Pthread DataWarp ADIOS2 gtest + LIB_OPTIONAL_TPLS HDF5 CGNS ParMETIS Faodel Cereal DLlib Pthread DataWarp ADIOS2 Catalyst2 GTest ) endif() diff --git a/packages/seacas/libraries/ioss/cmake/SEACASIoss_config.h.in b/packages/seacas/libraries/ioss/cmake/SEACASIoss_config.h.in index 7cf96c30f451..15b62c0beaa2 100644 --- a/packages/seacas/libraries/ioss/cmake/SEACASIoss_config.h.in +++ b/packages/seacas/libraries/ioss/cmake/SEACASIoss_config.h.in @@ -16,3 +16,5 @@ #cmakedefine SEACAS_HAVE_PAMGEN #cmakedefine HAVE_SEACASIOSS_ADIOS2 + +#cmakedefine SEACAS_HAVE_CATALYST2 diff --git a/packages/seacas/libraries/ioss/src/CMakeLists.txt b/packages/seacas/libraries/ioss/src/CMakeLists.txt index dbef2ebb4d71..b47b484470ec 100644 --- a/packages/seacas/libraries/ioss/src/CMakeLists.txt +++ b/packages/seacas/libraries/ioss/src/CMakeLists.txt @@ -22,6 +22,10 @@ IF (${TPL_ENABLE_Faodel}) SET(SEACAS_HAVE_FAODEL ON) ENDIF() +IF (TPL_ENABLE_Catalyst2) + SET(SEACAS_HAVE_CATALYST2 ON) +ENDIF() + IF (${PACKAGE_NAME}_ENABLE_SEACASExodus) SET(SEACAS_HAVE_EXODUS ON) ENDIF() @@ -109,9 +113,11 @@ IF (TPL_ENABLE_ADIOS2) ADD_SUBDIRECTORY(adios) ENDIF() -IF (NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows") -ADD_SUBDIRECTORY(visualization) +IF (TPL_ENABLE_Catalyst2) + ADD_SUBDIRECTORY(catalyst) ENDIF() + +ADD_SUBDIRECTORY(visualization) ADD_SUBDIRECTORY(generated) ADD_SUBDIRECTORY(gen_struc) ADD_SUBDIRECTORY(heartbeat) diff --git a/packages/seacas/libraries/ioss/src/Ioss_Assembly.C b/packages/seacas/libraries/ioss/src/Ioss_Assembly.C index dacafc1d310d..0d62ff340db4 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_Assembly.C +++ b/packages/seacas/libraries/ioss/src/Ioss_Assembly.C @@ -1,4 +1,4 @@ -// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// Copyright(C) 1999-2021 National Technology & Engineering Solutions // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with // NTESS, the U.S. Government retains certain rights in this software. // @@ -81,10 +81,9 @@ Ioss::Assembly::Assembly(Ioss::DatabaseIO *io_database, const std::string &my_na properties.add(Ioss::Property(this, "member_type", Ioss::Property::INTEGER)); } -Ioss::Assembly::Assembly(const Ioss::Assembly &other) : GroupingEntity(other) +Ioss::Assembly::Assembly(const Ioss::Assembly &other) + : GroupingEntity(other), m_members(other.m_members), m_type(other.m_type) { - m_members = other.m_members; - m_type = other.m_type; } const Ioss::EntityContainer &Ioss::Assembly::get_members() const { return m_members; } diff --git a/packages/seacas/libraries/ioss/src/Ioss_Compare.C b/packages/seacas/libraries/ioss/src/Ioss_Compare.C index eab01284ba5c..a520df4a8be1 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_Compare.C +++ b/packages/seacas/libraries/ioss/src/Ioss_Compare.C @@ -27,60 +27,63 @@ /* These messages indicate a value difference between the files * being compared. Use Ioss::OUTPUT(). */ -#define ATTRIBUTE_FIELD_VALUE_MISMATCH "FIELD data ({}): ATTRIBUTE FIELD data mismatch" -#define COMMUNICATION_FIELD_VALUE_MISMATCH "FIELD data ({}): COMMUNICATION FIELD data mismatch" -#define MESH_FIELD_VALUE_MISMATCH "FIELD data ({}): MESH FIELD data mismatch" -#define TRANSIENT_FIELD_VALUE_MISMATCH "FIELD data ({}): TRANSIENT FIELD data mismatch" -#define TRANSIENT_FIELD_STEP_VALUE_MISMATCH \ - "FIELD data ({} / step {}): TRANSIENT FIELD data mismatch" +#define ATTRIBUTE_FIELD_VALUE_MISMATCH "\n\nATTRIBUTE FIELD data mismatch ({})" +#define COMMUNICATION_FIELD_VALUE_MISMATCH "\n\nCOMMUNICATION FIELD data mismatch ({})" +#define MESH_FIELD_VALUE_MISMATCH "\n\nMESH FIELD data mismatch ({})" +#define TRANSIENT_FIELD_VALUE_MISMATCH "\n\nTRANSIENT FIELD data mismatch ({})" +#define TRANSIENT_FIELD_STEP_VALUE_MISMATCH "\n\nTRANSIENT FIELD data mismatch ({} at step {})" // For compare_database... namespace { - bool compare_properties(const Ioss::GroupingEntity *ige_1, const Ioss::GroupingEntity *ige_2); - bool compare_qa_info(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2); + bool compare_properties(const Ioss::GroupingEntity *ige_1, const Ioss::GroupingEntity *ige_2, + std::ostringstream &buf); + bool compare_qa_info(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, + std::ostringstream &buf); bool compare_nodeblock(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf); bool compare_elementblocks(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf); bool compare_edgeblocks(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf); bool compare_faceblocks(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf); bool compare_structuredblocks(const Ioss::Region & input_region_1, const Ioss::Region & input_region_2, - const Ioss::MeshCopyOptions &options); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf); bool compare_nodesets(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf); bool compare_edgesets(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf); bool compare_facesets(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf); bool compare_elemsets(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf); bool compare_sidesets(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf); bool compare_commsets(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf); bool compare_coordinate_frames(const Ioss::Region & input_region_1, const Ioss::Region & input_region_2, - const Ioss::MeshCopyOptions &options); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf); template bool compare_fields(const std::vector &in_entities_1, const std::vector &in_entities_2, - const Ioss::Field::RoleType role); + const Ioss::Field::RoleType role, std::ostringstream &buf); bool compare_fields(const Ioss::GroupingEntity *ige_1, const Ioss::GroupingEntity *ige_2, - const Ioss::Field::RoleType role); + const Ioss::Field::RoleType role, std::ostringstream &buf); template bool compare_field_data(const std::vector &in_entities_1, const std::vector &in_entities_2, DataPool &pool, - const Ioss::Field::RoleType role, const Ioss::MeshCopyOptions &options); + const Ioss::Field::RoleType role, const Ioss::MeshCopyOptions &options, + std::ostringstream &buf); bool compare_field_data(const Ioss::GroupingEntity *ige_1, const Ioss::GroupingEntity *ige_2, DataPool &pool, const Ioss::Field::RoleType role, - const Ioss::MeshCopyOptions &options, const std::string &prefix = ""); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf, + const std::string &prefix = ""); bool compare_field_data_internal(const Ioss::GroupingEntity *ige_1, const Ioss::GroupingEntity *ige_2, DataPool &in_pool, const std::string & field_name, - const Ioss::MeshCopyOptions &options); + const Ioss::MeshCopyOptions &options, std::ostringstream &buf); } // namespace bool Ioss::Compare::compare_database(Ioss::Region &input_region_1, Ioss::Region &input_region_2, @@ -91,220 +94,336 @@ bool Ioss::Compare::compare_database(Ioss::Region &input_region_1, Ioss::Region DataPool data_pool; // COMPARE all properties of input database... - if (compare_properties(&input_region_1, &input_region_2) == false) { - fmt::print(Ioss::OUTPUT(), "PROPERTIES mismatch ({})\n", input_region_1.name()); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, "PROPERTIES mismatch ({})\n", input_region_1.name()); + if (compare_properties(&input_region_1, &input_region_2, buf) == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } if (!options.ignore_qa_info) { - if (compare_qa_info(input_region_1, input_region_2) == false) { - fmt::print(Ioss::OUTPUT(), "\nQA INFO mismatch\n"); + std::ostringstream buf; + fmt::print(buf, "\nQA INFO mismatch\n"); + if (compare_qa_info(input_region_1, input_region_2, buf) == false) { overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); } } - if (compare_nodeblock(input_region_1, input_region_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nNODEBLOCK mismatch\n"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, "\nNODEBLOCK mismatch\n"); + if (compare_nodeblock(input_region_1, input_region_2, options, buf) == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - if (compare_edgeblocks(input_region_1, input_region_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nEDGEBLOCK mismatch\n"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, "\nEDGEBLOCK mismatch\n"); + if (compare_edgeblocks(input_region_1, input_region_2, options, buf) == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - - if (compare_faceblocks(input_region_1, input_region_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nFACEBLOCK mismatch\n"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, "\nFACEBLOCK mismatch\n"); + if (compare_faceblocks(input_region_1, input_region_2, options, buf) == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - - if (compare_elementblocks(input_region_1, input_region_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nELEMENTBLOCK mismatch\n"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, "\nELEMENTBLOCK mismatch\n"); + if (compare_elementblocks(input_region_1, input_region_2, options, buf) == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - if (compare_structuredblocks(input_region_1, input_region_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nSTRUCTUREDBLOCK mismatch\n"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, "\nSTRUCTUREDBLOCK mismatch\n"); + if (compare_structuredblocks(input_region_1, input_region_2, options, buf) == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - if (compare_nodesets(input_region_1, input_region_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nNODESET mismatch\n"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, "\nNODESET mismatch\n"); + if (compare_nodesets(input_region_1, input_region_2, options, buf) == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - if (compare_edgesets(input_region_1, input_region_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nEDGESET mismatch\n"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, "\nEDGESET mismatch\n"); + if (compare_edgesets(input_region_1, input_region_2, options, buf) == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - if (compare_facesets(input_region_1, input_region_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nFACESET mismatch\n"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, "\nFACESET mismatch\n"); + if (compare_facesets(input_region_1, input_region_2, options, buf) == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - if (compare_elemsets(input_region_1, input_region_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nELEMSET mismatch\n"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, "\nELEMSET mismatch\n"); + if (compare_elemsets(input_region_1, input_region_2, options, buf) == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - if (compare_sidesets(input_region_1, input_region_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nSIDESET mismatch\n"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, "\nSIDESET mismatch\n"); + if (compare_sidesets(input_region_1, input_region_2, options, buf) == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - if (compare_commsets(input_region_1, input_region_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nCOMMSET mismatch\n"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, "\nCOMMSET mismatch\n"); + if (compare_commsets(input_region_1, input_region_2, options, buf) == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - if (compare_coordinate_frames(input_region_1, input_region_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nCOORDINATE FRAME mismatch\n"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, "\nCOORDINATE FRAME mismatch\n"); + if (compare_coordinate_frames(input_region_1, input_region_2, options, buf) == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } bool node_major = input_region_2.node_major(); if (!node_major) { - rc = - compare_field_data(input_region_1.get_element_blocks(), input_region_2.get_element_blocks(), - data_pool, Ioss::Field::MESH, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), MESH_FIELD_VALUE_MISMATCH " (node_major = {})", "element blocks", - node_major); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, MESH_FIELD_VALUE_MISMATCH " (node_major = {})", "element blocks", node_major); + rc = compare_field_data(input_region_1.get_element_blocks(), + input_region_2.get_element_blocks(), data_pool, Ioss::Field::MESH, + options, buf); + if (rc == false) { + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + overall_result = false; + } } - rc = - compare_field_data(input_region_1.get_element_blocks(), input_region_2.get_element_blocks(), - data_pool, Ioss::Field::ATTRIBUTE, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), ATTRIBUTE_FIELD_VALUE_MISMATCH " (node_major = {})", - "element blocks", node_major); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, ATTRIBUTE_FIELD_VALUE_MISMATCH " (node_major = {})", "element blocks", + node_major); + + rc = compare_field_data(input_region_1.get_element_blocks(), + input_region_2.get_element_blocks(), data_pool, + Ioss::Field::ATTRIBUTE, options, buf); + if (rc == false) { + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + overall_result = false; + } } } if (input_region_1.mesh_type() != Ioss::MeshType::STRUCTURED) { assert(input_region_2.mesh_type() != Ioss::MeshType::STRUCTURED); + { + std::ostringstream buf; + fmt::print(buf, MESH_FIELD_VALUE_MISMATCH, "node blocks"); - rc = compare_field_data(input_region_1.get_node_blocks(), input_region_2.get_node_blocks(), - data_pool, Ioss::Field::MESH, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), MESH_FIELD_VALUE_MISMATCH, "node blocks"); - overall_result = false; + rc = compare_field_data(input_region_1.get_node_blocks(), input_region_2.get_node_blocks(), + data_pool, Ioss::Field::MESH, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - - rc = compare_field_data(input_region_1.get_node_blocks(), input_region_2.get_node_blocks(), - data_pool, Ioss::Field::ATTRIBUTE, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), ATTRIBUTE_FIELD_VALUE_MISMATCH, "node blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, ATTRIBUTE_FIELD_VALUE_MISMATCH, "node blocks"); + rc = compare_field_data(input_region_1.get_node_blocks(), input_region_2.get_node_blocks(), + data_pool, Ioss::Field::ATTRIBUTE, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } } if (node_major) { - rc = - compare_field_data(input_region_1.get_element_blocks(), input_region_2.get_element_blocks(), - data_pool, Ioss::Field::MESH, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), MESH_FIELD_VALUE_MISMATCH " (node_major = {})", "element blocks", + { + std::ostringstream buf; + fmt::print(buf, MESH_FIELD_VALUE_MISMATCH " (node_major = {})", "element blocks", node_major); + rc = compare_field_data(input_region_1.get_element_blocks(), + input_region_2.get_element_blocks(), data_pool, Ioss::Field::MESH, + options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } + } + { + std::ostringstream buf; + fmt::print(buf, ATTRIBUTE_FIELD_VALUE_MISMATCH " (node_major = {})", "element blocks", node_major); - overall_result = false; + rc = compare_field_data(input_region_1.get_element_blocks(), + input_region_2.get_element_blocks(), data_pool, + Ioss::Field::ATTRIBUTE, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } + } - rc = - compare_field_data(input_region_1.get_element_blocks(), input_region_2.get_element_blocks(), - data_pool, Ioss::Field::ATTRIBUTE, options); + { + std::ostringstream buf; + fmt::print(buf, MESH_FIELD_VALUE_MISMATCH, "structured blocks"); + rc = compare_field_data(input_region_1.get_structured_blocks(), + input_region_2.get_structured_blocks(), data_pool, Ioss::Field::MESH, + options, buf); if (rc == false) { - fmt::print(Ioss::OUTPUT(), ATTRIBUTE_FIELD_VALUE_MISMATCH " (node_major = {})", - "element blocks", node_major); overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); } } - rc = compare_field_data(input_region_1.get_structured_blocks(), - input_region_2.get_structured_blocks(), data_pool, Ioss::Field::MESH, - options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), MESH_FIELD_VALUE_MISMATCH, "structured blocks"); - overall_result = false; - } - - rc = compare_field_data(input_region_1.get_structured_blocks(), - input_region_2.get_structured_blocks(), data_pool, Ioss::Field::ATTRIBUTE, - options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), ATTRIBUTE_FIELD_VALUE_MISMATCH, "structured blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, ATTRIBUTE_FIELD_VALUE_MISMATCH, "structured blocks"); + rc = compare_field_data(input_region_1.get_structured_blocks(), + input_region_2.get_structured_blocks(), data_pool, + Ioss::Field::ATTRIBUTE, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_field_data(input_region_1.get_edge_blocks(), input_region_2.get_edge_blocks(), - data_pool, Ioss::Field::MESH, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), MESH_FIELD_VALUE_MISMATCH, "edge blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, MESH_FIELD_VALUE_MISMATCH, "edge blocks"); + rc = compare_field_data(input_region_1.get_edge_blocks(), input_region_2.get_edge_blocks(), + data_pool, Ioss::Field::MESH, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_field_data(input_region_1.get_edge_blocks(), input_region_2.get_edge_blocks(), - data_pool, Ioss::Field::ATTRIBUTE, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), ATTRIBUTE_FIELD_VALUE_MISMATCH, "edge blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, ATTRIBUTE_FIELD_VALUE_MISMATCH, "edge blocks"); + rc = compare_field_data(input_region_1.get_edge_blocks(), input_region_2.get_edge_blocks(), + data_pool, Ioss::Field::ATTRIBUTE, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_field_data(input_region_1.get_face_blocks(), input_region_2.get_face_blocks(), - data_pool, Ioss::Field::MESH, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), MESH_FIELD_VALUE_MISMATCH, "face blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, MESH_FIELD_VALUE_MISMATCH, "face blocks"); + rc = compare_field_data(input_region_1.get_face_blocks(), input_region_2.get_face_blocks(), + data_pool, Ioss::Field::MESH, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_field_data(input_region_1.get_face_blocks(), input_region_2.get_face_blocks(), - data_pool, Ioss::Field::ATTRIBUTE, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), ATTRIBUTE_FIELD_VALUE_MISMATCH, "face blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, ATTRIBUTE_FIELD_VALUE_MISMATCH, "face blocks"); + rc = compare_field_data(input_region_1.get_face_blocks(), input_region_2.get_face_blocks(), + data_pool, Ioss::Field::ATTRIBUTE, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_field_data(input_region_1.get_elementsets(), input_region_2.get_elementsets(), - data_pool, Ioss::Field::MESH, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), MESH_FIELD_VALUE_MISMATCH, "element sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, MESH_FIELD_VALUE_MISMATCH, "element sets"); + rc = compare_field_data(input_region_1.get_elementsets(), input_region_2.get_elementsets(), + data_pool, Ioss::Field::MESH, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_field_data(input_region_1.get_elementsets(), input_region_2.get_elementsets(), - data_pool, Ioss::Field::ATTRIBUTE, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), ATTRIBUTE_FIELD_VALUE_MISMATCH, "element sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, ATTRIBUTE_FIELD_VALUE_MISMATCH, "element sets"); + rc = compare_field_data(input_region_1.get_elementsets(), input_region_2.get_elementsets(), + data_pool, Ioss::Field::ATTRIBUTE, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_field_data(input_region_1.get_commsets(), input_region_2.get_commsets(), data_pool, - Ioss::Field::MESH, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), MESH_FIELD_VALUE_MISMATCH, "comm sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, MESH_FIELD_VALUE_MISMATCH, "comm sets"); + rc = compare_field_data(input_region_1.get_commsets(), input_region_2.get_commsets(), data_pool, + Ioss::Field::MESH, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_field_data(input_region_1.get_commsets(), input_region_2.get_commsets(), data_pool, - Ioss::Field::ATTRIBUTE, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), ATTRIBUTE_FIELD_VALUE_MISMATCH, "comm sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, ATTRIBUTE_FIELD_VALUE_MISMATCH, "comm sets"); + rc = compare_field_data(input_region_1.get_commsets(), input_region_2.get_commsets(), data_pool, + Ioss::Field::ATTRIBUTE, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_field_data(input_region_1.get_commsets(), input_region_2.get_commsets(), data_pool, - Ioss::Field::COMMUNICATION, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), COMMUNICATION_FIELD_VALUE_MISMATCH, "comm sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, COMMUNICATION_FIELD_VALUE_MISMATCH, "comm sets"); + rc = compare_field_data(input_region_1.get_commsets(), input_region_2.get_commsets(), data_pool, + Ioss::Field::COMMUNICATION, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - // Side Sets if (input_region_1.mesh_type() == Ioss::MeshType::UNSTRUCTURED) { // This should have already been checked. - assert(input_region_1.mesh_type() == Ioss::MeshType::UNSTRUCTURED); + assert(input_region_2.mesh_type() == Ioss::MeshType::UNSTRUCTURED); const auto &in_fss_1 = input_region_1.get_sidesets(); const auto &in_fss_2 = input_region_2.get_sidesets(); @@ -328,18 +447,25 @@ bool Ioss::Compare::compare_database(Ioss::Region &input_region_1, Ioss::Region continue; } - rc = compare_field_data(ifs, (*it), data_pool, Ioss::Field::MESH, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), MESH_FIELD_VALUE_MISMATCH, "side sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, MESH_FIELD_VALUE_MISMATCH, "side sets"); + rc = compare_field_data(ifs, (*it), data_pool, Ioss::Field::MESH, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_field_data(ifs, (*it), data_pool, Ioss::Field::ATTRIBUTE, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), ATTRIBUTE_FIELD_VALUE_MISMATCH, "side sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, ATTRIBUTE_FIELD_VALUE_MISMATCH, "side sets"); + rc = compare_field_data(ifs, (*it), data_pool, Ioss::Field::ATTRIBUTE, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - const auto &in_sbs_1 = ifs->get_side_blocks(); const auto &in_sbs_2 = (*it)->get_side_blocks(); @@ -361,16 +487,24 @@ bool Ioss::Compare::compare_database(Ioss::Region &input_region_1, Ioss::Region continue; } - rc = compare_field_data(isb, (*iter), data_pool, Ioss::Field::MESH, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), MESH_FIELD_VALUE_MISMATCH, "side blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, MESH_FIELD_VALUE_MISMATCH, "side blocks"); + rc = compare_field_data(isb, (*iter), data_pool, Ioss::Field::MESH, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_field_data(isb, (*iter), data_pool, Ioss::Field::ATTRIBUTE, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), ATTRIBUTE_FIELD_VALUE_MISMATCH, "side blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, ATTRIBUTE_FIELD_VALUE_MISMATCH, "side blocks"); + rc = compare_field_data(isb, (*iter), data_pool, Ioss::Field::ATTRIBUTE, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } } } @@ -389,75 +523,112 @@ bool Ioss::Compare::compare_database(Ioss::Region &input_region_1, Ioss::Region // For each 'TRANSIENT' field in the node blocks and element // blocks, transfer to the output node and element blocks. - rc = compare_fields(&input_region_1, &input_region_2, Ioss::Field::TRANSIENT); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "region"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "region"); + rc = compare_fields(&input_region_1, &input_region_2, Ioss::Field::TRANSIENT, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - - rc = compare_fields(input_region_1.get_node_blocks(), input_region_2.get_node_blocks(), - Ioss::Field::TRANSIENT); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "node blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "node blocks"); + rc = compare_fields(input_region_1.get_node_blocks(), input_region_2.get_node_blocks(), + Ioss::Field::TRANSIENT, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - - rc = compare_fields(input_region_1.get_edge_blocks(), input_region_2.get_edge_blocks(), - Ioss::Field::TRANSIENT); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "edge blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "edge blocks"); + rc = compare_fields(input_region_1.get_edge_blocks(), input_region_2.get_edge_blocks(), + Ioss::Field::TRANSIENT, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_fields(input_region_1.get_face_blocks(), input_region_2.get_face_blocks(), - Ioss::Field::TRANSIENT); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "face blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "face blocks"); + rc = compare_fields(input_region_1.get_face_blocks(), input_region_2.get_face_blocks(), + Ioss::Field::TRANSIENT, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_fields(input_region_1.get_element_blocks(), input_region_2.get_element_blocks(), - Ioss::Field::TRANSIENT); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "element blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "element blocks"); + rc = compare_fields(input_region_1.get_element_blocks(), input_region_2.get_element_blocks(), + Ioss::Field::TRANSIENT, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_fields(input_region_1.get_structured_blocks(), - input_region_2.get_structured_blocks(), Ioss::Field::TRANSIENT); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "structured blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "structured blocks"); + rc = compare_fields(input_region_1.get_structured_blocks(), + input_region_2.get_structured_blocks(), Ioss::Field::TRANSIENT, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_fields(input_region_1.get_nodesets(), input_region_2.get_nodesets(), - Ioss::Field::TRANSIENT); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "node sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "node sets"); + rc = compare_fields(input_region_1.get_nodesets(), input_region_2.get_nodesets(), + Ioss::Field::TRANSIENT, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_fields(input_region_1.get_edgesets(), input_region_2.get_edgesets(), - Ioss::Field::TRANSIENT); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "edge sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "edge sets"); + rc = compare_fields(input_region_1.get_edgesets(), input_region_2.get_edgesets(), + Ioss::Field::TRANSIENT, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_fields(input_region_1.get_facesets(), input_region_2.get_facesets(), - Ioss::Field::TRANSIENT); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "face sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "face sets"); + rc = compare_fields(input_region_1.get_facesets(), input_region_2.get_facesets(), + Ioss::Field::TRANSIENT, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - rc = compare_fields(input_region_1.get_elementsets(), input_region_2.get_elementsets(), - Ioss::Field::TRANSIENT); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "element sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "element sets"); + rc = compare_fields(input_region_1.get_elementsets(), input_region_2.get_elementsets(), + Ioss::Field::TRANSIENT, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - // Side Sets { const auto &in_sss_1 = input_region_1.get_sidesets(); @@ -478,12 +649,15 @@ bool Ioss::Compare::compare_database(Ioss::Region &input_region_1, Ioss::Region } { - rc = compare_fields(iss, (*it), Ioss::Field::TRANSIENT); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "side sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "side sets"); + rc = compare_fields(iss, (*it), Ioss::Field::TRANSIENT, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - const auto &in_sbs_1 = iss->get_side_blocks(); const auto &in_sbs_2 = (*it)->get_side_blocks(); if (in_sbs_1.size() != in_sbs_2.size()) { @@ -507,10 +681,14 @@ bool Ioss::Compare::compare_database(Ioss::Region &input_region_1, Ioss::Region continue; } - rc = compare_fields(isb, (*iter), Ioss::Field::TRANSIENT); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "side blocks"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "side blocks"); + rc = compare_fields(isb, (*iter), Ioss::Field::TRANSIENT, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } } } @@ -538,83 +716,114 @@ bool Ioss::Compare::compare_database(Ioss::Region &input_region_1, Ioss::Region input_region_1.begin_state(istep); input_region_2.begin_state(istep); - rc = compare_field_data(&input_region_1, &input_region_2, data_pool, Ioss::Field::TRANSIENT, - options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "region", istep); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "region", istep); + rc = compare_field_data(&input_region_1, &input_region_2, data_pool, Ioss::Field::TRANSIENT, + options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - // This should have already been checked assert(input_region_1.mesh_type() == input_region_2.mesh_type()); if (input_region_1.mesh_type() != Ioss::MeshType::STRUCTURED) { - rc = compare_field_data(input_region_1.get_node_blocks(), input_region_2.get_node_blocks(), - data_pool, Ioss::Field::TRANSIENT, options); + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "node blocks", istep); + rc = + compare_field_data(input_region_1.get_node_blocks(), input_region_2.get_node_blocks(), + data_pool, Ioss::Field::TRANSIENT, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } + } + } + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "edge blocks", istep); + rc = compare_field_data(input_region_1.get_edge_blocks(), input_region_2.get_edge_blocks(), + data_pool, Ioss::Field::TRANSIENT, options, buf); if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "node blocks", istep); overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); } } - - rc = compare_field_data(input_region_1.get_edge_blocks(), input_region_2.get_edge_blocks(), - data_pool, Ioss::Field::TRANSIENT, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "edge blocks", istep); - overall_result = false; - } - - rc = compare_field_data(input_region_1.get_face_blocks(), input_region_2.get_face_blocks(), - data_pool, Ioss::Field::TRANSIENT, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "face blocks", istep); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "face blocks", istep); + rc = compare_field_data(input_region_1.get_face_blocks(), input_region_2.get_face_blocks(), + data_pool, Ioss::Field::TRANSIENT, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - - rc = compare_field_data(input_region_1.get_element_blocks(), - input_region_2.get_element_blocks(), data_pool, - Ioss::Field::TRANSIENT, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "element blocks", istep); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "element blocks", istep); + rc = compare_field_data(input_region_1.get_element_blocks(), + input_region_2.get_element_blocks(), data_pool, + Ioss::Field::TRANSIENT, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - - rc = compare_field_data(input_region_1.get_structured_blocks(), - input_region_2.get_structured_blocks(), data_pool, - Ioss::Field::TRANSIENT, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "structured blocks", istep); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "structured blocks", istep); + rc = compare_field_data(input_region_1.get_structured_blocks(), + input_region_2.get_structured_blocks(), data_pool, + Ioss::Field::TRANSIENT, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - - rc = compare_field_data(input_region_1.get_nodesets(), input_region_2.get_nodesets(), - data_pool, Ioss::Field::TRANSIENT, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "node sets", istep); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "node sets", istep); + rc = compare_field_data(input_region_1.get_nodesets(), input_region_2.get_nodesets(), + data_pool, Ioss::Field::TRANSIENT, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - - rc = compare_field_data(input_region_1.get_edgesets(), input_region_2.get_edgesets(), - data_pool, Ioss::Field::TRANSIENT, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "edge sets", istep); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "edge sets", istep); + rc = compare_field_data(input_region_1.get_edgesets(), input_region_2.get_edgesets(), + data_pool, Ioss::Field::TRANSIENT, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - - rc = compare_field_data(input_region_1.get_facesets(), input_region_2.get_facesets(), - data_pool, Ioss::Field::TRANSIENT, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "face sets", istep); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "face sets", istep); + rc = compare_field_data(input_region_1.get_facesets(), input_region_2.get_facesets(), + data_pool, Ioss::Field::TRANSIENT, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - - rc = compare_field_data(input_region_1.get_elementsets(), input_region_2.get_elementsets(), - data_pool, Ioss::Field::TRANSIENT, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "element sets", istep); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_STEP_VALUE_MISMATCH, "element sets", istep); + rc = compare_field_data(input_region_1.get_elementsets(), input_region_2.get_elementsets(), + data_pool, Ioss::Field::TRANSIENT, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - // Side Sets const auto &in_sss_1 = input_region_1.get_sidesets(); const auto &in_sss_2 = input_region_2.get_sidesets(); @@ -639,12 +848,15 @@ bool Ioss::Compare::compare_database(Ioss::Region &input_region_1, Ioss::Region } { - rc = compare_field_data(iss, (*it), data_pool, Ioss::Field::TRANSIENT, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "side sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "side sets"); + rc = compare_field_data(iss, (*it), data_pool, Ioss::Field::TRANSIENT, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } - const auto &in_sbs_1 = iss->get_side_blocks(); const auto &in_sbs_2 = (*it)->get_side_blocks(); if (in_sbs_1.size() != in_sbs_2.size()) { @@ -668,10 +880,15 @@ bool Ioss::Compare::compare_database(Ioss::Region &input_region_1, Ioss::Region continue; } - rc = compare_field_data(isb, (*iter), data_pool, Ioss::Field::TRANSIENT, options); - if (rc == false) { - fmt::print(Ioss::OUTPUT(), TRANSIENT_FIELD_VALUE_MISMATCH, "side sets"); - overall_result = false; + { + std::ostringstream buf; + fmt::print(buf, TRANSIENT_FIELD_VALUE_MISMATCH, "side sets"); + rc = + compare_field_data(isb, (*iter), data_pool, Ioss::Field::TRANSIENT, options, buf); + if (rc == false) { + overall_result = false; + fmt::print(Ioss::OUTPUT(), "{}", buf.str()); + } } } } @@ -685,7 +902,8 @@ bool Ioss::Compare::compare_database(Ioss::Region &input_region_1, Ioss::Region } namespace { - bool compare_properties(const Ioss::GroupingEntity *ige_1, const Ioss::GroupingEntity *ige_2) + bool compare_properties(const Ioss::GroupingEntity *ige_1, const Ioss::GroupingEntity *ige_2, + std::ostringstream &buf) { bool overall_result = true; @@ -715,19 +933,19 @@ namespace { continue; } - Ioss::Property ige_property_1 = ige_1->get_property(property); - Ioss::Property ige_property_2 = ige_2->get_property(property); + const Ioss::Property &ige_property_1 = ige_1->get_property(property); + const Ioss::Property &ige_property_2 = ige_2->get_property(property); if (ige_property_1 != ige_property_2) { if (ige_property_1.get_type() == Ioss::Property::STRING) { - fmt::print(Ioss::OUTPUT(), "PROPERTY value mismatch ({}): ({} vs {})\n", property, + fmt::print(buf, "\tPROPERTY value mismatch ({}): ({} vs {})\n", property, ige_property_1.get_string(), ige_property_2.get_string()); } else if (ige_property_1.get_type() == Ioss::Property::INTEGER) { - fmt::print(Ioss::OUTPUT(), "PROPERTY value mismatch ({}): ({} vs {})\n", property, + fmt::print(buf, "\tPROPERTY value mismatch ({}): ({} vs {})\n", property, ige_property_1.get_int(), ige_property_2.get_int()); } else { - fmt::print(Ioss::OUTPUT(), "PROPERTY value mismatch ({}): unsupported type\n", property); + fmt::print(buf, "PROPERTY value mismatch ({}): unsupported type\n", property); } overall_result = false; @@ -737,7 +955,8 @@ namespace { return overall_result; } - bool compare_qa_info(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2) + bool compare_qa_info(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, + std::ostringstream &buf) { bool overall_result = true; @@ -786,7 +1005,7 @@ namespace { } if (in_qa_record_1.compare(*it) != 0) { - fmt::print(Ioss::OUTPUT(), VALUE_MISMATCH, "QA RECORD", in_qa_record_1, (*it)); + fmt::print(buf, VALUE_MISMATCH, "QA RECORD", in_qa_record_1, (*it)); overall_result = false; } } @@ -803,7 +1022,7 @@ namespace { } bool compare_nodeblock(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions & /* options */) + const Ioss::MeshCopyOptions & /* options */, std::ostringstream &buf) { bool overall_result = true; @@ -822,7 +1041,7 @@ namespace { overall_result = false; } else if (!inb->equal(*nb2)) { - fmt::print(Ioss::OUTPUT(), "NODEBLOCK {} mismatch", inb->name()); + fmt::print(buf, "NODEBLOCK {} mismatch", inb->name()); overall_result = false; } } @@ -831,75 +1050,67 @@ namespace { } template - bool compare_blocks(const std::vector &in_blocks_1, - const std::vector &in_blocks_const_2, - const Ioss::MeshCopyOptions & /* options */) + bool compare_blocks(const std::vector &in_blocks_1, const std::vector & in_blocks_2, + const Ioss::MeshCopyOptions & /* options */, std::ostringstream &buf) { bool overall_result = true; - if (in_blocks_1.size() != in_blocks_const_2.size()) { - fmt::print(Ioss::WARNING(), COUNT_MISMATCH, "BLOCK", in_blocks_1.size(), - in_blocks_const_2.size()); + if (in_blocks_1.size() != in_blocks_2.size()) { + fmt::print(Ioss::WARNING(), COUNT_MISMATCH, "BLOCK", in_blocks_1.size(), in_blocks_2.size()); return false; } - // COPY the const input vector so that we remove elements as they're matched without - // affecting the original data structure. - std::vector in_blocks_2 = in_blocks_const_2; - - if (!in_blocks_1.empty()) { - for (const auto &in_block_1 : in_blocks_1) { - typename std::vector::const_iterator it; - for (it = in_blocks_2.begin(); it != in_blocks_2.end(); it++) { - if (*(*it) == *in_block_1) { - break; + for (const auto &in_block_1 : in_blocks_1) { + const auto &name = in_block_1->name(); + bool found = false; + for (const auto &in_block_2 : in_blocks_2) { + if (in_block_2->name() == name) { + found = true; + if (!in_block_1->equal(*in_block_2)) { + overall_result = false; } - } - if (it == in_blocks_2.end()) { - fmt::print(Ioss::WARNING(), NOTFOUND_2, "BLOCK", in_block_1->name()); - overall_result = false; - } - else { - // Just to be sure, remove the OUTPUT nodeblock from the container so that we don't - // inadvertently match against it again - in_blocks_2.erase(it); + break; } } + if (!found) { + fmt::print(Ioss::WARNING(), NOTFOUND_2, "BLOCK", in_block_1->name()); + overall_result = false; + } } return overall_result; } bool compare_elementblocks(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options) + const Ioss::MeshCopyOptions &options, std::ostringstream &buf) { const auto &in_ebs_1 = input_region_1.get_element_blocks(); const auto &in_ebs_2 = input_region_2.get_element_blocks(); - if (compare_blocks(in_ebs_1, in_ebs_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nELEMENTBLOCKS mismatch\n"); + if (compare_blocks(in_ebs_1, in_ebs_2, options, buf) == false) { + fmt::print(buf, "\nELEMENTBLOCKS mismatch\n"); return false; } return true; } bool compare_edgeblocks(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options) + const Ioss::MeshCopyOptions &options, std::ostringstream &buf) { const auto &in_ebs_1 = input_region_1.get_edge_blocks(); const auto &in_ebs_2 = input_region_2.get_edge_blocks(); - if (compare_blocks(in_ebs_1, in_ebs_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nEDGEBLOCKS mismatch\n"); + if (compare_blocks(in_ebs_1, in_ebs_2, options, buf) == false) { + fmt::print(buf, "\nEDGEBLOCKS mismatch\n"); return false; } return true; } bool compare_faceblocks(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options) + const Ioss::MeshCopyOptions &options, std::ostringstream &buf) { const auto &in_fbs_1 = input_region_1.get_face_blocks(); const auto &in_fbs_2 = input_region_2.get_face_blocks(); - if (compare_blocks(in_fbs_1, in_fbs_2, options) == false) { - fmt::print(Ioss::OUTPUT(), "\nFACEBLOCKS mismatch\n"); + if (compare_blocks(in_fbs_1, in_fbs_2, options, buf) == false) { + fmt::print(buf, "\nFACEBLOCKS mismatch\n"); return false; } return true; @@ -907,7 +1118,8 @@ namespace { bool compare_structuredblocks(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions & /* options */) + const Ioss::MeshCopyOptions & /* options */, + std::ostringstream &buf) { bool overall_result = true; @@ -924,32 +1136,29 @@ namespace { return false; } - if (!in_blocks_1.empty()) { - for (const auto &in_block_1 : in_blocks_1) { - std::vector::iterator it; - for (it = in_blocks_2.begin(); it != in_blocks_2.end(); it++) { - if (*(*it) == *in_block_1) { - break; + for (const auto &in_block_1 : in_blocks_1) { + const auto &name = in_block_1->name(); + bool found = false; + for (const auto &in_block_2 : in_blocks_2) { + if (in_block_2->name() == name) { + found = true; + if (!in_block_1->equal(*in_block_2)) { + overall_result = false; } - } - if (it == in_blocks_2.end()) { - fmt::print(Ioss::WARNING(), NOTFOUND_2, "", in_block_1->name()); - overall_result = false; - } - else { - // Just to be sure, remove the OUTPUT nodeblock from the container so that we don't - // inadvertently match against it again - in_blocks_2.erase(it); + break; } } + if (!found) { + fmt::print(Ioss::WARNING(), NOTFOUND_2, "", in_block_1->name()); + overall_result = false; + } } - return overall_result; } template bool compare_sets(const std::vector &in_sets_1, const std::vector &in_sets_const_2, - const Ioss::MeshCopyOptions & /* options */) + const Ioss::MeshCopyOptions & /* options */, std::ostringstream &buf) { bool overall_result = true; @@ -964,21 +1173,23 @@ namespace { if (!in_sets_1.empty()) { for (const auto &in_set_1 : in_sets_1) { - typename std::vector::const_iterator it; - for (it = in_sets_2.begin(); it != in_sets_2.end(); it++) { - if (*(*it) == *in_set_1) { + const auto &name = in_set_1->name(); + // find a set in `in_sets_2` with the same name. + // if found, compare for equality... + bool found = false; + for (const auto &in_set_2 : in_sets_2) { + if (in_set_2->name() == name) { + found = true; + if (!in_set_1->equal(*in_set_2)) { + overall_result = false; + } break; } } - if (it == in_sets_2.end()) { + if (!found) { fmt::print(Ioss::WARNING(), NOTFOUND_2, "set", in_set_1->name()); overall_result = false; } - else { - // Just to be sure, remove the OUTPUT set from the container so that we don't - // inadvertently match against it again - in_sets_2.erase(it); - } } } @@ -986,155 +1197,121 @@ namespace { } bool compare_nodesets(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options) + const Ioss::MeshCopyOptions &options, std::ostringstream &buf) { const auto &in_nss_1 = input_region_1.get_nodesets(); const auto &in_nss_2 = input_region_2.get_nodesets(); - bool rc = compare_sets(in_nss_1, in_nss_2, options); + bool rc = compare_sets(in_nss_1, in_nss_2, options, buf); if (!rc) { - fmt::print(Ioss::OUTPUT(), "\nNODESET mismatch\n"); + fmt::print(buf, "\nNODESET mismatch\n"); } return rc; } bool compare_edgesets(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options) + const Ioss::MeshCopyOptions &options, std::ostringstream &buf) { const auto &in_ess_1 = input_region_1.get_edgesets(); const auto &in_ess_2 = input_region_2.get_edgesets(); - bool rc = compare_sets(in_ess_1, in_ess_2, options); + bool rc = compare_sets(in_ess_1, in_ess_2, options, buf); if (!rc) { - fmt::print(Ioss::OUTPUT(), "\nEDGESET mismatch\n"); + fmt::print(buf, "\nEDGESET mismatch\n"); } return rc; } bool compare_facesets(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options) + const Ioss::MeshCopyOptions &options, std::ostringstream &buf) { const auto &in_fss_1 = input_region_1.get_facesets(); const auto &in_fss_2 = input_region_2.get_facesets(); - bool rc = compare_sets(in_fss_1, in_fss_2, options); + bool rc = compare_sets(in_fss_1, in_fss_2, options, buf); if (!rc) { - fmt::print(Ioss::OUTPUT(), "\nFACESET mismatch\n"); + fmt::print(buf, "\nFACESET mismatch\n"); } return rc; } bool compare_elemsets(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options) + const Ioss::MeshCopyOptions &options, std::ostringstream &buf) { const auto &in_ess_1 = input_region_1.get_elementsets(); const auto &in_ess_2 = input_region_2.get_elementsets(); - bool rc = compare_sets(in_ess_1, in_ess_2, options); + bool rc = compare_sets(in_ess_1, in_ess_2, options, buf); if (!rc) { - fmt::print(Ioss::OUTPUT(), "\nELEMSET mismatch\n"); + fmt::print(buf, "\nELEMSET mismatch\n"); } return rc; } bool compare_sidesets(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions &options) + const Ioss::MeshCopyOptions &options, std::ostringstream &buf) { const auto &in_sss_1 = input_region_1.get_sidesets(); const auto &in_sss_2 = input_region_2.get_sidesets(); - bool rc = compare_sets(in_sss_1, in_sss_2, options); + bool rc = compare_sets(in_sss_1, in_sss_2, options, buf); if (!rc) { - fmt::print(Ioss::OUTPUT(), "\nSIDESET mismatch\n"); + fmt::print(buf, "\nSIDESET mismatch\n"); } return rc; } bool compare_commsets(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions & /* options */) + const Ioss::MeshCopyOptions &options, std::ostringstream &buf) { - bool overall_result = true; + const auto &in_css_1 = input_region_1.get_commsets(); + const auto &in_css_2 = input_region_2.get_commsets(); - const auto &in_css_1 = input_region_1.get_commsets(); - const auto &in_css_orig_2 = input_region_2.get_commsets(); - - if (in_css_1.size() != in_css_orig_2.size()) { - fmt::print(Ioss::WARNING(), COUNT_MISMATCH, "COMMSET", in_css_1.size(), in_css_orig_2.size()); - return false; - } - - // COPY the const input vector so that we remove elements as they're matched without - // affecting the original data structure. - std::vector in_css_2 = in_css_orig_2; - - if (!in_css_1.empty()) { - for (const auto &in_cs_1 : in_css_1) { - typename std::vector::const_iterator it; - for (it = in_css_2.begin(); it != in_css_2.end(); it++) { - if (*(*it) == *in_cs_1) { - break; - } - } - if (it == in_css_2.end()) { - fmt::print(Ioss::WARNING(), NOTFOUND_2, "COMMSET", in_cs_1->name()); - overall_result = false; - } - else { - // Just to be sure, remove the OUTPUT set from the container so that we don't - // inadvertently match against it again - in_css_2.erase(it); - } - } + bool rc = compare_sets(in_css_1, in_css_2, options, buf); + if (!rc) { + fmt::print(buf, "\nCOMMSET mismatch\n"); } - - return overall_result; + return rc; } bool compare_coordinate_frames(const Ioss::Region &input_region_1, const Ioss::Region &input_region_2, - const Ioss::MeshCopyOptions & /* options */) + const Ioss::MeshCopyOptions & /* options */, + std::ostringstream &buf) { bool overall_result = true; - const auto &in_cfs_1 = input_region_1.get_coordinate_frames(); - const auto &in_cfs_orig_2 = input_region_2.get_coordinate_frames(); + const auto &in_cfs_1 = input_region_1.get_coordinate_frames(); + const auto &in_cfs_2 = input_region_2.get_coordinate_frames(); - if (in_cfs_1.size() != in_cfs_orig_2.size()) { + if (in_cfs_1.size() != in_cfs_2.size()) { fmt::print(Ioss::WARNING(), COUNT_MISMATCH, "COORDINATE FRAME", in_cfs_1.size(), - in_cfs_orig_2.size()); + in_cfs_2.size()); return false; } - // COPY the const input vector so that we remove elements as they're matched without - // affecting the original data structure. - std::vector in_cfs_2 = in_cfs_orig_2; - - if (!in_cfs_1.empty()) { - for (const auto &in_cf_1 : in_cfs_1) { - typename std::vector::const_iterator it; - for (it = in_cfs_2.begin(); it != in_cfs_2.end(); it++) { - if ((*it) == in_cf_1) { - break; + for (const auto &in_cf_1 : in_cfs_1) { + bool found = false; + for (const auto &in_cf_2 : in_cfs_2) { + if (in_cf_1.id() == in_cf_2.id()) { + found = true; + if (!in_cf_1.equal(in_cf_2)) { + overall_result = false; } - } - if (it == in_cfs_2.end()) { - fmt::print(Ioss::WARNING(), NOTFOUND_2, "COORDINATE FRAME", in_cf_1.id()); - overall_result = false; - } - else { - // Just to be sure, remove the OUTPUT set from the container so that we don't - // inadvertently match against it again - in_cfs_2.erase(it); + break; } } + if (!found) { + fmt::print(Ioss::WARNING(), NOTFOUND_2, "COORDINATE FRAME", in_cf_1.id()); + overall_result = false; + } } - return overall_result; } template bool compare_fields(const std::vector &in_entities_1, const std::vector &in_entities_2, - const Ioss::Field::RoleType role) + const Ioss::Field::RoleType role, std::ostringstream &buf) { bool overall_result = true; @@ -1159,14 +1336,14 @@ namespace { continue; } - overall_result &= compare_fields(in_entity_1, (*it), role); + overall_result &= compare_fields(in_entity_1, (*it), role, buf); } return overall_result; } bool compare_fields(const Ioss::GroupingEntity *ige_1, const Ioss::GroupingEntity *ige_2, - const Ioss::Field::RoleType role) + const Ioss::Field::RoleType role, std::ostringstream &buf) { // Check for transient fields... Ioss::NameList in_fields_1; @@ -1180,24 +1357,26 @@ namespace { return false; } + bool result = true; // Iterate through results fields and transfer to output // database... If a prefix is specified, only transfer fields // whose names begin with the prefix for (const auto &field_name : in_fields_1) { - Ioss::Field ige_field_1 = ige_1->get_field(field_name); - Ioss::Field ige_field_2 = ige_2->get_field(field_name); - if (ige_field_1 != ige_field_2) { - fmt::print(Ioss::OUTPUT(), "FIELD ({}) mismatch\n", field_name); + const Ioss::Field &ige_field_1 = ige_1->get_field(field_name); + const Ioss::Field &ige_field_2 = ige_2->get_field(field_name); + if (!ige_field_1.equal(ige_field_2)) { + fmt::print(buf, "\n\tFIELD ({}) mismatch", field_name); + result = false; } } - - return true; + return result; } template bool compare_field_data(const std::vector &in_entities_1, const std::vector &in_entities_2, DataPool &pool, - const Ioss::Field::RoleType role, const Ioss::MeshCopyOptions &options) + const Ioss::Field::RoleType role, const Ioss::MeshCopyOptions &options, + std::ostringstream &buf) { bool overall_result = true; @@ -1222,7 +1401,7 @@ namespace { continue; } - overall_result &= compare_field_data(in_entity_1, (*it), pool, role, options); + overall_result &= compare_field_data(in_entity_1, (*it), pool, role, options, buf); } return overall_result; @@ -1230,7 +1409,8 @@ namespace { bool compare_field_data(const Ioss::GroupingEntity *ige_1, const Ioss::GroupingEntity *ige_2, DataPool &pool, const Ioss::Field::RoleType role, - const Ioss::MeshCopyOptions &options, const std::string &prefix) + const Ioss::MeshCopyOptions &options, std::ostringstream &buf, + const std::string &prefix) { bool overall_result = true; @@ -1245,7 +1425,7 @@ namespace { // Complication here is that if the 'role' is 'Ioss::Field::MESH', // then the 'ids' field must be transferred first... if (ige_1->field_exists("ids") != ige_2->field_exists("ids")) { - fmt::print(Ioss::OUTPUT(), + fmt::print(buf, "FIELD data: field MISMATCH --> " "ige_1->field_exists(\"ids\") = {} / ige_2->field_exists(\"ids\") = {}\n", ige_1->field_exists("ids"), ige_2->field_exists("ids")); @@ -1254,7 +1434,7 @@ namespace { if (role == Ioss::Field::MESH && ige_1->field_exists("ids")) { assert(ige_2->field_exists("ids")); - overall_result &= compare_field_data_internal(ige_1, ige_2, pool, "ids", options); + overall_result &= compare_field_data_internal(ige_1, ige_2, pool, "ids", options, buf); } for (const auto &field_name : in_state_fields_1) { @@ -1271,17 +1451,37 @@ namespace { } if (Ioss::Utils::substr_equal(prefix, field_name)) { assert(ige_2->field_exists(field_name)); - overall_result &= compare_field_data_internal(ige_1, ige_2, pool, field_name, options); + overall_result &= compare_field_data_internal(ige_1, ige_2, pool, field_name, options, buf); } } return overall_result; } + template + bool compare_field_data(T *data1, T *data2, size_t count, const std::string &field_name, + std::ostringstream &buf) + { + bool first = true; + for (size_t i = 0; i < count; i++) { + if (data1[i] != data2[i]) { + if (first) { + fmt::print(buf, "\n\tFIELD ({}) mismatch at index[{}]: {} vs. {}", field_name, i, + data1[i], data2[i]); + first = false; + } + else { + fmt::print(buf, ", [{}]: {} vs. {}", i, data1[i], data2[i]); + } + } + } + return first; + } + bool compare_field_data_internal(const Ioss::GroupingEntity *ige_1, const Ioss::GroupingEntity *ige_2, DataPool &in_pool, const std::string & field_name, - const Ioss::MeshCopyOptions &options) + const Ioss::MeshCopyOptions &options, std::ostringstream &buf) { size_t isize = ige_1->get_field(field_name).get_size(); size_t osize = ige_2->get_field(field_name).get_size(); @@ -1289,7 +1489,9 @@ namespace { DataPool in_pool_2; if (isize != osize) { - fmt::print(Ioss::WARNING(), "FIELD size mismatch ({} vs. {})\n", isize, osize); + fmt::print(buf, "\n\tFIELD size mismatch for field '{}', ({} vs. {})", field_name, isize, + osize); + return false; } if (field_name == "mesh_model_coordinates_x") { @@ -1350,24 +1552,33 @@ namespace { assert(in_pool_2.data.size() >= isize); switch (options.data_storage_type) { - case 1: + case 1: { ige_1->get_field_data(field_name, in_pool.data.data(), isize); ige_2->get_field_data(field_name, in_pool_2.data.data(), isize); - for (unsigned int i = 0; i < isize; i++) { - if (in_pool.data[i] != in_pool_2.data[i]) { - fmt::print(Ioss::OUTPUT(), "FIELD data ({}) mismatch at index[{}]: 0x{} vs. 0x{}", - field_name, i, in_pool.data[i], in_pool_2.data[i]); - return false; - } + const Ioss::Field &field = ige_1->get_field(field_name); + + switch (field.get_type()) { + case Ioss::Field::REAL: + return compare_field_data((double *)in_pool.data.data(), (double *)in_pool_2.data.data(), + field.raw_count(), field_name, buf); + case Ioss::Field::INTEGER: + return compare_field_data((int *)in_pool.data.data(), (int *)in_pool_2.data.data(), + field.raw_count(), field_name, buf); + case Ioss::Field::INT64: + return compare_field_data((int64_t *)in_pool.data.data(), (int64_t *)in_pool_2.data.data(), + field.raw_count(), field_name, buf); + default: + fmt::print(Ioss::WARNING(), "Field data_storage type {} not recognized for field {}.", + field.type_string(), field_name); + return false; } - break; + } break; default: if (field_name == "mesh_model_coordinates") { fmt::print(Ioss::WARNING(), "data_storage option not recognized."); } return false; } - return true; } } // namespace diff --git a/packages/seacas/libraries/ioss/src/Ioss_CopyDatabase.C b/packages/seacas/libraries/ioss/src/Ioss_CopyDatabase.C index d5058d01526a..00c92bac9723 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_CopyDatabase.C +++ b/packages/seacas/libraries/ioss/src/Ioss_CopyDatabase.C @@ -231,8 +231,14 @@ void Ioss::copy_database(Ioss::Region ®ion, Ioss::Region &output_region, // Minimize number of times that we grow the memory buffer used for transferring field data. auto max_field = calculate_maximum_field_size(region); if (options.verbose && rank == 0) { - fmt::print(Ioss::DEBUG(), "\n Maximum Field size = {:L} bytes for field '{}'.\n", - max_field.first, max_field.second); + std::string label = "MiB"; + double size = (double)max_field.first / 1024 / 1024; + if (size > 1024.0) { + label = "GiB"; + size /= 1024.0; + } + fmt::print(Ioss::DEBUG(), "\n Maximum Field size = {:L} bytes ({:.3} {}) for field '{}'.\n", + max_field.first, size, label, max_field.second); } DataPool data_pool; diff --git a/packages/seacas/libraries/ioss/src/Ioss_DatabaseIO.C b/packages/seacas/libraries/ioss/src/Ioss_DatabaseIO.C index 1bf938c97eec..8570ea2eebe5 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_DatabaseIO.C +++ b/packages/seacas/libraries/ioss/src/Ioss_DatabaseIO.C @@ -377,7 +377,7 @@ namespace Ioss { void DatabaseIO::closeDW() const { if (using_dw()) { - if (!using_parallel_io() || (using_parallel_io() && myProcessor == 0)) { + if (!using_parallel_io() || myProcessor == 0) { #if defined SEACAS_HAVE_DATAWARP int complete = 0, pending = 0, deferred = 0, failed = 0; dw_query_file_stage(get_dwname().c_str(), &complete, &pending, &deferred, &failed); @@ -717,7 +717,6 @@ namespace Ioss { // This is used in other code speed up some tests. // Spheres and Circle have no faces/edges, so handle them special... - bool all_sphere = true; if (sideTopology.empty()) { // Set contains (parent_element, boundary_topology) pairs... @@ -725,6 +724,7 @@ namespace Ioss { const ElementBlockContainer &element_blocks = get_region()->get_element_blocks(); + bool all_sphere = true; for (auto &block : element_blocks) { const ElementTopology *elem_type = block->topology(); const ElementTopology *side_type = elem_type->boundary_type(); diff --git a/packages/seacas/libraries/ioss/src/Ioss_Decomposition.C b/packages/seacas/libraries/ioss/src/Ioss_Decomposition.C index f3cee1c27db7..00792c5cd967 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_Decomposition.C +++ b/packages/seacas/libraries/ioss/src/Ioss_Decomposition.C @@ -220,10 +220,9 @@ namespace Ioss { // First iterate the local element indices and count number in // each block. - size_t b = 0; for (auto loc_elem : localElementMap) { size_t elem = loc_elem + m_elementOffset; - b = Ioss::Utils::find_index_location(elem, m_fileBlockIndex); + size_t b = Ioss::Utils::find_index_location(elem, m_fileBlockIndex); assert(elem >= m_fileBlockIndex[b] && elem < m_fileBlockIndex[b + 1]); size_t off = std::max(m_fileBlockIndex[b], m_elementOffset); @@ -233,7 +232,6 @@ namespace Ioss { // Now iterate the imported element list... // Find number of imported elements that are less than the current // local_map[0] - b = 0; size_t proc = 0; std::vector imp_index(el_blocks.size()); for (size_t i = 0; i < importElementMap.size(); i++) { @@ -242,7 +240,7 @@ namespace Ioss { proc++; } - b = Ioss::Utils::find_index_location(elem, m_fileBlockIndex); + size_t b = Ioss::Utils::find_index_location(elem, m_fileBlockIndex); size_t off = std::max(m_fileBlockIndex[b], m_elementOffset); if (!el_blocks[b].localMap.empty() && elem < el_blocks[b].localMap[0] + off) { @@ -257,14 +255,13 @@ namespace Ioss { // Now for the exported data... proc = 0; - b = 0; for (size_t i = 0; i < exportElementMap.size(); i++) { size_t elem = exportElementMap[i]; while (i >= (size_t)exportElementIndex[proc + 1]) { proc++; } - b = Ioss::Utils::find_index_location(elem, m_fileBlockIndex); + size_t b = Ioss::Utils::find_index_location(elem, m_fileBlockIndex); size_t off = std::max(m_fileBlockIndex[b], m_elementOffset); el_blocks[b].exportMap.push_back(elem - off); diff --git a/packages/seacas/libraries/ioss/src/Ioss_FaceGenerator.C b/packages/seacas/libraries/ioss/src/Ioss_FaceGenerator.C index cbaca286c24a..194edf595fc1 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_FaceGenerator.C +++ b/packages/seacas/libraries/ioss/src/Ioss_FaceGenerator.C @@ -5,6 +5,7 @@ // See packages/seacas/LICENSE for details #include + #include #include #include @@ -80,7 +81,7 @@ namespace { std::vector elem_ids; if (local_ids) { elem_ids.resize(eb->entity_count()); - std::iota(elem_ids.begin(), elem_ids.end(), eb->get_offset() + 1); + std::iota(elem_ids.begin(), elem_ids.end(), static_cast(eb->get_offset() + 1)); } else { eb->get_field_data("ids", elem_ids); diff --git a/packages/seacas/libraries/ioss/src/Ioss_Field.C b/packages/seacas/libraries/ioss/src/Ioss_Field.C index 9d5d58bdaa66..544a2a905f94 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_Field.C +++ b/packages/seacas/libraries/ioss/src/Ioss_Field.C @@ -21,27 +21,14 @@ namespace { size_t internal_get_size(Ioss::Field::BasicType type, size_t count, const Ioss::VariableType *storage); - std::string type_string(Ioss::Field::BasicType type) - { - switch (type) { - case Ioss::Field::REAL: return std::string("real"); - case Ioss::Field::INTEGER: return std::string("integer"); - case Ioss::Field::INT64: return std::string("64-bit integer"); - case Ioss::Field::COMPLEX: return std::string("complex"); - case Ioss::Field::STRING: return std::string("string"); - case Ioss::Field::CHARACTER: return std::string("char"); - case Ioss::Field::INVALID: return std::string("invalid"); - default: return std::string("internal error"); - } - } - void error_message(const Ioss::Field &field, Ioss::Field::BasicType requested_type) { std::ostringstream errmsg; fmt::print(errmsg, "ERROR: For field named '{}', code requested value of type '{}', but field type is " "'{}'. Types must match\n", - field.get_name(), type_string(requested_type), type_string(field.get_type())); + field.get_name(), Ioss::Field::type_string(requested_type), + Ioss::Field::type_string(field.get_type())); IOSS_ERROR(errmsg); } } // namespace @@ -239,33 +226,32 @@ bool Ioss::Field::transform(void *data) return true; } -bool Ioss::Field::equal_(const Ioss::Field rhs, bool quiet) const +bool Ioss::Field::equal_(const Ioss::Field &rhs, bool quiet) const { if (Ioss::Utils::str_equal(this->name_, rhs.name_) == false) { if (!quiet) { - fmt::print(Ioss::OUTPUT(), "FIELD name mismatch ({} v. {})\n", this->name_.c_str(), - rhs.name_.c_str()); + fmt::print(Ioss::OUTPUT(), "\n\tFIELD name mismatch ({} v. {})", this->name_, rhs.name_); } return false; } if (this->type_ != rhs.type_) { if (!quiet) { - fmt::print(Ioss::OUTPUT(), "FIELD type mismatch ({} v. {})\n", this->type_, rhs.type_); + fmt::print(Ioss::OUTPUT(), "\n\tFIELD type mismatch ({} v. {})", this->type_, rhs.type_); } return false; } if (this->role_ != rhs.role_) { if (!quiet) { - fmt::print(Ioss::OUTPUT(), "FIELD role mismatch ({} v. {})\n", this->role_, rhs.role_); + fmt::print(Ioss::OUTPUT(), "\n\tFIELD role mismatch ({} v. {})", this->role_, rhs.role_); } return false; } if (this->rawCount_ != rhs.rawCount_) { if (!quiet) { - fmt::print(Ioss::OUTPUT(), "FIELD rawCount mismatch ({} v. {})\n", this->rawCount_, + fmt::print(Ioss::OUTPUT(), "\n\tFIELD rawCount mismatch ({} v. {})", this->rawCount_, rhs.rawCount_); } return false; @@ -273,7 +259,7 @@ bool Ioss::Field::equal_(const Ioss::Field rhs, bool quiet) const if (this->transCount_ != rhs.transCount_) { if (!quiet) { - fmt::print(Ioss::OUTPUT(), "FIELD transCount mismatch ({} v. {})\n", this->transCount_, + fmt::print(Ioss::OUTPUT(), "\n\tFIELD transCount mismatch ({} v. {})", this->transCount_, rhs.transCount_); } return false; @@ -281,7 +267,7 @@ bool Ioss::Field::equal_(const Ioss::Field rhs, bool quiet) const if (this->get_size() != rhs.get_size()) { if (!quiet) { - fmt::print(Ioss::OUTPUT(), "FIELD size mismatch ({} v. {})\n", this->get_size(), + fmt::print(Ioss::OUTPUT(), "\n\tFIELD size mismatch ({} v. {})", this->get_size(), rhs.get_size()); } return false; @@ -290,11 +276,27 @@ bool Ioss::Field::equal_(const Ioss::Field rhs, bool quiet) const return true; } -bool Ioss::Field::operator==(const Ioss::Field rhs) const { return equal_(rhs, true); } +bool Ioss::Field::operator==(const Ioss::Field &rhs) const { return equal_(rhs, true); } -bool Ioss::Field::operator!=(const Ioss::Field rhs) const { return !(*this == rhs); } +bool Ioss::Field::operator!=(const Ioss::Field &rhs) const { return !(*this == rhs); } -bool Ioss::Field::equal(const Ioss::Field rhs) const { return equal_(rhs, false); } +bool Ioss::Field::equal(const Ioss::Field &rhs) const { return equal_(rhs, false); } + +std::string Ioss::Field::type_string() const { return type_string(get_type()); } + +std::string Ioss::Field::type_string(Ioss::Field::BasicType type) +{ + switch (type) { + case Ioss::Field::REAL: return std::string("real"); + case Ioss::Field::INTEGER: return std::string("integer"); + case Ioss::Field::INT64: return std::string("64-bit integer"); + case Ioss::Field::COMPLEX: return std::string("complex"); + case Ioss::Field::STRING: return std::string("string"); + case Ioss::Field::CHARACTER: return std::string("char"); + case Ioss::Field::INVALID: return std::string("invalid"); + default: return std::string("internal error"); + } +} namespace { size_t internal_get_size(Ioss::Field::BasicType type, size_t count, diff --git a/packages/seacas/libraries/ioss/src/Ioss_Field.h b/packages/seacas/libraries/ioss/src/Ioss_Field.h index a1482424db4a..bec1dd70041d 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_Field.h +++ b/packages/seacas/libraries/ioss/src/Ioss_Field.h @@ -1,4 +1,4 @@ -// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// Copyright(C) 1999-2021 National Technology & Engineering Solutions // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with // NTESS, the U.S. Government retains certain rights in this software. // @@ -97,9 +97,9 @@ namespace Ioss { // Compare two fields (used for STL container) bool operator<(const Field &other) const; - bool operator==(const Ioss::Field rhs) const; - bool operator!=(const Ioss::Field rhs) const; - bool equal(const Ioss::Field rhs) const; + bool operator==(const Ioss::Field &rhs) const; + bool operator!=(const Ioss::Field &rhs) const; + bool equal(const Ioss::Field &rhs) const; ~Field(); @@ -143,7 +143,9 @@ namespace Ioss { // throws exception if the types don't match. void check_type(BasicType the_type) const; - bool is_type(BasicType the_type) const { return the_type == type_; } + bool is_type(BasicType the_type) const { return the_type == type_; } + std::string type_string() const; + static std::string type_string(BasicType type); bool add_transform(Transform *my_transform); bool transform(void *data); @@ -164,7 +166,7 @@ namespace Ioss { const VariableType *transStorage_{}; // Storage type after transformation std::vector transforms_; - bool equal_(const Ioss::Field rhs, bool quiet) const; + bool equal_(const Ioss::Field &rhs, bool quiet) const; }; } // namespace Ioss #endif diff --git a/packages/seacas/libraries/ioss/src/Ioss_Glob.h b/packages/seacas/libraries/ioss/src/Ioss_Glob.h index 3b792d38bbed..8c403a060419 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_Glob.h +++ b/packages/seacas/libraries/ioss/src/Ioss_Glob.h @@ -426,27 +426,23 @@ namespace glob { bool Check(const String &str, size_t pos) override { + bool r = false; switch (type_) { case Type::BASIC: case Type::AT: case Type::ANY: case Type::STAR: case Type::PLUS: { - bool r; std::tie(r, std::ignore) = BasicCheck(str, pos); - return r; break; } case Type::NEG: { - bool r; std::tie(r, std::ignore) = BasicCheck(str, pos); - return !r; break; } - - default: return false; break; } + return r; } std::tuple Next(const String &str, size_t pos) override @@ -457,27 +453,22 @@ namespace glob { case Type::BASIC: case Type::AT: { return NextBasic(str, pos); - break; } case Type::ANY: { return NextAny(str, pos); - break; } case Type::STAR: { return NextStar(str, pos); - break; } case Type::PLUS: { return NextPlus(str, pos); - break; } case Type::NEG: { return NextNeg(str, pos); - break; } } return std::tuple(0, 0); @@ -1110,34 +1101,25 @@ namespace glob { Token &tk = GetToken(); switch (tk.Kind()) { - case TokenKind::QUESTION: - Advance(); - return AstNodePtr(new AnyNode()); - break; + case TokenKind::QUESTION: Advance(); return AstNodePtr(new AnyNode()); - case TokenKind::STAR: - Advance(); - return AstNodePtr(new StarNode()); - break; + case TokenKind::STAR: Advance(); return AstNodePtr(new StarNode()); - case TokenKind::SUB: - Advance(); - return AstNodePtr(new CharNode('-')); - break; + case TokenKind::SUB: Advance(); return AstNodePtr(new CharNode('-')); - case TokenKind::CHAR: return ParserChar(); break; + case TokenKind::CHAR: return ParserChar(); case TokenKind::LBRACKET: - case TokenKind::NEGLBRACKET: return ParserSet(); break; + case TokenKind::NEGLBRACKET: return ParserSet(); case TokenKind::LPAREN: case TokenKind::QUESTLPAREN: case TokenKind::STARLPAREN: case TokenKind::PLUSLPAREN: case TokenKind::NEGLPAREN: - case TokenKind::ATLPAREN: return ParserGroup(); break; + case TokenKind::ATLPAREN: return ParserGroup(); - default: throw Error("basic glob expected"); break; + default: throw Error("basic glob expected"); } } @@ -1179,9 +1161,9 @@ namespace glob { switch (tk.Kind()) { case TokenKind::EOS: case TokenKind::RPAREN: - case TokenKind::UNION: return true; break; + case TokenKind::UNION: return true; - default: return false; break; + default: return false; } }; diff --git a/packages/seacas/libraries/ioss/src/Ioss_Hex8.C b/packages/seacas/libraries/ioss/src/Ioss_Hex8.C index 1e383dd271d9..a81f28dfaec9 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_Hex8.C +++ b/packages/seacas/libraries/ioss/src/Ioss_Hex8.C @@ -1,4 +1,4 @@ -// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// Copyright(C) 1999-2021 National Technology & Engineering Solutions // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with // NTESS, the U.S. Government retains certain rights in this software. // diff --git a/packages/seacas/libraries/ioss/src/Ioss_Hex9.C b/packages/seacas/libraries/ioss/src/Ioss_Hex9.C new file mode 100644 index 000000000000..84fb11121a42 --- /dev/null +++ b/packages/seacas/libraries/ioss/src/Ioss_Hex9.C @@ -0,0 +1,170 @@ +// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with +// NTESS, the U.S. Government retains certain rights in this software. +// +// See packages/seacas/LICENSE for details + +#include "Ioss_CodeTypes.h" // for IntVector +#include "Ioss_ElementTopology.h" // for ElementTopology +#include // for ElementVariableType +#include +#include // for assert + +//------------------------------------------------------------------------ +// Define a variable type for storage of this elements connectivity +namespace Ioss { + const char *Hex9::name = "hex9"; + + class St_Hex9 : public ElementVariableType + { + public: + static void factory() { static St_Hex9 registerThis; } + + protected: + St_Hex9() : ElementVariableType(Ioss::Hex9::name, 9) {} + }; +} // namespace Ioss + +// ======================================================================== +namespace { + struct Constants + { + static const int nnode = 9; + static const int nedge = 12; + static const int nedgenode = 2; + static const int nface = 6; + static const int nfacenode = 4; + static const int nfaceedge = 4; + static int edge_node_order[nedge][nedgenode]; + static int face_node_order[nface][nfacenode]; + static int face_edge_order[nface][nfaceedge]; + static int nodes_per_face[nface + 1]; + static int edges_per_face[nface + 1]; + }; + + // Edge numbers are zero-based [0..number_edges) + int Constants::edge_node_order[nedge][nedgenode] = // [edge][edge_node] + {{0, 1}, {1, 2}, {2, 3}, {3, 0}, {4, 5}, {5, 6}, + {6, 7}, {7, 4}, {0, 4}, {1, 5}, {2, 6}, {3, 7}}; + + // Face numbers are zero-based [0..number_faces) + int Constants::face_node_order[nface][nfacenode] = // [face][face_node] + {{0, 1, 5, 4}, {1, 2, 6, 5}, {2, 3, 7, 6}, {0, 4, 7, 3}, {0, 3, 2, 1}, {4, 5, 6, 7}}; + + int Constants::face_edge_order[nface][nfaceedge] = // [face][face_edge] + {{0, 9, 4, 8}, {1, 10, 5, 9}, {2, 11, 6, 10}, {8, 7, 11, 3}, {3, 2, 1, 0}, {4, 5, 6, 7}}; + + // face 0 returns number of nodes for all faces if homogeneous + // returns -1 if faces have differing topology + int Constants::nodes_per_face[nface + 1] = {4, 4, 4, 4, 4, 4, 4}; + + // face 0 returns number of edges for all faces if homogeneous + // returns -1 if faces have differing topology + int Constants::edges_per_face[nface + 1] = {4, 4, 4, 4, 4, 4, 4}; +} // namespace + +void Ioss::Hex9::factory() +{ + static Ioss::Hex9 registerThis; + Ioss::St_Hex9::factory(); +} + +Ioss::Hex9::Hex9() : Ioss::ElementTopology(Ioss::Hex9::name, "Hexahedron_9") +{ + Ioss::ElementTopology::alias(Ioss::Hex9::name, "Solid_Hex_9_3D"); +} + +Ioss::Hex9::~Hex9() = default; + +int Ioss::Hex9::parametric_dimension() const { return 3; } +int Ioss::Hex9::spatial_dimension() const { return 3; } +int Ioss::Hex9::order() const { return 1; } + +int Ioss::Hex9::number_corner_nodes() const { return 8; } +int Ioss::Hex9::number_nodes() const { return Constants::nnode; } +int Ioss::Hex9::number_edges() const { return Constants::nedge; } +int Ioss::Hex9::number_faces() const { return Constants::nface; } + +int Ioss::Hex9::number_nodes_edge(int /* edge */) const { return Constants::nedgenode; } + +int Ioss::Hex9::number_nodes_face(int face) const +{ + // face is 1-based. 0 passed in for all faces. + assert(face >= 0 && face <= number_faces()); + return Constants::nodes_per_face[face]; +} + +int Ioss::Hex9::number_edges_face(int face) const +{ + // face is 1-based. 0 passed in for all faces. + assert(face >= 0 && face <= number_faces()); + return Constants::edges_per_face[face]; +} + +Ioss::IntVector Ioss::Hex9::edge_connectivity(int edge_number) const +{ + assert(edge_number > 0 && edge_number <= Constants::nedge); + Ioss::IntVector connectivity(Constants::nedgenode); + + for (int i = 0; i < Constants::nedgenode; i++) { + connectivity[i] = Constants::edge_node_order[edge_number - 1][i]; + } + + return connectivity; +} + +Ioss::IntVector Ioss::Hex9::face_connectivity(int face_number) const +{ + assert(face_number > 0 && face_number <= number_faces()); + Ioss::IntVector connectivity(Constants::nodes_per_face[face_number]); + + for (int i = 0; i < Constants::nodes_per_face[face_number]; i++) { + connectivity[i] = Constants::face_node_order[face_number - 1][i]; + } + + return connectivity; +} + +Ioss::IntVector Ioss::Hex9::element_connectivity() const +{ + Ioss::IntVector connectivity(number_nodes()); + for (int i = 0; i < number_nodes(); i++) { + connectivity[i] = i; + } + return connectivity; +} + +Ioss::ElementTopology *Ioss::Hex9::face_type(int face_number) const +{ + // face_number == 0 returns topology for all faces if + // all faces are the same topology; otherwise, returns nullptr + // face_number is 1-based. + + assert(face_number >= 0 && face_number <= number_faces()); + // return Ioss::ElementTopology::factory("quadface4"); + return Ioss::ElementTopology::factory("quad4"); +} + +Ioss::ElementTopology *Ioss::Hex9::edge_type(int edge_number) const +{ + // edge_number == 0 returns topology for all edges if + // all edges are the same topology; otherwise, returns nullptr + // edge_number is 1-based. + + assert(edge_number >= 0 && edge_number <= number_edges()); + return Ioss::ElementTopology::factory("edge2"); +} + +Ioss::IntVector Ioss::Hex9::face_edge_connectivity(int face_number) const +{ + assert(face_number > 0 && face_number <= Constants::nface); + + int nface_edge = number_edges_face(face_number); + Ioss::IntVector fcon(nface_edge); + + for (int i = 0; i < nface_edge; i++) { + fcon[i] = Constants::face_edge_order[face_number - 1][i]; + } + + return fcon; +} diff --git a/packages/seacas/libraries/ioss/src/Ioss_Hex9.h b/packages/seacas/libraries/ioss/src/Ioss_Hex9.h new file mode 100644 index 000000000000..703f21907084 --- /dev/null +++ b/packages/seacas/libraries/ioss/src/Ioss_Hex9.h @@ -0,0 +1,58 @@ +// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with +// NTESS, the U.S. Government retains certain rights in this software. +// +// See packages/seacas/LICENSE for details + +#ifndef IOSS_Ioss_Hex9_h +#define IOSS_Ioss_Hex9_h + +#include // for IntVector +#include // for ElementTopology + +// STL Includes + +namespace Ioss { + class Hex9 : public Ioss::ElementTopology + { + + public: + static const char *name; + + static void factory(); + ~Hex9() override; + + ElementShape shape() const override { return ElementShape::HEX; } + int spatial_dimension() const override; + int parametric_dimension() const override; + bool is_element() const override { return true; } + int order() const override; + + int number_corner_nodes() const override; + int number_nodes() const override; + int number_edges() const override; + int number_faces() const override; + + int number_nodes_edge(int edge = 0) const override; + int number_nodes_face(int face = 0) const override; + int number_edges_face(int face = 0) const override; + + Ioss::IntVector edge_connectivity(int edge_number) const override; + Ioss::IntVector face_connectivity(int face_number) const override; + Ioss::IntVector element_connectivity() const override; + + Ioss::IntVector face_edge_connectivity(int face_number) const override; + + Ioss::ElementTopology *face_type(int face_number = 0) const override; + Ioss::ElementTopology *edge_type(int edge_number = 0) const override; + + protected: + Hex9(); + + private: + static Hex9 instance_; + + Hex9(const Hex9 &) = delete; + }; +} // namespace Ioss +#endif diff --git a/packages/seacas/libraries/ioss/src/Ioss_IOFactory.C b/packages/seacas/libraries/ioss/src/Ioss_IOFactory.C index e45d9555d63d..890e1d75654b 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_IOFactory.C +++ b/packages/seacas/libraries/ioss/src/Ioss_IOFactory.C @@ -129,6 +129,12 @@ std::string Ioss::IOFactory::show_configuration() #if defined(SEACAS_HAVE_MPI) fmt::print(config, "\nSupported decomposition methods:\n\t{}\n", fmt::join(Ioss::valid_decomp_methods(), ", ")); + { + char version[MPI_MAX_LIBRARY_VERSION_STRING]; + int length = 0; + MPI_Get_library_version(version, &length); + fmt::print(config, "MPI Version: {}\n", version); + } #endif fmt::print(config, "\nThird-Party Library Configuration Information:\n\n"); diff --git a/packages/seacas/libraries/ioss/src/Ioss_Initializer.C b/packages/seacas/libraries/ioss/src/Ioss_Initializer.C index 649ae51ae839..56628a7b39c1 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_Initializer.C +++ b/packages/seacas/libraries/ioss/src/Ioss_Initializer.C @@ -29,6 +29,7 @@ Ioss::Initializer::Initializer() Ioss::ShellLine2D3::factory(); Ioss::Hex8::factory(); + Ioss::Hex9::factory(); Ioss::Hex16::factory(); Ioss::Hex20::factory(); Ioss::Hex27::factory(); diff --git a/packages/seacas/libraries/ioss/src/Ioss_MeshCopyOptions.h b/packages/seacas/libraries/ioss/src/Ioss_MeshCopyOptions.h index 125c58e68585..26691d0254d6 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_MeshCopyOptions.h +++ b/packages/seacas/libraries/ioss/src/Ioss_MeshCopyOptions.h @@ -16,15 +16,18 @@ namespace Ioss { double minimum_time{0.0}; double maximum_time{0.0}; double delay{0.0}; - int data_storage_type{0}; - bool memory_statistics{false}; - bool debug{false}; - bool verbose{false}; - bool ints_64_bit{false}; - bool delete_timesteps{false}; - bool reverse{false}; // Used for testing CGNS - bool add_proc_id{false}; // CGNS: Add proc_id field. - bool boundary_sideset{false}; // Output a sideset of the boundary faces of the model + + // POINTER=1, STD_VECTOR=2, KOKKOS_VIEW_1D=3, KOKKOS_VIEW_2D=4, + // KOKKOS_VIEW_2D_LAYOUTRIGHT_HOSTSPACE=5 + int data_storage_type{0}; + bool memory_statistics{false}; + bool debug{false}; + bool verbose{false}; + bool ints_64_bit{false}; + bool delete_timesteps{false}; + bool reverse{false}; // Used for testing CGNS + bool add_proc_id{false}; // CGNS: Add proc_id field. + bool boundary_sideset{false}; // Output a sideset of the boundary faces of the model // only used by Catalyst calls to `copy_database`; if false the // copy process skips the defining of the mesh geometry and the diff --git a/packages/seacas/libraries/ioss/src/Ioss_Property.C b/packages/seacas/libraries/ioss/src/Ioss_Property.C index c3a92c7d7a3e..8e335c432268 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_Property.C +++ b/packages/seacas/libraries/ioss/src/Ioss_Property.C @@ -186,7 +186,7 @@ Ioss::Property &Ioss::Property::operator=(Ioss::Property rhs) return *this; } -bool Ioss::Property::operator==(const Ioss::Property rhs) const +bool Ioss::Property::operator==(const Ioss::Property &rhs) const { if (this->name_.compare(rhs.name_) != 0) { return false; @@ -249,7 +249,7 @@ bool Ioss::Property::operator==(const Ioss::Property rhs) const return true; } -bool Ioss::Property::operator!=(const Ioss::Property rhs) const { return !(*this == rhs); } +bool Ioss::Property::operator!=(const Ioss::Property &rhs) const { return !(*this == rhs); } /** \brief Get the property value if it is of type STRING. * diff --git a/packages/seacas/libraries/ioss/src/Ioss_Property.h b/packages/seacas/libraries/ioss/src/Ioss_Property.h index 8259cb575d0a..10d4d8692af3 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_Property.h +++ b/packages/seacas/libraries/ioss/src/Ioss_Property.h @@ -1,4 +1,4 @@ -// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// Copyright(C) 1999-2021 National Technology & Engineering Solutions // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with // NTESS, the U.S. Government retains certain rights in this software. // @@ -97,8 +97,8 @@ namespace Ioss { */ BasicType get_type() const { return type_; } - bool operator!=(const Ioss::Property rhs) const; - bool operator==(const Ioss::Property rhs) const; + bool operator!=(const Ioss::Property &rhs) const; + bool operator==(const Ioss::Property &rhs) const; private: std::string name_{}; diff --git a/packages/seacas/libraries/ioss/src/Ioss_Region.C b/packages/seacas/libraries/ioss/src/Ioss_Region.C index b91d24b0f968..cebb2bee13a4 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_Region.C +++ b/packages/seacas/libraries/ioss/src/Ioss_Region.C @@ -4,6 +4,8 @@ // // See packages/seacas/LICENSE for details +#include + #include #include #include diff --git a/packages/seacas/libraries/ioss/src/Ioss_ScopeGuard.h b/packages/seacas/libraries/ioss/src/Ioss_ScopeGuard.h index e58c34aef406..092246d46c92 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_ScopeGuard.h +++ b/packages/seacas/libraries/ioss/src/Ioss_ScopeGuard.h @@ -27,7 +27,7 @@ template inline RefHolder ByRef(T &t) { return RefHolder(t); } class ScopeGuardImplBase { - ScopeGuardImplBase &operator=(const ScopeGuardImplBase &); + ScopeGuardImplBase &operator=(const ScopeGuardImplBase &) = delete; protected: ~ScopeGuardImplBase() = default; diff --git a/packages/seacas/libraries/ioss/src/Ioss_StandardElementTypes.h b/packages/seacas/libraries/ioss/src/Ioss_StandardElementTypes.h index a7c22dc8428c..9f132fc4b380 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_StandardElementTypes.h +++ b/packages/seacas/libraries/ioss/src/Ioss_StandardElementTypes.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include diff --git a/packages/seacas/libraries/ioss/src/Ioss_Utils.C b/packages/seacas/libraries/ioss/src/Ioss_Utils.C index 3df165204431..f44c47764d36 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_Utils.C +++ b/packages/seacas/libraries/ioss/src/Ioss_Utils.C @@ -1019,10 +1019,9 @@ unsigned int Ioss::Utils::hash(const std::string &name) const char * symbol = name.c_str(); unsigned int hashval; - unsigned int g; for (hashval = 0; *symbol != '\0'; symbol++) { hashval = (hashval << 4) + *symbol; - g = hashval & 0xf0000000; + auto g = hashval & 0xf0000000; if (g != 0) { hashval = hashval ^ (g >> 24); hashval = hashval ^ g; @@ -1085,7 +1084,8 @@ bool Ioss::Utils::substr_equal(const std::string &prefix, const std::string &str std::string Ioss::Utils::uppercase(std::string name) { - std::transform(name.begin(), name.end(), name.begin(), [](char c) { return std::toupper(c); }); + std::transform(name.begin(), name.end(), name.begin(), + [](char c) { return static_cast(std::toupper(c)); }); return name; } diff --git a/packages/seacas/libraries/ioss/src/Ioss_Utils.h b/packages/seacas/libraries/ioss/src/Ioss_Utils.h index f72bcc788a61..d36da054b93a 100644 --- a/packages/seacas/libraries/ioss/src/Ioss_Utils.h +++ b/packages/seacas/libraries/ioss/src/Ioss_Utils.h @@ -31,7 +31,6 @@ namespace Ioss { } // namespace Ioss #define IOSS_ERROR(errmsg) throw std::runtime_error((errmsg).str()) -using namespace std::string_literals; namespace { // SEE: http://lemire.me/blog/2017/04/10/removing-duplicates-from-lists-quickly diff --git a/packages/seacas/libraries/ioss/src/catalyst/CMakeLists.txt b/packages/seacas/libraries/ioss/src/catalyst/CMakeLists.txt new file mode 100644 index 000000000000..35efe6a41b54 --- /dev/null +++ b/packages/seacas/libraries/ioss/src/catalyst/CMakeLists.txt @@ -0,0 +1,24 @@ +SET(HEADERS "") +SET(SOURCES "") + +SET_AND_INC_DIRS(DIR ${CMAKE_CURRENT_SOURCE_DIR}) +APPEND_GLOB(HEADERS ${DIR}/*.h*) +APPEND_GLOB(SOURCES ${DIR}/*.C) + +INCLUDE_DIRECTORIES( + "${CMAKE_CURRENT_SOURCE_DIR}/../" + "${CMAKE_CURRENT_BINARY_DIR}/../" +) + +TRIBITS_ADD_LIBRARY( + Iocatalyst + HEADERS ${HEADERS} + SOURCES ${SOURCES} + DEPLIBS Ioss ${DEPLIBS} +) + +#IF (BUILD_TESTING) +# ENABLE_TESTING() +#ENDIF (BUILD_TESTING) +# +#TRIBITS_ADD_TEST_DIRECTORIES(utest) diff --git a/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_DatabaseIO.C b/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_DatabaseIO.C new file mode 100644 index 000000000000..a57b7f23e8ee --- /dev/null +++ b/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_DatabaseIO.C @@ -0,0 +1,783 @@ +// Copyright(C) 1999-2021 National Technology & Engineering Solutions +// of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with +// NTESS, the U.S. Government retains certain rights in this software. +// +// See packages/seacas/LICENSE for details + +#include + +#include "Ioss_CommSet.h" // for CommSet +#include "Ioss_DBUsage.h" // for DatabaseUsage, etc +#include "Ioss_DatabaseIO.h" // for DatabaseIO +#include "Ioss_EdgeBlock.h" // for EdgeBlock +#include "Ioss_EdgeSet.h" // for EdgeSet +#include "Ioss_ElementBlock.h" // for ElementBlock +#include "Ioss_ElementSet.h" // for ElementSet +#include "Ioss_EntityType.h" // for EntityType::ELEMENTBLOCK +#include "Ioss_FaceBlock.h" // for FaceBlock +#include "Ioss_FaceSet.h" // for FaceSet +#include "Ioss_FileInfo.h" // for FileInfo +#include "Ioss_Map.h" // for Map, MapContainer +#include "Ioss_NodeBlock.h" // for NodeBlock +#include "Ioss_NodeSet.h" // for NodeSet +#include "Ioss_Property.h" // for Property +#include "Ioss_SideBlock.h" // for SideBlock +#include // for Assembly +#include // for HAVE_MPI +#include // for NameList +#include // for ParallelUtils, etc +#include // for Region +#include // for SerializeIO +#include // for StructuredBlock + +#include // for Utils, IOSS_ERROR, etc +#include +#include + +#include +#include + +namespace Ioss { + class PropertyManager; +} + +namespace Iocatalyst { + + namespace detail { + + class FieldNonExistent + { + }; + + template + GroupingEntityT *createEntityGroup(const conduit_cpp::Node &node, Ioss::DatabaseIO *dbase); + + template <> + Ioss::NodeBlock *createEntityGroup(const conduit_cpp::Node &node, + Ioss::DatabaseIO * dbase) + { + const auto name = node.name(); + return new Ioss::NodeBlock(dbase, name, node["properties/entity_count/value"].as_int64(), + node["properties/component_degree/value"].as_int64()); + } + + template <> + Ioss::ElementBlock *createEntityGroup(const conduit_cpp::Node &node, + Ioss::DatabaseIO * dbase) + { + const auto name = node.name(); + return new Ioss::ElementBlock(dbase, name, + node["properties/original_topology_type/value"].as_string(), + node["properties/entity_count/value"].as_int64()); + } + + template <> + Ioss::NodeSet *createEntityGroup(const conduit_cpp::Node &node, + Ioss::DatabaseIO * dbase) + { + const auto name = node.name(); + return new Ioss::NodeSet(dbase, name, node["properties/entity_count/value"].as_int64()); + } + + template <> + Ioss::SideSet *createEntityGroup(const conduit_cpp::Node &node, + Ioss::DatabaseIO * dbase) + { + const auto name = node.name(); + return new Ioss::SideSet(dbase, name); + } + + } // namespace detail + + class DatabaseIO::ImplementationT + { + conduit_cpp::Node Root; + conduit_cpp::Node DBNode; + mutable Ioss::Map NodeMap; + + public: + conduit_cpp::Node &databaseNode() { return this->DBNode; } + void setDatabaseNode(conduit_node *c_node) + { + this->DBNode = conduit_cpp::Node(); + conduit_node_set_external_node(conduit_cpp::c_node(&this->DBNode), c_node); + } + conduit_cpp::Node &root() { return this->Root; } + + void print() + { + auto &node = this->DBNode; + node.print_detailed(); + } + + bool defineModel(Ioss::Region *region) + { + assert(region->model_defined()); + + auto &node = this->DBNode; + node = conduit_cpp::Node(); + + node["database/int_byte_size_api"].set_int8(region->get_database()->int_byte_size_api()); + + this->defineEntityGroup(node["node_blocks"], region->get_node_blocks()); + this->defineEntityGroup(node["edge_blocks"], region->get_edge_blocks()); + this->defineEntityGroup(node["face_blocks"], region->get_face_blocks()); + this->defineEntityGroup(node["element_blocks"], region->get_element_blocks()); + // TODO: need to handle side-blocks. + this->defineEntityGroup(node["sidesets"], region->get_sidesets()); + this->defineEntityGroup(node["nodesets"], region->get_nodesets()); + this->defineEntityGroup(node["edgesets"], region->get_edgesets()); + this->defineEntityGroup(node["facesets"], region->get_facesets()); + this->defineEntityGroup(node["elementsets"], region->get_elementsets()); + this->defineEntityGroup(node["structured_blocks"], region->get_structured_blocks()); + this->defineEntityGroup(node["assemblies"], region->get_assemblies()); + return true; + } + + bool readModel(Ioss::Region *region) + { + auto &node = this->DBNode; + region->get_database()->set_int_byte_size_api( + static_cast(node["database/int_byte_size_api"].as_int8())); + this->readEntityGroup(node["node_blocks"], region); + this->readEntityGroup(node["element_blocks"], region); + // this->readEntityGroup(node["edge_blocks"], region); + // this->readEntityGroup(node["face_blocks"], region); + // this->readEntityGroup(node["sidesets"], region); + // this->readEntityGroup(node["nodesets"], region); + // this->readEntityGroup(node["edgesets"], region); + // this->readEntityGroup(node["facesets"], region); + // this->readEntityGroup(node["elementsets"], region); + // this->readEntityGroup(node["structured_blocks"], region); + // this->readEntityGroup(node["assemblies"], region); + + return this->readTime(region); + } + + bool readTime(Ioss::Region *region) + { + auto & node = this->DBNode; + const auto time = node["state_time"].as_float64(); + region->add_state(time); + return true; + } + + int64_t putField(const std::string &containerName, const Ioss::GroupingEntity *entityGroup, + const Ioss::Field &field, void *data, size_t data_size, bool deep_copy) + { + const auto groupName = entityGroup->generic_name(); + const auto num_to_get = field.verify(data_size); + const auto num_components = field.raw_storage()->component_count(); + if (num_to_get > 0) { + auto &&node = this->DBNode[containerName + "/" + groupName + "/fields/" + field.get_name()]; + node["role"].set(static_cast(field.get_role())); + node["type"].set(static_cast(field.get_type())); + node["count"].set(static_cast(field.verify(data_size))); + node["index"].set(static_cast(field.get_index())); + node["component_count"].set(static_cast(num_components)); + node["storage"].set(field.raw_storage()->name()); + switch (field.get_type()) { + case Ioss::Field::BasicType::DOUBLE: + if (deep_copy) { + node["value"].set(static_cast(data), num_to_get * num_components); + } + else { + node["value"].set_external(static_cast(data), num_to_get * num_components); + } + break; + case Ioss::Field::BasicType::INT32: + if (deep_copy) { + node["value"].set(static_cast(data), num_to_get * num_components); + } + else { + node["value"].set_external(static_cast(data), + num_to_get * num_components); + } + break; + case Ioss::Field::BasicType::INT64: + if (deep_copy) { + node["value"].set(static_cast(data), num_to_get * num_components); + } + else { + node["value"].set_external(static_cast(data), + num_to_get * num_components); + } + break; + case Ioss::Field::BasicType::CHARACTER: + if (deep_copy) { + node["value"].set(static_cast(data), num_to_get * num_components); + } + else { + node["value"].set_external(static_cast(data), + num_to_get * num_components); + } + break; + default: + fmt::print(stderr, "ERROR in {} {}: {} ({}), unsupported field type: {}\n", __func__, + containerName, field.get_name(), num_to_get, field.type_string()); + } + } + fmt::print(stderr, "put_field {}: {} ({})\n", containerName, field.get_name(), num_to_get); + return num_to_get; + } + + int64_t getField(const std::string &containerName, const Ioss::GroupingEntity *entityGroup, + const Ioss::Field &field, void *data, size_t data_size) + { + const auto groupName = entityGroup->generic_name(); + auto num_to_get = field.verify(data_size); + const auto num_components = field.raw_storage()->component_count(); + if (num_to_get > 0) { + auto path = containerName + "/" + groupName + "/fields/" + field.get_name() + "/value"; + if (!this->DBNode.has_path(path)) { + throw detail::FieldNonExistent(); + ; + } + const auto &&node = this->DBNode[path]; + switch (field.get_type()) { + case Ioss::Field::BasicType::DOUBLE: + std::copy_n(reinterpret_cast(node.element_ptr(0)), + num_to_get * num_components, reinterpret_cast(data)); + break; + + case Ioss::Field::BasicType::INT32: + std::copy_n(reinterpret_cast(node.element_ptr(0)), + num_to_get * num_components, reinterpret_cast(data)); + break; + + case Ioss::Field::BasicType::INT64: + std::copy_n(reinterpret_cast(node.element_ptr(0)), + num_to_get * num_components, reinterpret_cast(data)); + break; + + case Ioss::Field::BasicType::CHARACTER: + std::copy_n(reinterpret_cast(node.element_ptr(0)), + num_to_get * num_components, reinterpret_cast(data)); + break; + default: + fmt::print(stderr, "ERROR in {} {}: {} ({}), unsupported field type: {}\n", __func__, + containerName, field.get_name(), num_to_get, field.type_string()); + } + } + fmt::print(stderr, "get_field {}: {} ({})\n", containerName, field.get_name(), num_to_get); + return num_to_get; + } + + Ioss::Map &get_node_map(const Ioss::DatabaseIO *dbase) const + { + if (this->NodeMap.defined()) { + return this->NodeMap; + } + + auto &&idsNode = this->DBNode["node_blocks/nodeblock_1/fields/ids"]; + auto node_ids = const_cast(idsNode["value"].element_ptr(0)); + this->NodeMap.set_size(idsNode["count"].as_int64()); + if (idsNode["type"].as_int8() == Ioss::Field::BasicType::INT32) { + this->NodeMap.set_map(reinterpret_cast(node_ids), idsNode["count"].as_int64(), + 0); + } + if (idsNode["type"].as_int8() == Ioss::Field::BasicType::INT64) { + this->NodeMap.set_map(reinterpret_cast(node_ids), idsNode["count"].as_int64(), + 0); + } + + this->NodeMap.set_defined(true); + return this->NodeMap; + } + + private: + template + bool defineEntityGroup(conduit_cpp::Node parent, + const std::vector &container) + { + for (auto group : container) { + this->addProperties(parent[group->generic_name()], group); + } + return true; + } + + template + bool addProperties(conduit_cpp::Node parent, GroupingEntityT *entityGroup) + { + Ioss::NameList names; + // skip implicit properties. + entityGroup->property_describe(Ioss::Property::INTERNAL, &names); + entityGroup->property_describe(Ioss::Property::EXTERNAL, &names); + entityGroup->property_describe(Ioss::Property::ATTRIBUTE, &names); + + auto &&propertiesNode = parent["properties"]; + for (const auto &name : names) { + auto property = entityGroup->get_property(name); + + auto &&node = propertiesNode[name]; + node["type"].set(static_cast(property.get_type())); + node["origin"].set(static_cast(property.get_origin())); + switch (property.get_type()) { + case Ioss::Property::BasicType::REAL: node["value"].set(property.get_real()); break; + + case Ioss::Property::BasicType::INTEGER: node["value"].set(property.get_int()); break; + + case Ioss::Property::BasicType::STRING: node["value"].set(property.get_string()); break; + + case Ioss::Property::BasicType::VEC_INTEGER: + node["value"].set(property.get_vec_int()); + break; + + case Ioss::Property::BasicType::VEC_DOUBLE: + node["value"].set(property.get_vec_double()); + break; + + case Ioss::Property::BasicType::POINTER: + case Ioss::Property::BasicType::INVALID: + default: return false; + } + } + return true; + } + + template + bool readEntityGroup(conduit_cpp::Node &&parent, Ioss::Region *region) + { + for (conduit_index_t idx = 0, max = parent.number_of_children(); idx < max; ++idx) { + auto &&child = parent[idx]; + auto block = detail::createEntityGroup(child, region->get_database()); + region->add(block); + this->readProperties(child["properties"], block); + + // read fields (meta-data only) + this->readFields(child["fields"], block); + } + return true; + } + + template + bool readProperties(const conduit_cpp::Node &&parent, GroupingEntityT *block) const + { + for (conduit_index_t idx = 0, max = parent.number_of_children(); idx < max; ++idx) { + auto && child = parent[idx]; + const auto name = child.name(); + const auto origin = static_cast(child["origin"].as_int8()); + switch (child["type"].as_int8()) { + // TODO: missing origin + case Ioss::Property::BasicType::REAL: + block->property_add(Ioss::Property(name, child["value"].as_float64(), origin)); + break; + + case Ioss::Property::BasicType::INTEGER: + block->property_add(Ioss::Property(name, child["value"].as_int64(), origin)); + break; + + case Ioss::Property::BasicType::STRING: + block->property_add(Ioss::Property(name, child["value"].as_string(), origin)); + break; + + case Ioss::Property::BasicType::VEC_INTEGER: + case Ioss::Property::BasicType::VEC_DOUBLE: abort(); // TODO: + } + } + return true; + } + + template + bool readFields(const conduit_cpp::Node &&parent, GroupingEntityT *block) const + { + for (conduit_index_t idx = 0, max = parent.number_of_children(); idx < max; ++idx) { + auto && child = parent[idx]; + const auto name = child.name(); + const auto type = static_cast(child["type"].as_int8()); + const auto role = static_cast(child["role"].as_int8()); + const auto count = child["count"].as_int64(); + const auto index = child["index"].as_int64(); + const auto storage = child["storage"].as_string(); + if (!block->field_exists(name)) { + block->field_add(Ioss::Field(name, type, storage, role, count, index)); + } + else { + // TODO verify field details. + assert(block->get_field(name).get_type() == type); + } + } + + return true; + } + }; + + DatabaseIO::DatabaseIO(Ioss::Region *region, const std::string &filename, + Ioss::DatabaseUsage db_usage, MPI_Comm communicator, + const Ioss::PropertyManager &props) + : Ioss::DatabaseIO(region, filename, db_usage, communicator, props), + Impl(new DatabaseIO::ImplementationT()), useDeepCopy(true) + + { + dbState = Ioss::STATE_UNKNOWN; + //// Always 64 bits + // dbIntSizeAPI = Ioss::USE_INT64_API; + dbIntSizeAPI = Ioss::USE_INT32_API; + + bool shallowCopy = false; + if (Ioss::Utils::check_set_bool_property(properties, "SHALLOW_COPY_FIELDS", shallowCopy)) { + this->useDeepCopy = !shallowCopy; + } + + if (is_input()) { + auto &pm = get_property_manager(); + if (pm.exists("CATALYST_CONDUIT_NODE")) { + auto c_node_ptr = reinterpret_cast( + get_property_manager().get("CATALYST_CONDUIT_NODE").get_pointer()); + this->Impl->setDatabaseNode(c_node_ptr); + } + else { + // we'll use filename as the location for the data dumps and read those. + std::ostringstream path; + path << filename << "/execute_invc43_params.conduit_bin." << util().parallel_size() << "." + << util().parallel_rank(); + auto &root = this->Impl->root(); + auto &dbase = this->Impl->databaseNode(); + conduit_node_load(conduit_cpp::c_node(&root), path.str().c_str(), "conduit_bin"); + conduit_node_set_external_node( + conduit_cpp::c_node(&dbase), + conduit_node_fetch(conduit_cpp::c_node(&root), "catalyst/channels/dataset/data")); + } + } + else { + // in output-mode, we're pass data on to Catalyst. + conduit_cpp::Node node; + +#if defined(SEACAS_HAVE_MPI) + // pass communicator is using MPI. + // TODO: skip for now. + // node["catalyst"]["mpi_comm"].set(MPI_Comm_c2f(communicator)); +#endif + + // TODO: Here, we need to pass pipeline scripts to execute to the Catalyst + // implementation. ParaView Catalyst supports multiple scripts with args. + // We could support that via the property manager. There are several + // options: + // + // 1. the simulation can add bunch of properties that indicate which scripts + // to run and we use those. + // 2. we could use the filename passed to the database as the + // configuration file that provides the scripts to runs. + // 3. we could simply treat the filename passed to the database as the + // pipeline script. + // + // Here, I am using the simplest option #3. I do that since that makes it + // easy for me to use existing tools like `io_shell` to test this database + node["catalyst"]["scripts"]["sample0"]["filename"].set(filename); + catalyst_initialize(conduit_cpp::c_node(&node)); + } + } + + DatabaseIO::~DatabaseIO() + { + if (!is_input()) { + conduit_cpp::Node node; + catalyst_finalize(conduit_cpp::c_node(&node)); + } + } + + bool DatabaseIO::begin__(Ioss::State state) + { + this->dbState = state; + return true; + } + + bool DatabaseIO::end__(Ioss::State state) + { + assert(this->dbState == state); + + if (!is_input()) { + auto region = this->get_region(); + assert(region != nullptr); + + auto &impl = (*this->Impl.get()); + switch (state) { + case Ioss::STATE_DEFINE_MODEL: + // here the basic structure for the model is defined i.e. + // number of blocks/sets/names etc. + if (!impl.defineModel(region)) { + return false; + } + break; + + case Ioss::STATE_MODEL: + // here the model has meshdata e.g mesh fields, ids, coordinates etc. + // impl.print(); + break; + + case Ioss::STATE_DEFINE_TRANSIENT: break; + + case Ioss::STATE_TRANSIENT: break; + + case Ioss::STATE_LAST_ENTRY: break; + case Ioss::STATE_UNKNOWN: + case Ioss::STATE_INVALID: + case Ioss::STATE_READONLY: + case Ioss::STATE_CLOSED: break; + } + } + + dbState = Ioss::STATE_UNKNOWN; + return true; + } + + bool DatabaseIO::begin_state__(int state, double time) { return true; } + + // common + bool DatabaseIO::end_state__(int state, double time) + { + if (this->is_input()) { + } + else { + // invoke catalyst. + auto &impl = (*this->Impl.get()); + + // state is 1-based, need to offset by 1 to make it 0-based. + // timesteps start with 0. + conduit_cpp::Node node; + node["catalyst/state/timestep"].set(state - 1); + node["catalyst/state/cycle"].set(state - 1); + node["catalyst/state/time"].set(time); + node["catalyst/channels/dataset/type"].set(std::string("ioss")); + node["catalyst/channels/dataset/data"].set_external(impl.databaseNode()); + node["catalyst/channels/dataset/data/state_time"].set(time); + catalyst_execute(conduit_cpp::c_node(&node)); + } + return true; + } + + unsigned DatabaseIO::entity_field_support() const + { + return Ioss::NODEBLOCK | Ioss::EDGEBLOCK | Ioss::FACEBLOCK | Ioss::ELEMENTBLOCK | + Ioss::NODESET | Ioss::EDGESET | Ioss::FACESET | Ioss::ELEMENTSET | Ioss::SIDESET | + Ioss::SIDEBLOCK | Ioss::REGION; + } + + void DatabaseIO::read_meta_data__() + { + auto region = this->get_region(); + assert(region != nullptr); + + auto &impl = (*this->Impl.get()); + impl.readModel(region); + } + + void DatabaseIO::get_step_times__() + { + auto region = this->get_region(); + assert(region != nullptr); + + auto &impl = (*this->Impl.get()); + impl.readTime(region); + } + + int64_t DatabaseIO::put_field_internal(const Ioss::NodeBlock *nb, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.putField("node_blocks", nb, field, data, data_size, this->deep_copy()); + } + + int64_t DatabaseIO::put_field_internal(const Ioss::EdgeBlock *eb, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.putField("edge_blocks", eb, field, data, data_size, this->deep_copy()); + } + + int64_t DatabaseIO::put_field_internal(const Ioss::FaceBlock *fb, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.putField("face_blocks", fb, field, data, data_size, this->deep_copy()); + } + + int64_t DatabaseIO::put_field_internal(const Ioss::ElementBlock *eb, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.putField("element_blocks", eb, field, data, data_size, this->deep_copy()); + } + + int64_t DatabaseIO::put_field_internal(const Ioss::SideBlock *sb, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.putField("sideblocks", sb, field, data, data_size, this->deep_copy()); + } + + int64_t DatabaseIO::put_field_internal(const Ioss::NodeSet *ns, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.putField("nodesets", ns, field, data, data_size, this->deep_copy()); + } + + int64_t DatabaseIO::put_field_internal(const Ioss::EdgeSet *es, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.putField("edgesets", es, field, data, data_size, this->deep_copy()); + } + + int64_t DatabaseIO::put_field_internal(const Ioss::FaceSet *fs, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.putField("facesets", fs, field, data, data_size, this->deep_copy()); + } + + int64_t DatabaseIO::put_field_internal(const Ioss::ElementSet *es, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.putField("elementsets", es, field, data, data_size, this->deep_copy()); + } + + int64_t DatabaseIO::put_field_internal(const Ioss::SideSet *ss, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.putField("sidesets", ss, field, data, data_size, this->deep_copy()); + } + + int64_t DatabaseIO::put_field_internal(const Ioss::CommSet *cs, const Ioss::Field &field, + void *data, size_t data_size) const + { + return -1; + } + + int64_t DatabaseIO::put_field_internal(const Ioss::Assembly * /*as*/, + const Ioss::Field & /*field*/, void * /*data*/, + size_t /*data_size*/) const + { + return -1; + } + + int64_t DatabaseIO::put_field_internal(const Ioss::Blob * /*bl*/, const Ioss::Field & /*field*/, + void * /*data*/, size_t /*data_size*/) const + { + return -1; + } + + int64_t DatabaseIO::put_field_internal(const Ioss::StructuredBlock *sb, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.putField("structured_blocks", sb, field, data, data_size, this->deep_copy()); + } + + int64_t DatabaseIO::get_field_internal(const Ioss::NodeBlock *nb, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.getField("node_blocks", nb, field, data, data_size); + } + + int64_t DatabaseIO::get_field_internal(const Ioss::ElementBlock *eb, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + try { + return impl.getField("element_blocks", eb, field, data, data_size); + } + catch (detail::FieldNonExistent &) { + if (field.get_name() == "connectivity_raw") { + // maybe the data has 'connectivity' provided, so we convert it to 'connectivity_raw'. + auto count = this->get_field_internal(eb, eb->get_field("connectivity"), data, data_size); + if (count <= 0) { + return count; + } + + impl.get_node_map(this).reverse_map_data( + data, field, field.verify(data_size) * field.raw_storage()->component_count()); + return count; + } + else if (field.get_name() == "connectivity") { + // maybe the data has 'connectivity_raw' is provided, so we convert it to 'connectivity. + auto count = + this->get_field_internal(eb, eb->get_field("connectivity_raw"), data, data_size); + if (count <= 0) { + return count; + } + + impl.get_node_map(this).map_data( + data, field, field.verify(data_size) * field.raw_storage()->component_count()); + return count; + } + return -1; + } + } + + int64_t DatabaseIO::get_field_internal(const Ioss::EdgeBlock *eb, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.getField("edge_blocks", eb, field, data, data_size); + } + + int64_t DatabaseIO::get_field_internal(const Ioss::FaceBlock *fb, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.getField("face_blocks", fb, field, data, data_size); + } + int64_t DatabaseIO::get_field_internal(const Ioss::SideBlock *fb, const Ioss::Field &field, + void *data, size_t data_size) const + { + return -1; + } + int64_t DatabaseIO::get_field_internal(const Ioss::NodeSet *ns, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.getField("nodesets", ns, field, data, data_size); + } + int64_t DatabaseIO::get_field_internal(const Ioss::EdgeSet *ns, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.getField("edgesets", ns, field, data, data_size); + } + int64_t DatabaseIO::get_field_internal(const Ioss::FaceSet *ns, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.getField("facesets", ns, field, data, data_size); + } + int64_t DatabaseIO::get_field_internal(const Ioss::ElementSet *ns, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.getField("elementsets", ns, field, data, data_size); + } + int64_t DatabaseIO::get_field_internal(const Ioss::SideSet *fs, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.getField("sidesets", fs, field, data, data_size); + } + int64_t DatabaseIO::get_field_internal(const Ioss::CommSet *cs, const Ioss::Field &field, + void *data, size_t data_size) const + { + return -1; + } + int64_t DatabaseIO::get_field_internal(const Ioss::Assembly * /*as*/, + const Ioss::Field & /*field*/, void * /*data*/, + size_t /*data_size*/) const + { + return -1; + } + int64_t DatabaseIO::get_field_internal(const Ioss::Blob * /*bl*/, const Ioss::Field & /*field*/, + void * /*data*/, size_t /*data_size*/) const + { + return -1; + } + int64_t DatabaseIO::get_field_internal(const Ioss::StructuredBlock *sb, const Ioss::Field &field, + void *data, size_t data_size) const + { + auto &impl = (*this->Impl.get()); + return impl.getField("structured_blocks", sb, field, data, data_size); + } + +} // namespace Iocatalyst diff --git a/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_DatabaseIO.h b/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_DatabaseIO.h new file mode 100644 index 000000000000..b1079a674175 --- /dev/null +++ b/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_DatabaseIO.h @@ -0,0 +1,170 @@ +// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with +// NTESS, the U.S. Government retains certain rights in this software. +// +// See packages/seacas/LICENSE for details + +#ifndef IOSS_Iocatalyst_DatabaseIO_h +#define IOSS_Iocatalyst_DatabaseIO_h + +#include "Ioss_EntitySet.h" +#include "Ioss_Region.h" // for Region, SideSetContainer, etc +#include "Ioss_SideSet.h" // for SideBlockContainer, SideSet +#include +#include + +#include "Ioss_Field.h" // for Field, etc + +#include // for std::unique_ptr + +/** \brief A namespace for the Catalyst 2.0 database format. + */ +namespace Iocatalyst { + class DatabaseIO : public Ioss::DatabaseIO + { + using Superclass = Ioss::DatabaseIO; + + public: + DatabaseIO(Ioss::Region *region, const std::string &filename, Ioss::DatabaseUsage db_usage, + MPI_Comm communicator, const Ioss::PropertyManager &props); + ~DatabaseIO() override; + + // Check capabilities of input/output database... Returns an + // unsigned int with the supported Ioss::EntityTypes or'ed + // together. If "return_value & Ioss::EntityType" is set, then the + // database supports that type (e.g. return_value & Ioss::FACESET) + unsigned entity_field_support() const override; + + /** If there is a single block of nodes in the model, then it is + * considered a node_major() database. If instead the nodes are + * local to each element block or structured block, then it is + * not a node_major database. Exodus is node major, CGNS is not. + */ + bool node_major() const override { return true; } + + // Do anything that might be needed to the database prior to it + // being closed and destructed. + void finalize_database() const override {} + + /** Return a string specifying underlying format of database (exodus, cgns, ...) */ + const std::string get_format() const { return "CATALYST2"; } + + /** \brief Determine whether the database needs information about process ownership of nodes. + * + * \returns True if database needs information about process ownership of nodes. + */ + bool needs_shared_node_information() const override { return false; } + + bool internal_edges_available() const override { return false; } + bool internal_faces_available() const override { return false; } + + /** \brief Get the length of the longest name in the database file. + * + * \returns The length, or 0 for unlimited. + */ + int maximum_symbol_length() const override { return 0; } // Default is unlimited... + void set_maximum_symbol_length(int /* requested_symbol_size */) override { + } // Default does nothing... + + int int_byte_size_db() const override { return 8; } //! Returns 4 or 8 + + /** + * \returns True is fields must be deep-copied. Set 'SHALLOW_COPY_FIELDS' on + * database properties to avoid deep-copying of fields. Default is to + * shallow copy. + */ + bool deep_copy() const { return this->useDeepCopy; } + + private: + bool open_group__(const std::string & /* group_name */) override { return false; } + bool create_subgroup__(const std::string & /* group_name */) override { return false; } + + bool begin__(Ioss::State state) override; + bool end__(Ioss::State state) override; + + void read_meta_data__() override; + void get_step_times__() override; + + bool begin_state__(int state, double time) override; + bool end_state__(int state, double time) override; + + void + compute_block_membership__(Ioss::SideBlock * /* efblock */, + std::vector & /* block_membership */) const override + { + } + + int64_t get_field_internal(const Ioss::Region *reg, const Ioss::Field &field, void *data, + size_t data_size) const override + { + return -1; + } + int64_t get_field_internal(const Ioss::NodeBlock *nb, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t get_field_internal(const Ioss::EdgeBlock *nb, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t get_field_internal(const Ioss::FaceBlock *nb, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t get_field_internal(const Ioss::ElementBlock *eb, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t get_field_internal(const Ioss::SideBlock *fb, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t get_field_internal(const Ioss::NodeSet *ns, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t get_field_internal(const Ioss::EdgeSet *ns, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t get_field_internal(const Ioss::FaceSet *ns, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t get_field_internal(const Ioss::ElementSet *ns, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t get_field_internal(const Ioss::SideSet *fs, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t get_field_internal(const Ioss::CommSet *cs, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t get_field_internal(const Ioss::Assembly * /*as*/, const Ioss::Field & /*field*/, + void * /*data*/, size_t /*data_size*/) const override; + int64_t get_field_internal(const Ioss::Blob * /*bl*/, const Ioss::Field & /*field*/, + void * /*data*/, size_t /*data_size*/) const override; + int64_t get_field_internal(const Ioss::StructuredBlock * /*sb*/, const Ioss::Field & /*field*/, + void * /*data*/, size_t /*data_size*/) const override; + + int64_t put_field_internal(const Ioss::Region *reg, const Ioss::Field &field, void *data, + size_t data_size) const override + { + return -1; + } + int64_t put_field_internal(const Ioss::NodeBlock *nb, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t put_field_internal(const Ioss::EdgeBlock *nb, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t put_field_internal(const Ioss::FaceBlock *nb, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t put_field_internal(const Ioss::ElementBlock *eb, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t put_field_internal(const Ioss::SideBlock *fb, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t put_field_internal(const Ioss::NodeSet *ns, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t put_field_internal(const Ioss::EdgeSet *ns, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t put_field_internal(const Ioss::FaceSet *ns, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t put_field_internal(const Ioss::ElementSet *ns, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t put_field_internal(const Ioss::SideSet *fs, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t put_field_internal(const Ioss::CommSet *cs, const Ioss::Field &field, void *data, + size_t data_size) const override; + int64_t put_field_internal(const Ioss::Assembly * /*as*/, const Ioss::Field & /*field*/, + void * /*data*/, size_t /*data_size*/) const override; + int64_t put_field_internal(const Ioss::Blob * /*bl*/, const Ioss::Field & /*field*/, + void * /*data*/, size_t /*data_size*/) const override; + int64_t put_field_internal(const Ioss::StructuredBlock * /*sb*/, const Ioss::Field & /*field*/, + void * /*data*/, size_t /*data_size*/) const override; + + class ImplementationT; + std::unique_ptr Impl; + bool useDeepCopy; + }; +} // namespace Iocatalyst +#endif diff --git a/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_IOFactory.C b/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_IOFactory.C new file mode 100644 index 000000000000..864a58a683db --- /dev/null +++ b/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_IOFactory.C @@ -0,0 +1,55 @@ +// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with +// NTESS, the U.S. Government retains certain rights in this software. +// +// See packages/seacas/LICENSE for details + +#include "Ioss_DBUsage.h" // for DatabaseUsage +#include "Ioss_IOFactory.h" // for IOFactory +#include // for DatabaseIO +#include +#include // for nullptr +#include // for string + +#include +#include + +namespace Ioss { + class PropertyManager; +} + +namespace Iocatalyst { + + const IOFactory *IOFactory::factory() + { + static IOFactory registerThis; + return ®isterThis; + } + + IOFactory::IOFactory() : Ioss::IOFactory("catalyst") + { + Ioss::IOFactory::alias("catalyst", "catalyst2"); + Ioss::IOFactory::alias("catalyst", "catalyst_conduit"); + } + + Ioss::DatabaseIO *IOFactory::make_IO(const std::string &filename, Ioss::DatabaseUsage db_usage, + MPI_Comm communicator, + const Ioss::PropertyManager &properties) const + { + return new DatabaseIO(nullptr, filename, db_usage, communicator, properties); + } + + std::string IOFactory::show_config() const + { + std::stringstream config; + fmt::print(config, "\tCatalyst Library Version: {}\n", CATALYST_VERSION); + fmt::print(config, "\t\tCatalyst ABI Version: {}\n", CATALYST_ABI_VERSION); + + conduit_cpp::Node node; + catalyst_about(conduit_cpp::c_node(&node)); + auto implementation = node.has_path("catalyst/implementation") ? + node["catalyst/implementation"].as_string() : std::string("stub"); + fmt::print(config, "\t\tImplementation: {}\n\n", implementation.c_str()); + return config.str(); + } +} // namespace Iocatalyst diff --git a/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_IOFactory.h b/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_IOFactory.h new file mode 100644 index 000000000000..eb34c7a33ef9 --- /dev/null +++ b/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_IOFactory.h @@ -0,0 +1,35 @@ +// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with +// NTESS, the U.S. Government retains certain rights in this software. +// +// See packages/seacas/LICENSE for details + +#ifndef IOSS_Iocatalyst_IOFactory_h +#define IOSS_Iocatalyst_IOFactory_h + +#include "Ioss_DatabaseIO.h" // for DatabaseIO +#include +#include // for DatabaseUsage +#include // for IOFactory +#include // for string + +namespace Ioss { + class PropertyManager; +} + +namespace Iocatalyst { + + class IOFactory : public Ioss::IOFactory + { + public: + static const IOFactory *factory(); + + private: + IOFactory(); + Ioss::DatabaseIO *make_IO(const std::string &filename, Ioss::DatabaseUsage db_usage, + MPI_Comm communicator, + const Ioss::PropertyManager &properties) const override; + std::string show_config() const override; + }; +} // namespace Iocatalyst +#endif // IOSS_Iocatalyst_IOFactory_h diff --git a/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_Initializer.C b/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_Initializer.C new file mode 100644 index 000000000000..fbe3e5754420 --- /dev/null +++ b/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_Initializer.C @@ -0,0 +1,35 @@ +// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with +// NTESS, the U.S. Government retains certain rights in this software. +// +// See packages/seacas/LICENSE for details + +#include +#include + +namespace Iocatalyst { + int Initializer::useCount = 0; + + /** \brief Initialize the adios database parts of the Ioss library. + * + * Calls appropriate internal functions and methods to + * initialize the adios database parts of the Ioss library. + */ + Initializer::Initializer() + { + if (useCount == 0) + Iocatalyst::IOFactory::factory(); + useCount++; + } + + Initializer::~Initializer() + { + try { + // Put code here that should run after sierra is finished + // executing... + useCount--; + } + catch (...) { + } + } +} // namespace Iocatalyst diff --git a/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_Initializer.h b/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_Initializer.h new file mode 100644 index 000000000000..1f9afc7c84d9 --- /dev/null +++ b/packages/seacas/libraries/ioss/src/catalyst/Iocatalyst_Initializer.h @@ -0,0 +1,28 @@ +// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with +// NTESS, the U.S. Government retains certain rights in this software. +// +// See packages/seacas/LICENSE for details + +#ifndef IOSS_Iocatalyst_Initializer_h +#define IOSS_Iocatalyst_Initializer_h + +namespace Iocatalyst { + /** \brief Initialization of the Catalyst database parts of the Ioss library. + * + * If any input or output type is catalyst, catalyst2 or catalyst_conduit, + * then an object of this type must be created before using any other functions + * or methods in the Ioss library except Ioss::Init::Initializer(). + */ + class Initializer + { + public: + Initializer(); + ~Initializer(); + // Copy constructor + // Assignment operator + private: + static int useCount; + }; +} // namespace Iocatalyst +#endif diff --git a/packages/seacas/libraries/ioss/src/catalyst/cmake/Dependencies.cmake b/packages/seacas/libraries/ioss/src/catalyst/cmake/Dependencies.cmake new file mode 100644 index 000000000000..83430660a331 --- /dev/null +++ b/packages/seacas/libraries/ioss/src/catalyst/cmake/Dependencies.cmake @@ -0,0 +1,3 @@ +TRIBITS_PACKAGE_DEFINE_DEPENDENCIES( + LIB_REQUIRED_TPLS Catalyst2 +) diff --git a/packages/seacas/libraries/ioss/src/cgns/Iocgns_DatabaseIO.C b/packages/seacas/libraries/ioss/src/cgns/Iocgns_DatabaseIO.C index 87b15f934f66..50d5010e32e6 100644 --- a/packages/seacas/libraries/ioss/src/cgns/Iocgns_DatabaseIO.C +++ b/packages/seacas/libraries/ioss/src/cgns/Iocgns_DatabaseIO.C @@ -4,7 +4,7 @@ // * Single Base. // * ZoneGridConnectivity is 1to1 with point lists for unstructured -// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// Copyright(C) 1999-2021 National Technology & Engineering Solutions // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with // NTESS, the U.S. Government retains certain rights in this software. // @@ -1312,7 +1312,6 @@ namespace Iocgns { if (zone > 1) { // Donor zone is always lower numbered, so zone 1 has no donor zone. int nconn = 0; CGCHECKM(cg_nconns(get_file_pointer(), base, zone, &nconn)); - cgsize_t num_shared = 0; for (int i = 0; i < nconn; i++) { char connectname[CGNS_MAX_NAME_LENGTH + 1]; CG_GridLocation_t location; @@ -1353,7 +1352,6 @@ namespace Iocgns { // A "previous" zone will have a lower zone number this this zone... auto donor_iter = m_zoneNameMap.find(donorname); if (donor_iter != m_zoneNameMap.end() && (*donor_iter).second < zone) { - num_shared += npnts; #if IOSS_DEBUG_OUTPUT fmt::print("Zone {} shares {} nodes with {}\n", zone, npnts, donorname); #endif @@ -2122,24 +2120,24 @@ namespace Iocgns { size_t eb_offset_plus_one = eb->get_offset() + 1; if (field.get_type() == Ioss::Field::INT64) { auto *idata = static_cast(data); - std::iota(idata, idata + my_element_count, eb_offset_plus_one); + std::iota(idata, idata + my_element_count, static_cast(eb_offset_plus_one)); } else { SMART_ASSERT(field.get_type() == Ioss::Field::INT32); int *idata = static_cast(data); - std::iota(idata, idata + my_element_count, eb_offset_plus_one); + std::iota(idata, idata + my_element_count, static_cast(eb_offset_plus_one)); } } else if (field.get_name() == "implicit_ids") { size_t eb_offset_plus_one = eb->get_offset() + 1; if (field.get_type() == Ioss::Field::INT64) { auto *idata = static_cast(data); - std::iota(idata, idata + my_element_count, eb_offset_plus_one); + std::iota(idata, idata + my_element_count, static_cast(eb_offset_plus_one)); } else { SMART_ASSERT(field.get_type() == Ioss::Field::INT32); int *idata = static_cast(data); - std::iota(idata, idata + my_element_count, eb_offset_plus_one); + std::iota(idata, idata + my_element_count, static_cast(eb_offset_plus_one)); } } else { @@ -2274,7 +2272,7 @@ namespace Iocgns { // ======================================================================== // Repetitive code for each coordinate direction; use a lambda to consolidate... - auto coord_lambda = [this, base, zone, &coord, rmin, rmax, phys_dimension, num_to_get, + auto coord_lambda = [this, base, zone, &coord, &rmin, &rmax, phys_dimension, num_to_get, &rdata](const char *ord_name, int ordinate) { CGCHECKM(cg_coord_read(get_file_pointer(), base, zone, ord_name, CG_RealDouble, rmin, rmax, coord.data())); diff --git a/packages/seacas/libraries/ioss/src/cgns/Iocgns_ParallelDatabaseIO.C b/packages/seacas/libraries/ioss/src/cgns/Iocgns_ParallelDatabaseIO.C index 5e121c98f1e7..a9ca5de6ab76 100644 --- a/packages/seacas/libraries/ioss/src/cgns/Iocgns_ParallelDatabaseIO.C +++ b/packages/seacas/libraries/ioss/src/cgns/Iocgns_ParallelDatabaseIO.C @@ -3,7 +3,7 @@ // * Single Base. // * ZoneGridConnectivity is 1to1 with point lists for unstructured -// Copyright(C) 1999-2020 National Technology & Engineering Solutions +// Copyright(C) 1999-2021 National Technology & Engineering Solutions // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with // NTESS, the U.S. Government retains certain rights in this software. // @@ -1268,7 +1268,6 @@ namespace Iocgns { // Get the StructuredBlock that this NodeBlock is contained in: const Ioss::GroupingEntity *sb = nb->contained_in(); - int base = 1; int zone = Iocgns::Utils::get_db_zone(sb); cgsize_t num_to_get = field.verify(data_size); @@ -1278,6 +1277,7 @@ namespace Iocgns { // TODO: do this at read_meta_data() and store... int step = get_region()->get_current_state(); + int base = 1; int solution_index = Utils::find_solution_index(get_file_pointer(), base, zone, step, CG_Vertex); diff --git a/packages/seacas/libraries/ioss/src/cgns/Iocgns_Utils.C b/packages/seacas/libraries/ioss/src/cgns/Iocgns_Utils.C index fbc386134bd6..7d93467e89f3 100644 --- a/packages/seacas/libraries/ioss/src/cgns/Iocgns_Utils.C +++ b/packages/seacas/libraries/ioss/src/cgns/Iocgns_Utils.C @@ -58,6 +58,9 @@ } #ifdef _WIN32 +#ifdef _MSC_VER +#define strncasecmp strnicmp +#endif char *strcasestr(char *haystack, const char *needle) { char *c; @@ -175,7 +178,7 @@ namespace { ssize_t min_proc = -1; for (ssize_t i = 0; i < (ssize_t)work.size(); i++) { if (work[i] < min_work && - proc_adam_map.find(std::make_pair(zone->m_adam->m_zone, i)) == proc_adam_map.end()) { + proc_adam_map.find(std::make_pair(zone->m_adam->m_zone, static_cast(i))) == proc_adam_map.end()) { min_work = work[i]; min_proc = i; if (min_work == 0) { diff --git a/packages/seacas/libraries/ioss/src/exodus/Ioex_BaseDatabaseIO.C b/packages/seacas/libraries/ioss/src/exodus/Ioex_BaseDatabaseIO.C index 5e0478e84e06..33f450ec4dd8 100644 --- a/packages/seacas/libraries/ioss/src/exodus/Ioex_BaseDatabaseIO.C +++ b/packages/seacas/libraries/ioss/src/exodus/Ioex_BaseDatabaseIO.C @@ -513,7 +513,7 @@ namespace Ioex { // See if the client added any "information_records" size_t info_rec_size = informationRecords.size(); size_t in_lines = input_lines.size(); - size_t qa_lines = 2; // Platform info and Version info... + size_t qa_lines = 1; // Platform info size_t config_lines = lines.size(); size_t total_lines = in_lines + qa_lines + info_rec_size + config_lines; @@ -524,8 +524,6 @@ namespace Ioex { int i = 0; Ioss::Utils::copy_string(info[i++], Ioss::Utils::platform_information(), max_line_length + 1); - Ioss::Utils::copy_string(info[i++], Ioex::Version(), max_line_length + 1); - // Copy input file lines into 'info' array... for (size_t j = 0; j < input_lines.size(); j++, i++) { Ioss::Utils::copy_string(info[i], input_lines[j], max_line_length + 1); diff --git a/packages/seacas/libraries/ioss/src/exodus/Ioex_Internals.C b/packages/seacas/libraries/ioss/src/exodus/Ioex_Internals.C index 22526834ae02..27cc62038388 100644 --- a/packages/seacas/libraries/ioss/src/exodus/Ioex_Internals.C +++ b/packages/seacas/libraries/ioss/src/exodus/Ioex_Internals.C @@ -79,8 +79,8 @@ Redefine::Redefine(int exoid) : exodusFilePtr(exoid) int status = nc_redef(exodusFilePtr); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - std::string errmsg; - errmsg = fmt::format("Error: failed to put file id {} into define mode", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to put file id {} into define mode", exodusFilePtr); ex_err_fn(exoid, __func__, errmsg.c_str(), status); exit(EXIT_FAILURE); } @@ -92,9 +92,8 @@ Redefine::~Redefine() int status = nc_enddef(exodusFilePtr); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - std::string errmsg; - errmsg = fmt::format("Error: failed to complete variable definitions in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to complete variable definitions in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); exit(EXIT_FAILURE); } @@ -381,9 +380,7 @@ int Internals::initialize_state_file(Mesh &mesh, const ex_var_params &var_params mesh.get_global_counts(); } - int ierr = 0; - std::string errmsg; - + int ierr = 0; { Redefine the_database(exodusFilePtr); int old_fill = 0; @@ -398,8 +395,8 @@ int Internals::initialize_state_file(Mesh &mesh, const ex_var_params &var_params if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define 'base_database' attribute to file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to define 'base_database' attribute to file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -407,7 +404,8 @@ int Internals::initialize_state_file(Mesh &mesh, const ex_var_params &var_params // Time Dimension... int timedim; if ((status = nc_def_dim(exodusFilePtr, DIM_TIME, NC_UNLIMITED, &timedim)) != NC_NOERR) { - errmsg = fmt::format("Error: failed to define time dimension in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define time dimension in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -417,7 +415,7 @@ int Internals::initialize_state_file(Mesh &mesh, const ex_var_params &var_params status = nc_def_dim(exodusFilePtr, DIM_STR_NAME, maximumNameLength + 1, &namestrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = + std::string errmsg = fmt::format("Error: failed to define 'name string length' in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); @@ -429,7 +427,7 @@ int Internals::initialize_state_file(Mesh &mesh, const ex_var_params &var_params status = nc_def_dim(exodusFilePtr, DIM_NUM_NODES, mesh.nodeblocks[0].entityCount, &numnoddim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = + std::string errmsg = fmt::format("Error: failed to define number of nodes in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); @@ -448,7 +446,7 @@ int Internals::initialize_state_file(Mesh &mesh, const ex_var_params &var_params status = nc_def_dim(exodusFilePtr, DIM_NUM_ELEM, elem_count, &numelemdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = + std::string errmsg = fmt::format("Error: failed to define number of elements in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); @@ -469,7 +467,7 @@ int Internals::initialize_state_file(Mesh &mesh, const ex_var_params &var_params status = nc_def_dim(exodusFilePtr, DIM_NUM_FACE, face_count, &numfacedim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = + std::string errmsg = fmt::format("Error: failed to define number of faces in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); @@ -490,7 +488,7 @@ int Internals::initialize_state_file(Mesh &mesh, const ex_var_params &var_params status = nc_def_dim(exodusFilePtr, DIM_NUM_EDGE, edge_count, &numedgedim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = + std::string errmsg = fmt::format("Error: failed to define number of edges in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); @@ -593,8 +591,8 @@ int Internals::initialize_state_file(Mesh &mesh, const ex_var_params &var_params dim[0] = timedim; if ((status = nc_def_var(exodusFilePtr, VAR_WHOLE_TIME, nc_flt_code(exodusFilePtr), 1, dim, &varid)) != NC_NOERR) { - errmsg = fmt::format("Error: failed to define whole time step variable in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to define whole time step variable in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -604,8 +602,8 @@ int Internals::initialize_state_file(Mesh &mesh, const ex_var_params &var_params file->time_varid = varid; } - ex__compress_variable(exodusFilePtr, varid, 2); - } // Exit redefine mode + ex__compress_variable(exodusFilePtr, varid, -2); /* don't compress, but do set collective io */ + } // Exit redefine mode bool output_global_data = (mesh.comm.outputNemesis && mesh.comm.processorCount > 1); if (var_params.num_edge > 0) { @@ -1062,12 +1060,11 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) int bulk_type = get_type(exodusFilePtr, EX_BULK_INT64_DB); int ids_type = get_type(exodusFilePtr, EX_IDS_INT64_DB); - std::string errmsg; - int rootid = static_cast(exodusFilePtr) & EX_FILE_ID_MASK; if (rootid == exodusFilePtr && nc_inq_dimid(exodusFilePtr, DIM_NUM_DIM, &numdimdim) == NC_NOERR) { - errmsg = fmt::format("Error: initialization already done for file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: initialization already done for file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), EX_MSG); return (EX_FATAL); } @@ -1081,7 +1078,8 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) // define some attributes... if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define title attribute to file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define title attribute to file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1089,14 +1087,12 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) // For use later as a consistency check, define the number of processors and // the current processor id as an attribute of the file... if (comm.outputNemesis && comm.processorCount > 1) { - int ltempsv[2]; - ltempsv[0] = comm.processorCount; - ltempsv[1] = comm.processorId; - status = nc_put_att_int(rootid, NC_GLOBAL, "processor_info", NC_INT, 2, ltempsv); + int ltempsv[] = {comm.processorCount, comm.processorId}; + status = nc_put_att_int(rootid, NC_GLOBAL, "processor_info", NC_INT, 2, ltempsv); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define processor info attribute to file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to define processor info attribute to file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1109,8 +1105,8 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) status = nc_put_att_double(rootid, NC_GLOBAL, "last_written_time", NC_DOUBLE, 1, &fake_time); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define 'last_written_time' attribute to file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to define 'last_written_time' attribute to file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1124,8 +1120,8 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) status = nc_put_att_int(rootid, NC_GLOBAL, ATT_MAX_NAME_LENGTH, NC_INT, 1, ¤t_len); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define ATT_MAX_NAME_LENGTH attribute to file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to define ATT_MAX_NAME_LENGTH attribute to file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1136,7 +1132,8 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) int status = nc_inq_dimid(rootid, DIM_STR, &strdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1149,7 +1146,7 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) status = nc_def_dim(rootid, DIM_STR_NAME, maximumNameLength + 1, &namestrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = + std::string errmsg = fmt::format("Error: failed to define name string length in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); @@ -1159,14 +1156,15 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) status = nc_def_dim(exodusFilePtr, DIM_NUM_DIM, mesh.dimensionality, &numdimdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = + std::string errmsg = fmt::format("Error: failed to define number of dimensions in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } if ((status = nc_def_dim(exodusFilePtr, DIM_TIME, NC_UNLIMITED, &timedim)) != NC_NOERR) { - errmsg = fmt::format("Error: failed to define time dimension in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define time dimension in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1175,8 +1173,8 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) dim[0] = timedim; if ((status = nc_def_var(exodusFilePtr, VAR_WHOLE_TIME, nc_flt_code(exodusFilePtr), 1, dim, &varid)) != NC_NOERR) { - errmsg = fmt::format("Error: failed to define whole time step variable in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to define whole time step variable in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1186,33 +1184,32 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) file->time_varid = varid; } } - - ex__compress_variable(exodusFilePtr, varid, 2); + ex__compress_variable(exodusFilePtr, varid, -2); if (!mesh.nodeblocks.empty() && mesh.nodeblocks[0].entityCount > 0) { status = nc_def_dim(exodusFilePtr, DIM_NUM_NODES, mesh.nodeblocks[0].entityCount, &numnoddim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of nodes in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of nodes in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } // Define the node map here to avoid a later redefine call if (mesh.use_node_map) { - int dims[1]; - dims[0] = numnoddim; - status = nc_def_var(exodusFilePtr, VAR_NODE_NUM_MAP, map_type, 1, dims, &varid); + int dims[] = {numnoddim}; + status = nc_def_var(exodusFilePtr, VAR_NODE_NUM_MAP, map_type, 1, dims, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = + std::string errmsg = fmt::format("Error: node numbering map already exists in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create node numbering map array in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to create node numbering map array in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -1227,24 +1224,25 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) &numattrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of attributes in node block {}" - " in file id {}", - static_cast(mesh.nodeblocks[0].id), exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of attributes in node block {}" + " in file id {}", + static_cast(mesh.nodeblocks[0].id), exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } - int dims[2]; - dims[0] = numnoddim; - dims[1] = numattrdim; - status = nc_def_var(exodusFilePtr, VAR_NATTRIB, nc_flt_code(exodusFilePtr), 2, dims, &varid); + int dims[] = {numnoddim, numattrdim}; + status = nc_def_var(exodusFilePtr, VAR_NATTRIB, nc_flt_code(exodusFilePtr), 2, dims, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attributes for node block {} in file id {}", - mesh.nodeblocks[0].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define attributes for node block {} in file id {}", + mesh.nodeblocks[0].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 2); // Attribute name array... dims[0] = numattrdim; @@ -1253,9 +1251,10 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) status = nc_def_var(exodusFilePtr, VAR_NAME_NATTRIB, NC_CHAR, 2, dims, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attribute name array for node block {}" - " in file id {}", - mesh.nodeblocks[0].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define attribute name array for node block {}" + " in file id {}", + mesh.nodeblocks[0].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1272,7 +1271,7 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) status = nc_def_dim(exodusFilePtr, DIM_NUM_ELEM, elem_count, &numelemdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = + std::string errmsg = fmt::format("Error: failed to define number of elements in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); @@ -1280,20 +1279,19 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) // Define the element map here to avoid a later redefine call if (mesh.use_elem_map) { - int dims[1]; - dims[0] = numelemdim; - varid = 0; - status = nc_def_var(exodusFilePtr, VAR_ELEM_NUM_MAP, map_type, 1, dims, &varid); + int dims[] = {numelemdim}; + varid = 0; + status = nc_def_var(exodusFilePtr, VAR_ELEM_NUM_MAP, map_type, 1, dims, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: element numbering map already exists in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: element numbering map already exists in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create element numbering map in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to create element numbering map in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -1312,31 +1310,32 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) status = nc_def_dim(exodusFilePtr, DIM_NUM_FACE, face_count, &numfacedim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of faces in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of faces in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } // Define the face map here to avoid a later redefine call if (mesh.use_face_map) { - int dims[1]; - dims[0] = numfacedim; - varid = 0; - status = nc_def_var(exodusFilePtr, VAR_FACE_NUM_MAP, map_type, 1, dims, &varid); + int dims[] = {numfacedim}; + varid = 0; + status = nc_def_var(exodusFilePtr, VAR_FACE_NUM_MAP, map_type, 1, dims, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = + std::string errmsg = fmt::format("Error: face numbering map already exists in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create face numbering map in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to create face numbering map in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 1); } } @@ -1350,31 +1349,32 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) status = nc_def_dim(exodusFilePtr, DIM_NUM_EDGE, edge_count, &numedgedim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of edges in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of edges in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } // Define the edge map here to avoid a later redefine call if (mesh.use_edge_map) { - int dims[1]; - dims[0] = numedgedim; - varid = 0; - status = nc_def_var(exodusFilePtr, VAR_EDGE_NUM_MAP, map_type, 1, dims, &varid); + int dims[1] = {numedgedim}; + varid = 0; + status = nc_def_var(exodusFilePtr, VAR_EDGE_NUM_MAP, map_type, 1, dims, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = + std::string errmsg = fmt::format("Error: edge numbering map already exists in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create edge numbering map in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to create edge numbering map in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 1); } } @@ -1442,8 +1442,8 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) status = nc_def_dim(exodusFilePtr, DIM_NUM_PROCS, ltempsv, &procdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to dimension \"{}\" in file ID {}", DIM_NUM_PROCS, - exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to dimension \"{}\" in file ID {}", + DIM_NUM_PROCS, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1458,8 +1458,8 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) status = nc_def_dim(exodusFilePtr, DIM_NUM_PROCS_F, ltempsv, &dimid_npf); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to dimension \"{}\" in file ID {}", DIM_NUM_PROCS_F, - exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to dimension \"{}\" in file ID {}", + DIM_NUM_PROCS_F, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1471,7 +1471,8 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) status = nc_def_var(exodusFilePtr, VAR_FILE_TYPE, NC_INT, 0, nullptr, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define file type in file ID {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define file type in file ID {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1491,8 +1492,8 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) status = nc_def_dim(exodusFilePtr, DIM_NUM_NODES_GLOBAL, ltempsv, &glonoddim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to dimension \"{}\" in file ID {}", - DIM_NUM_NODES_GLOBAL, exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to dimension \"{}\" in file ID {}", + DIM_NUM_NODES_GLOBAL, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1505,8 +1506,8 @@ int Internals::put_metadata(const Mesh &mesh, const CommunicationMetaData &comm) status = nc_def_dim(exodusFilePtr, DIM_NUM_ELEMS_GLOBAL, ltempsv, &gloelemdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to dimension \"{}\" in file ID {}", - DIM_NUM_ELEMS_GLOBAL, exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to dimension \"{}\" in file ID {}", + DIM_NUM_ELEMS_GLOBAL, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1692,12 +1693,9 @@ int Internals::put_metadata(const std::vector &assemblies) if (assemblies.empty()) { return EX_NOERR; } - int dims[1]; - - std::string errmsg; - int status; + int status; if ((status = ex__check_valid_file_id(exodusFilePtr, __func__)) != EX_NOERR) { - errmsg = fmt::format("Error: Invalid exodus file handle: {}", exodusFilePtr); + std::string errmsg = fmt::format("Error: Invalid exodus file handle: {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1715,19 +1713,19 @@ int Internals::put_metadata(const std::vector &assemblies) status = nc_def_dim(exodusFilePtr, numentryptr, assembly.entityCount, &dimid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of entities in assembly in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to define number of entities in assembly in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } /* create variable array in which to store the entry lists */ int entlst_id; - dims[0] = dimid; + int dims[1] = {dimid}; if ((status = nc_def_var(exodusFilePtr, VAR_ENTITY_ASSEMBLY(assembly.id), int_type, 1, dims, &entlst_id)) != NC_NOERR) { - errmsg = fmt::format("Error: failed to define entity assembly variable in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to define entity assembly variable in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1743,8 +1741,8 @@ int Internals::put_metadata(const std::vector &assemblies) } if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define '{}' attribute to file id {}", EX_ATTRIBUTE_ID, - exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to define '{}' attribute to file id {}", + EX_ATTRIBUTE_ID, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1753,8 +1751,8 @@ int Internals::put_metadata(const std::vector &assemblies) status = nc_put_att_int(exodusFilePtr, entlst_id, EX_ATTRIBUTE_TYPE, NC_INT, 1, &type); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define '{}' attribute to file id {}", - EX_ATTRIBUTE_TYPE, exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to define '{}' attribute to file id {}", + EX_ATTRIBUTE_TYPE, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1763,8 +1761,8 @@ int Internals::put_metadata(const std::vector &assemblies) assembly.name.c_str()); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define '{}' attribute to file id {}", - EX_ATTRIBUTE_NAME, exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to define '{}' attribute to file id {}", + EX_ATTRIBUTE_NAME, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1775,8 +1773,8 @@ int Internals::put_metadata(const std::vector &assemblies) strlen(contains) + 1, contains); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define '{}' attribute to file id {}", - EX_ATTRIBUTE_TYPENAME, exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to define '{}' attribute to file id {}", + EX_ATTRIBUTE_TYPENAME, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1797,13 +1795,12 @@ int Internals::put_metadata(const std::vector &blobs) return EX_NOERR; } - std::string errmsg; - int status; + int status; int n1dim; if ((status = nc_def_dim(exodusFilePtr, DIM_N1, 1L, &n1dim)) != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = + std::string errmsg = fmt::format("Error: failed to define number \"1\" dimension in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); @@ -1816,8 +1813,9 @@ int Internals::put_metadata(const std::vector &blobs) int dimid; if ((status = nc_def_dim(exodusFilePtr, numentryptr, blob.entityCount, &dimid)) != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of entries in blob {} in file id {}", - blob.id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of entries in blob {} in file id {}", blob.id, + exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1828,12 +1826,12 @@ int Internals::put_metadata(const std::vector &blobs) int entlst; if ((status = nc_def_var(exodusFilePtr, VAR_ENTITY_BLOB(blob.id), NC_INT, 1, dims, &entlst)) != NC_NOERR) { - errmsg = fmt::format("Error: failed to create entity for blob {} in file id {}", blob.id, - exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to create entity for blob {} in file id {}", + blob.id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } - ex__compress_variable(exodusFilePtr, entlst, 1); + ex__set_compact_storage(exodusFilePtr, entlst); if (ex_int64_status(exodusFilePtr) & EX_IDS_INT64_DB) { long long tmp = blob.id; @@ -1844,7 +1842,7 @@ int Internals::put_metadata(const std::vector &blobs) status = nc_put_att_int(exodusFilePtr, entlst, EX_ATTRIBUTE_ID, NC_INT, 1, &id); } if (status != NC_NOERR) { - errmsg = + std::string errmsg = fmt::format("Error: failed to store blob id {} in file id {}", blob.id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); @@ -1852,8 +1850,8 @@ int Internals::put_metadata(const std::vector &blobs) if ((status = nc_put_att_text(exodusFilePtr, entlst, EX_ATTRIBUTE_NAME, blob.name.length() + 1, blob.name.c_str())) != NC_NOERR) { - errmsg = fmt::format("Error: failed to store blob name {} in file id {}", blob.name, - exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to store blob name {} in file id {}", + blob.name, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1863,9 +1861,6 @@ int Internals::put_metadata(const std::vector &blobs) int Internals::put_metadata(const std::vector &blocks, bool count_only) { - std::string errmsg; - int dims[2]; - int status = 0; // clear error code if (blocks.empty()) { @@ -1880,7 +1875,8 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl status = nc_inq_dimid(exodusFilePtr, DIM_NUM_EL_BLK, &dimid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: no element blocks defined in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: no element blocks defined in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1889,7 +1885,8 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl status = nc_inq_dimid(exodusFilePtr, DIM_STR_NAME, &namestrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1897,7 +1894,7 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl status = nc_inq_dimlen(exodusFilePtr, dimid, &num_elem_blk); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = + std::string errmsg = fmt::format("Error: failed to get number of element blocks in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); @@ -1920,15 +1917,16 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl if (status != NC_NOERR) { if (status == NC_ENAMEINUSE) { // duplicate entry ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: element block {} already defined in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = fmt::format("Error: element block {} already defined in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of elements/block for block {}" - " file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of elements/block for block {}" + " file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -1942,25 +1940,24 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl &nelnoddim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of nodes/element for block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of nodes/element for block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } // element connectivity array { - dims[0] = numelbdim; - dims[1] = nelnoddim; - - int connid; - status = nc_def_var(exodusFilePtr, VAR_CONN(iblk + 1), bulk_type, 2, dims, &connid); + int dims[] = {numelbdim, nelnoddim}; + int connid = 0; + status = nc_def_var(exodusFilePtr, VAR_CONN(iblk + 1), bulk_type, 2, dims, &connid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to create connectivity array for block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to create connectivity array for block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1972,8 +1969,9 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl blocks[iblk].elType); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to store element type name {} in file id {}", - blocks[iblk].elType, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to store element type name {} in file id {}", + blocks[iblk].elType, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -1985,27 +1983,29 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl &neledgdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of edges/element for block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of edges/element for block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } // element->edge connectivity array - dims[0] = numelbdim; - dims[1] = neledgdim; + int dims[] = {numelbdim, neledgdim}; - int connid; - status = nc_def_var(exodusFilePtr, VAR_ECONN(iblk + 1), bulk_type, 2, dims, &connid); + int connid = 0; + status = nc_def_var(exodusFilePtr, VAR_ECONN(iblk + 1), bulk_type, 2, dims, &connid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to create element->edge connectivity array for block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create element->edge connectivity array for block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, connid, 1); } if (blocks[iblk].facesPerEntity > 0) { @@ -2014,27 +2014,29 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl &nelfacdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of faces/element for block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of faces/element for block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } // element->face connectivity array - dims[0] = numelbdim; - dims[1] = nelfacdim; + int dims[] = {numelbdim, nelfacdim}; - int connid; - status = nc_def_var(exodusFilePtr, VAR_FCONN(iblk + 1), bulk_type, 2, dims, &connid); + int connid = 0; + status = nc_def_var(exodusFilePtr, VAR_FCONN(iblk + 1), bulk_type, 2, dims, &connid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to create element->edge connectivity array for block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create element->edge connectivity array for block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, connid, 1); } // element attribute array @@ -2044,55 +2046,59 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl &numattrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of attributes in block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to define number of attributes in block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } - dims[0] = numelbdim; - dims[1] = numattrdim; - int varid = 0; - status = nc_def_var(exodusFilePtr, VAR_ATTRIB(iblk + 1), nc_flt_code(exodusFilePtr), 2, dims, - &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attributes for element block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); - } - ex__compress_variable(exodusFilePtr, varid, 2); + { + int dims[] = {numelbdim, numattrdim}; + int varid = 0; + status = nc_def_var(exodusFilePtr, VAR_ATTRIB(iblk + 1), nc_flt_code(exodusFilePtr), 2, + dims, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = + fmt::format("Error: failed to define attributes for element block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); + } + ex__compress_variable(exodusFilePtr, varid, 2); #if defined(PARALLEL_AWARE_EXODUS) - // There is currently a bug in netcdf-4.5.1-devel and earlier - // for partial parallel output of strided arrays in collective - // mode for netcdf-4-based output. If the number of attributes > - // 1 and in parallel mode, set the mode to independent. - if (blocks[iblk].attributeCount > 1) { - struct ex__file_item *file = ex__find_file_item(exodusFilePtr); - if (file && file->is_parallel && file->is_hdf5) { - nc_var_par_access(exodusFilePtr, varid, NC_INDEPENDENT); - } - } + // There is currently a bug in netcdf-4.5.1-devel and earlier + // for partial parallel output of strided arrays in collective + // mode for netcdf-4-based output. If the number of attributes > + // 1 and in parallel mode, set the mode to independent. + if (blocks[iblk].attributeCount > 1) { + struct ex__file_item *file = ex__find_file_item(exodusFilePtr); + if (file && file->is_parallel && file->is_hdf5) { + nc_var_par_access(exodusFilePtr, varid, NC_INDEPENDENT); + } + } #endif + } - // Attribute name array... - dims[0] = numattrdim; - dims[1] = namestrdim; - - status = nc_def_var(exodusFilePtr, VAR_NAME_ATTRIB(iblk + 1), NC_CHAR, 2, dims, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attribute name array for element block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); + { + // Attribute name array... + int dims[] = {numattrdim, namestrdim}; + int varid = 0; + status = nc_def_var(exodusFilePtr, VAR_NAME_ATTRIB(iblk + 1), NC_CHAR, 2, dims, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = + fmt::format("Error: failed to define attribute name array for element block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); + } + ex__set_compact_storage(exodusFilePtr, varid); } - ex__set_compact_storage(exodusFilePtr, varid); } } return (EX_NOERR); @@ -2100,9 +2106,6 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl int Internals::put_metadata(const std::vector &blocks, bool count_only) { - std::string errmsg; - int dims[2]; - int status = 0; // clear error code int bulk_type = get_type(exodusFilePtr, EX_BULK_INT64_DB); @@ -2117,7 +2120,7 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl status = nc_inq_dimid(exodusFilePtr, DIM_NUM_FA_BLK, &dimid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: no face blocks defined in file id {}", exodusFilePtr); + std::string errmsg = fmt::format("Error: no face blocks defined in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2126,7 +2129,8 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl status = nc_inq_dimid(exodusFilePtr, DIM_STR_NAME, &namestrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2134,7 +2138,8 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl status = nc_inq_dimlen(exodusFilePtr, dimid, &num_face_blk); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to get number of face blocks in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to get number of face blocks in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2156,15 +2161,16 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl if (status != NC_NOERR) { if (status == NC_ENAMEINUSE) { // duplicate entry ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: face block {} already defined in file id {}", blocks[iblk].id, - exodusFilePtr); + std::string errmsg = fmt::format("Error: face block {} already defined in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of faces/block for block {}" - " file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of faces/block for block {}" + " file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -2178,9 +2184,9 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl &nelnoddim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of nodes/face for block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to define number of nodes/face for block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2192,66 +2198,74 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl &numattrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of attributes in block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to define number of attributes in block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } - dims[0] = numelbdim; - dims[1] = numattrdim; - int varid = 0; - status = nc_def_var(exodusFilePtr, VAR_FATTRIB(iblk + 1), nc_flt_code(exodusFilePtr), 2, dims, - &varid); + { + int dims[] = {numelbdim, numattrdim}; + int varid = 0; + status = nc_def_var(exodusFilePtr, VAR_FATTRIB(iblk + 1), nc_flt_code(exodusFilePtr), 2, + dims, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = + fmt::format("Error: failed to define attributes for face block {} in file id {}", + blocks[iblk].id, exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); + } + ex__compress_variable(exodusFilePtr, varid, 2); + } + { + // Attribute name array... + int dims[] = {numattrdim, namestrdim}; + int varid = 0; + + status = nc_def_var(exodusFilePtr, VAR_NAME_FATTRIB(iblk + 1), NC_CHAR, 2, dims, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = + fmt::format("Error: failed to define attribute name array for face block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); + } + ex__set_compact_storage(exodusFilePtr, varid); + } + } + + { + // face connectivity array + int dims[] = {numelbdim, nelnoddim}; + int connid = 0; + status = nc_def_var(exodusFilePtr, VAR_FBCONN(iblk + 1), bulk_type, 2, dims, &connid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attributes for face block {} in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create connectivity array for block {} in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, connid, 1); - // Attribute name array... - dims[0] = numattrdim; - dims[1] = namestrdim; - - status = nc_def_var(exodusFilePtr, VAR_NAME_FATTRIB(iblk + 1), NC_CHAR, 2, dims, &varid); + // store element type as attribute of connectivity variable + status = nc_put_att_text(exodusFilePtr, connid, ATT_NAME_ELB, + static_cast(std::strlen(blocks[iblk].elType)) + 1, + blocks[iblk].elType); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attribute name array for face block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to store element type name {} in file id {}", + blocks[iblk].elType, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } - ex__set_compact_storage(exodusFilePtr, varid); - } - - // face connectivity array - dims[0] = numelbdim; - dims[1] = nelnoddim; - - int connid; - status = nc_def_var(exodusFilePtr, VAR_FBCONN(iblk + 1), bulk_type, 2, dims, &connid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to create connectivity array for block {} in file id {}", - blocks[iblk].id, exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); - } - - // store element type as attribute of connectivity variable - status = nc_put_att_text(exodusFilePtr, connid, ATT_NAME_ELB, - static_cast(std::strlen(blocks[iblk].elType)) + 1, - blocks[iblk].elType); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to store element type name {} in file id {}", - blocks[iblk].elType, exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); } } return (EX_NOERR); @@ -2259,13 +2273,6 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl int Internals::put_metadata(const std::vector &blocks, bool count_only) { - std::string errmsg; - int dims[2]; - - int bulk_type = get_type(exodusFilePtr, EX_BULK_INT64_DB); - - int status = 0; // clear error code - if (blocks.empty()) { return (EX_NOERR); } @@ -2273,10 +2280,10 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl // Get number of edge blocks defined for this file int dimid; size_t num_edge_blk = 0; - status = nc_inq_dimid(exodusFilePtr, DIM_NUM_ED_BLK, &dimid); + int status = nc_inq_dimid(exodusFilePtr, DIM_NUM_ED_BLK, &dimid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: no edge blocks defined in file id {}", exodusFilePtr); + std::string errmsg = fmt::format("Error: no edge blocks defined in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2285,7 +2292,8 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl status = nc_inq_dimid(exodusFilePtr, DIM_STR_NAME, &namestrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2293,7 +2301,8 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl status = nc_inq_dimlen(exodusFilePtr, dimid, &num_edge_blk); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to get number of edge blocks in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to get number of edge blocks in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2315,15 +2324,16 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl if (status != NC_NOERR) { if (status == NC_ENAMEINUSE) { // duplicate entry ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: edge block {} already defined in file id {}", blocks[iblk].id, - exodusFilePtr); + std::string errmsg = fmt::format("Error: edge block {} already defined in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of edges/block for block {}" - " file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of edges/block for block {}" + " file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -2337,9 +2347,10 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl &nelnoddim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of nodes/edge ({}) for block {}" - " in file id {}", - blocks[iblk].nodesPerEntity, blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of nodes/edge ({}) for block {}" + " in file id {}", + blocks[iblk].nodesPerEntity, blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2351,25 +2362,26 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl &numattrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of attributes in block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to define number of attributes in block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } - dims[0] = numelbdim; - dims[1] = numattrdim; - int varid = 0; + int dims[] = {numelbdim, numattrdim}; + int varid = 0; status = nc_def_var(exodusFilePtr, VAR_EATTRIB(iblk + 1), nc_flt_code(exodusFilePtr), 2, dims, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attributes for edge block {} in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define attributes for edge block {} in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 2); // Attribute name array... dims[0] = numattrdim; @@ -2378,9 +2390,10 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl status = nc_def_var(exodusFilePtr, VAR_NAME_EATTRIB(iblk + 1), NC_CHAR, 2, dims, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attribute name array for edge block {}" - " in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define attribute name array for edge block {}" + " in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2388,26 +2401,29 @@ int Internals::put_metadata(const std::vector &blocks, bool count_onl } // edge connectivity array - dims[0] = numelbdim; - dims[1] = nelnoddim; + int dims[] = {numelbdim, nelnoddim}; - int connid; - status = nc_def_var(exodusFilePtr, VAR_EBCONN(iblk + 1), bulk_type, 2, dims, &connid); + int connid = 0; + int bulk_type = get_type(exodusFilePtr, EX_BULK_INT64_DB); + status = nc_def_var(exodusFilePtr, VAR_EBCONN(iblk + 1), bulk_type, 2, dims, &connid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to create connectivity array for block {} in file id {}", - blocks[iblk].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create connectivity array for block {} in file id {}", + blocks[iblk].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, connid, 1); + // store element type as attribute of connectivity variable status = nc_put_att_text(exodusFilePtr, connid, ATT_NAME_ELB, static_cast(std::strlen(blocks[iblk].elType)) + 1, blocks[iblk].elType); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to store element type name {} in file id {}", - blocks[iblk].elType, exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to store element type name {} in file id {}", + blocks[iblk].elType, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2419,15 +2435,15 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ { // Metadata that must be written outside of define mode... if (comm.outputNemesis) { - std::string errmsg; - int status = 0; + int status = 0; // Output the file type int varid; status = nc_inq_varid(exodusFilePtr, VAR_FILE_TYPE, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to locate file type in file ID {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to locate file type in file ID {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2436,7 +2452,7 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ status = nc_put_var1_int(exodusFilePtr, varid, nullptr, &lftype); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = + std::string errmsg = fmt::format("Error: unable to output file type variable in file ID {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); @@ -2450,8 +2466,8 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ status = nc_put_var_int(exodusFilePtr, nodeMapVarID[0], &nmstat); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to output status for internal node map in file ID {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to output status for internal node map in file ID {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2460,8 +2476,8 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ status = nc_put_var_int(exodusFilePtr, nodeMapVarID[1], &nmstat); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to output status for border node map in file ID {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to output status for border node map in file ID {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2470,8 +2486,8 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ status = nc_put_var_int(exodusFilePtr, nodeMapVarID[2], &nmstat); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to output status for external node map in file ID {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to output status for external node map in file ID {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2480,8 +2496,8 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ status = nc_put_var_int(exodusFilePtr, elementMapVarID[0], &nmstat); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to output status for internal elem map in file ID {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to output status for internal elem map in file ID {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2490,8 +2506,8 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ status = nc_put_var_int(exodusFilePtr, elementMapVarID[1], &nmstat); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to output status for border elem map in file ID {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to output status for border elem map in file ID {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2507,8 +2523,9 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ status = nc_inq_varid(exodusFilePtr, VAR_N_COMM_STAT, &n_varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to find variable ID for \"{}\" in file ID {}", - VAR_N_COMM_STAT, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to find variable ID for \"{}\" in file ID {}", + VAR_N_COMM_STAT, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2521,7 +2538,8 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ status = nc_put_var1_int(exodusFilePtr, n_varid, start, &nmstat); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: unable to output variable in file ID {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: unable to output variable in file ID {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2533,8 +2551,8 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ status = nc_inq_varid(exodusFilePtr, VAR_N_COMM_DATA_IDX, &commIndexVar); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to locate node communication map in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to locate node communication map in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2542,8 +2560,8 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to output node communication map index in file ID {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to output node communication map index in file ID {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2571,8 +2589,9 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ status = nc_inq_varid(exodusFilePtr, VAR_E_COMM_STAT, &e_varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to find variable ID for \"{}\" in file ID {}", - VAR_E_COMM_STAT, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to find variable ID for \"{}\" in file ID {}", + VAR_E_COMM_STAT, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2586,7 +2605,8 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ status = nc_put_var1_int(exodusFilePtr, e_varid, start, &nmstat); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: unable to output variable in file ID {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: unable to output variable in file ID {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2598,16 +2618,16 @@ int Internals::put_non_define_data(const CommunicationMetaData &comm, bool full_ status = nc_inq_varid(exodusFilePtr, VAR_E_COMM_DATA_IDX, &elemCommIndexVar); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to locate element communication map in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to locate element communication map in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } status = nc_put_var1_longlong(exodusFilePtr, elemCommIndexVar, start, &nl_ecnt_cmap); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to output int elem map index in file ID {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to output int elem map index in file ID {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2738,9 +2758,8 @@ int Internals::put_non_define_data(const std::vector &blocks, bool ou int status = nc_inq_varid(exodusFilePtr, VAR_NAME_ATTRIB(iblk + 1), &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - std::string errmsg; - errmsg = fmt::format("Error: failed to locate variable name attribute in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to locate variable name attribute in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2792,10 +2811,9 @@ int Internals::put_non_define_data(const std::vector &blocks) int varid; int status = nc_inq_varid(exodusFilePtr, VAR_NAME_FATTRIB(iblk + 1), &varid); if (status != NC_NOERR) { - std::string errmsg; ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to locate face variable name attribute in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to locate face variable name attribute in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2848,8 +2866,7 @@ int Internals::put_non_define_data(const std::vector &blocks) int status = nc_inq_varid(exodusFilePtr, VAR_NAME_EATTRIB(iblk + 1), &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - std::string errmsg; - errmsg = + std::string errmsg = fmt::format("Error: failed to locate element variable name attribute in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); @@ -2873,25 +2890,20 @@ int Internals::put_metadata(const std::vector &nodesets, bool count_onl return (EX_NOERR); } - std::string errmsg; - int dims[2]; - - int status = 0; // clear error code - int bulk_type = get_type(exodusFilePtr, EX_BULK_INT64_DB); // Get number of node sets defined for this file int dimid; int num_node_sets = 0; - status = nc_inq_dimid(exodusFilePtr, DIM_NUM_NS, &dimid); + int status = nc_inq_dimid(exodusFilePtr, DIM_NUM_NS, &dimid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_EBADDIM) { - errmsg = fmt::format("Error: no node sets defined for file id {}", exodusFilePtr); + std::string errmsg = fmt::format("Error: no node sets defined for file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = + std::string errmsg = fmt::format("Error: failed to locate node sets defined in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } @@ -2905,7 +2917,8 @@ int Internals::put_metadata(const std::vector &nodesets, bool count_onl status = nc_inq_dimid(exodusFilePtr, DIM_STR_NAME, &namestrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2927,13 +2940,14 @@ int Internals::put_metadata(const std::vector &nodesets, bool count_onl if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: node set {} already defined in file id {}", nodesets[i].id, - exodusFilePtr); + std::string errmsg = fmt::format("Error: node set {} already defined in file id {}", + nodesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to define number of nodes for set {} in file id {}", - nodesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of nodes for set {} in file id {}", + nodesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -2943,20 +2957,22 @@ int Internals::put_metadata(const std::vector &nodesets, bool count_onl } // define variable to store node set node list here instead of in expns - dims[0] = dimid; + int dims[1] = {dimid}; int varid; status = nc_def_var(exodusFilePtr, VAR_NODE_NS(cur_num_node_sets + 1), bulk_type, 1, dims, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: node set {} node list already defined in file id {}", - nodesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: node set {} node list already defined in file id {}", + nodesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create node set {} node list in file id {}", - nodesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create node set {} node list in file id {}", + nodesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -2969,10 +2985,10 @@ int Internals::put_metadata(const std::vector &nodesets, bool count_onl if (nodesets[i].dfCount != nodesets[i].entityCount) { status = EX_FATAL; ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: # dist fact ({}) not equal to # nodes ({}) " - "in node set {} file id {}", - nodesets[i].dfCount, nodesets[i].entityCount, nodesets[i].id, - exodusFilePtr); + std::string errmsg = fmt::format("Error: # dist fact ({}) not equal to # nodes ({}) " + "in node set {} file id {}", + nodesets[i].dfCount, nodesets[i].entityCount, + nodesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -2982,61 +2998,69 @@ int Internals::put_metadata(const std::vector &nodesets, bool count_onl if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: node set {} dist factors already exist in file id {}", - nodesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: node set {} dist factors already exist in file id {}", + nodesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create node set {} dist factors in file id {}", - nodesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create node set {} dist factors in file id {}", + nodesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); } - ex__compress_variable(exodusFilePtr, varid, 2); } + if (nodesets[i].attributeCount > 0) { int numattrdim; status = nc_def_dim(exodusFilePtr, DIM_NUM_ATT_IN_NS(cur_num_node_sets + 1), nodesets[i].attributeCount, &numattrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of attributes in nodeset {}" - " in file id {}", - nodesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of attributes in nodeset {}" + " in file id {}", + nodesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } - dims[0] = dimid; - dims[1] = numattrdim; - status = nc_def_var(exodusFilePtr, VAR_NSATTRIB(cur_num_node_sets + 1), - nc_flt_code(exodusFilePtr), 2, dims, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attributes for element nodeset {}" - " in file id {}", - nodesets[i].id, exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); + { + int dims[] = {dimid, numattrdim}; + status = nc_def_var(exodusFilePtr, VAR_NSATTRIB(cur_num_node_sets + 1), + nc_flt_code(exodusFilePtr), 2, dims, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = + fmt::format("Error: failed to define attributes for element nodeset {}" + " in file id {}", + nodesets[i].id, exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); + } + ex__compress_variable(exodusFilePtr, varid, 2); } - // Attribute name array... - dims[0] = numattrdim; - dims[1] = namestrdim; + { + // Attribute name array... + int dims[] = {numattrdim, namestrdim}; - status = nc_def_var(exodusFilePtr, VAR_NAME_NSATTRIB(cur_num_node_sets + 1), NC_CHAR, 2, dims, - &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attribute name array for nodeset {}" - " in file id {}", - nodesets[i].id, exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); + status = nc_def_var(exodusFilePtr, VAR_NAME_NSATTRIB(cur_num_node_sets + 1), NC_CHAR, 2, + dims, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = + fmt::format("Error: failed to define attribute name array for nodeset {}" + " in file id {}", + nodesets[i].id, exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); + } + ex__set_compact_storage(exodusFilePtr, varid); } - ex__set_compact_storage(exodusFilePtr, varid); } } return (EX_NOERR); @@ -3048,26 +3072,20 @@ int Internals::put_metadata(const std::vector &edgesets, bool count_onl if (edgesets.empty()) { return (EX_NOERR); } - - std::string errmsg; - int dims[2]; - - int status = 0; // clear error code - int bulk_type = get_type(exodusFilePtr, EX_BULK_INT64_DB); // Get number of edge sets defined for this file int dimid; int num_edge_sets = 0; - status = nc_inq_dimid(exodusFilePtr, DIM_NUM_ES, &dimid); + int status = nc_inq_dimid(exodusFilePtr, DIM_NUM_ES, &dimid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_EBADDIM) { - errmsg = fmt::format("Error: no edge sets defined for file id {}", exodusFilePtr); + std::string errmsg = fmt::format("Error: no edge sets defined for file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = + std::string errmsg = fmt::format("Error: failed to locate edge sets defined in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } @@ -3083,7 +3101,8 @@ int Internals::put_metadata(const std::vector &edgesets, bool count_onl status = nc_inq_dimid(exodusFilePtr, DIM_STR_NAME, &namestrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -3103,13 +3122,14 @@ int Internals::put_metadata(const std::vector &edgesets, bool count_onl if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: edge set {} already defined in file id {}", edgesets[i].id, - exodusFilePtr); + std::string errmsg = fmt::format("Error: edge set {} already defined in file id {}", + edgesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to define number of edges for set {} in file id {}", - edgesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of edges for set {} in file id {}", + edgesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -3119,42 +3139,48 @@ int Internals::put_metadata(const std::vector &edgesets, bool count_onl } // define variable to store edge set edge list here instead of in expns - dims[0] = dimid; + int dims1[1] = {dimid}; int varid; status = - nc_def_var(exodusFilePtr, VAR_EDGE_ES(cur_num_edge_sets + 1), bulk_type, 1, dims, &varid); + nc_def_var(exodusFilePtr, VAR_EDGE_ES(cur_num_edge_sets + 1), bulk_type, 1, dims1, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: edge set {} edge list already defined in file id {}", - edgesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: edge set {} edge list already defined in file id {}", + edgesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create edge set {} edge list in file id {}", - edgesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create edge set {} edge list in file id {}", + edgesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 1); // Orientation variable status = - nc_def_var(exodusFilePtr, VAR_ORNT_ES(cur_num_edge_sets + 1), bulk_type, 1, dims, &varid); + nc_def_var(exodusFilePtr, VAR_ORNT_ES(cur_num_edge_sets + 1), bulk_type, 1, dims1, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: extra list already exists for edge set {} in file id {}", - edgesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: extra list already exists for edge set {} in file id {}", + edgesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create extra list for edge set {} in file id {}", - edgesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create extra list for edge set {} in file id {}", + edgesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 1); // Create variable for distribution factors if required if (edgesets[i].dfCount > 0) { @@ -3162,30 +3188,33 @@ int Internals::put_metadata(const std::vector &edgesets, bool count_onl if (edgesets[i].dfCount != edgesets[i].entityCount) { status = EX_FATAL; ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: # dist fact ({}) not equal to # edges ({}) " - "in edge set {} file id {}", - edgesets[i].dfCount, edgesets[i].entityCount, edgesets[i].id, - exodusFilePtr); + std::string errmsg = fmt::format("Error: # dist fact ({}) not equal to # edges ({}) " + "in edge set {} file id {}", + edgesets[i].dfCount, edgesets[i].entityCount, + edgesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } // create variable for distribution factors status = nc_def_var(exodusFilePtr, VAR_FACT_ES(cur_num_edge_sets + 1), - nc_flt_code(exodusFilePtr), 1, dims, &varid); + nc_flt_code(exodusFilePtr), 1, dims1, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: edge set {} dist factors already exist in file id {}", - edgesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: edge set {} dist factors already exist in file id {}", + edgesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create edge set {} dist factors in file id {}", - edgesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create edge set {} dist factors in file id {}", + edgesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 2); } if (edgesets[i].attributeCount > 0) { int numattrdim; @@ -3193,42 +3222,47 @@ int Internals::put_metadata(const std::vector &edgesets, bool count_onl edgesets[i].attributeCount, &numattrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of attributes in edgeset {}" - " in file id {}", - edgesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of attributes in edgeset {}" + " in file id {}", + edgesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } - dims[0] = dimid; - dims[1] = numattrdim; - - status = nc_def_var(exodusFilePtr, VAR_ESATTRIB(cur_num_edge_sets + 1), - nc_flt_code(exodusFilePtr), 2, dims, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attributes for element edgeset {}" - " in file id {}", - edgesets[i].id, exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); + { + int dims[] = {dimid, numattrdim}; + status = nc_def_var(exodusFilePtr, VAR_ESATTRIB(cur_num_edge_sets + 1), + nc_flt_code(exodusFilePtr), 2, dims, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = + fmt::format("Error: failed to define attributes for element edgeset {}" + " in file id {}", + edgesets[i].id, exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); + } + ex__compress_variable(exodusFilePtr, varid, 2); } - // Attribute name array... - dims[0] = numattrdim; - dims[1] = namestrdim; + { + // Attribute name array... + int dims[] = {numattrdim, namestrdim}; - status = nc_def_var(exodusFilePtr, VAR_NAME_ESATTRIB(cur_num_edge_sets + 1), NC_CHAR, 2, dims, - &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attribute name array for edgeset {}" - " in file id {}", - edgesets[i].id, exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); + status = nc_def_var(exodusFilePtr, VAR_NAME_ESATTRIB(cur_num_edge_sets + 1), NC_CHAR, 2, + dims, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = + fmt::format("Error: failed to define attribute name array for edgeset {}" + " in file id {}", + edgesets[i].id, exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); + } + ex__set_compact_storage(exodusFilePtr, varid); } - ex__set_compact_storage(exodusFilePtr, varid); } } return (EX_NOERR); @@ -3240,26 +3274,20 @@ int Internals::put_metadata(const std::vector &facesets, bool count_onl if (facesets.empty()) { return (EX_NOERR); } - - std::string errmsg; - int dims[2]; - - int status = 0; // clear error code - int bulk_type = get_type(exodusFilePtr, EX_BULK_INT64_DB); // Get number of face sets defined for this file int dimid; int num_face_sets = 0; - status = nc_inq_dimid(exodusFilePtr, DIM_NUM_FS, &dimid); + int status = nc_inq_dimid(exodusFilePtr, DIM_NUM_FS, &dimid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_EBADDIM) { - errmsg = fmt::format("Error: no face sets defined for file id {}", exodusFilePtr); + std::string errmsg = fmt::format("Error: no face sets defined for file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = + std::string errmsg = fmt::format("Error: failed to locate face sets defined in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } @@ -3275,7 +3303,8 @@ int Internals::put_metadata(const std::vector &facesets, bool count_onl status = nc_inq_dimid(exodusFilePtr, DIM_STR_NAME, &namestrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -3295,13 +3324,14 @@ int Internals::put_metadata(const std::vector &facesets, bool count_onl if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: face set {} already defined in file id {}", facesets[i].id, - exodusFilePtr); + std::string errmsg = fmt::format("Error: face set {} already defined in file id {}", + facesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to define number of faces for set {} in file id {}", - facesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of faces for set {} in file id {}", + facesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -3311,42 +3341,48 @@ int Internals::put_metadata(const std::vector &facesets, bool count_onl } // define variable to store face set face list here instead of in expns - dims[0] = dimid; + int dims1[1] = {dimid}; int varid; status = - nc_def_var(exodusFilePtr, VAR_FACE_FS(cur_num_face_sets + 1), bulk_type, 1, dims, &varid); + nc_def_var(exodusFilePtr, VAR_FACE_FS(cur_num_face_sets + 1), bulk_type, 1, dims1, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: face set {} face list already defined in file id {}", - facesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: face set {} face list already defined in file id {}", + facesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create face set {} face list in file id {}", - facesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create face set {} face list in file id {}", + facesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 1); // Orientation variable status = - nc_def_var(exodusFilePtr, VAR_ORNT_FS(cur_num_face_sets + 1), bulk_type, 1, dims, &varid); + nc_def_var(exodusFilePtr, VAR_ORNT_FS(cur_num_face_sets + 1), bulk_type, 1, dims1, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: extra list already exists for face set {} in file id {}", - facesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: extra list already exists for face set {} in file id {}", + facesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create extra list for face set {} in file id {}", - facesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create extra list for face set {} in file id {}", + facesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 1); // Create variable for distribution factors if required if (facesets[i].dfCount > 0) { @@ -3354,30 +3390,33 @@ int Internals::put_metadata(const std::vector &facesets, bool count_onl if (facesets[i].dfCount != facesets[i].entityCount) { status = EX_FATAL; ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: # dist fact ({}) not equal to # faces ({}) " - "in face set {} file id {}", - facesets[i].dfCount, facesets[i].entityCount, facesets[i].id, - exodusFilePtr); + std::string errmsg = fmt::format("Error: # dist fact ({}) not equal to # faces ({}) " + "in face set {} file id {}", + facesets[i].dfCount, facesets[i].entityCount, + facesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } // create variable for distribution factors status = nc_def_var(exodusFilePtr, VAR_FACT_FS(cur_num_face_sets + 1), - nc_flt_code(exodusFilePtr), 1, dims, &varid); + nc_flt_code(exodusFilePtr), 1, dims1, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: face set {} dist factors already exist in file id {}", - facesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: face set {} dist factors already exist in file id {}", + facesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create face set {} dist factors in file id {}", - facesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create face set {} dist factors in file id {}", + facesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 2); } if (facesets[i].attributeCount > 0) { int numattrdim; @@ -3385,41 +3424,47 @@ int Internals::put_metadata(const std::vector &facesets, bool count_onl facesets[i].attributeCount, &numattrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of attributes in faceset {}" - " in file id {}", - facesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of attributes in faceset {}" + " in file id {}", + facesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } - dims[0] = dimid; - dims[1] = numattrdim; - status = nc_def_var(exodusFilePtr, VAR_FSATTRIB(cur_num_face_sets + 1), - nc_flt_code(exodusFilePtr), 2, dims, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attributes for element faceset {}" - " in file id {}", - facesets[i].id, exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); + { + int dims[] = {dimid, numattrdim}; + status = nc_def_var(exodusFilePtr, VAR_FSATTRIB(cur_num_face_sets + 1), + nc_flt_code(exodusFilePtr), 2, dims, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = + fmt::format("Error: failed to define attributes for element faceset {}" + " in file id {}", + facesets[i].id, exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); + } + ex__compress_variable(exodusFilePtr, varid, 2); } - // Attribute name array... - dims[0] = numattrdim; - dims[1] = namestrdim; + { + // Attribute name array... + int dims[] = {numattrdim, namestrdim}; - status = nc_def_var(exodusFilePtr, VAR_NAME_FSATTRIB(cur_num_face_sets + 1), NC_CHAR, 2, dims, - &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attribute name array for faceset {}" - " in file id {}", - facesets[i].id, exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); + status = nc_def_var(exodusFilePtr, VAR_NAME_FSATTRIB(cur_num_face_sets + 1), NC_CHAR, 2, + dims, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = + fmt::format("Error: failed to define attribute name array for faceset {}" + " in file id {}", + facesets[i].id, exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); + } + ex__set_compact_storage(exodusFilePtr, varid); } - ex__set_compact_storage(exodusFilePtr, varid); } } return (EX_NOERR); @@ -3431,24 +3476,19 @@ int Internals::put_metadata(const std::vector &elemsets, bool count_onl if (elemsets.empty()) { return (EX_NOERR); } - - std::string errmsg; - int dims[2]; - - int status = 0; // clear error code - // Get number of element sets defined for this file int dimid; int num_elem_sets = 0; - status = nc_inq_dimid(exodusFilePtr, DIM_NUM_ELS, &dimid); + int status = nc_inq_dimid(exodusFilePtr, DIM_NUM_ELS, &dimid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_EBADDIM) { - errmsg = fmt::format("Error: no element sets defined for file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: no element sets defined for file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = + std::string errmsg = fmt::format("Error: failed to locate element sets defined in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } @@ -3464,7 +3504,8 @@ int Internals::put_metadata(const std::vector &elemsets, bool count_onl status = nc_inq_dimid(exodusFilePtr, DIM_STR_NAME, &namestrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to get string length in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -3484,13 +3525,14 @@ int Internals::put_metadata(const std::vector &elemsets, bool count_onl if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: elem set {} already defined in file id {}", elemsets[i].id, - exodusFilePtr); + std::string errmsg = fmt::format("Error: elem set {} already defined in file id {}", + elemsets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to define number of elems for set {} in file id {}", - elemsets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of elems for set {} in file id {}", + elemsets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -3500,25 +3542,28 @@ int Internals::put_metadata(const std::vector &elemsets, bool count_onl } // define variable to store element set element list here instead of in expns - dims[0] = dimid; + int dims1[1] = {dimid}; int varid; status = nc_def_var(exodusFilePtr, VAR_ELEM_ELS(cur_num_elem_sets + 1), - get_type(exodusFilePtr, EX_BULK_INT64_DB), 1, dims, &varid); + get_type(exodusFilePtr, EX_BULK_INT64_DB), 1, dims1, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: element set {} element list already defined in file id {}", - elemsets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: element set {} element list already defined in file id {}", + elemsets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create element set {} element list in file id {}", - elemsets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create element set {} element list in file id {}", + elemsets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 1); // Create variable for distribution factors if required if (elemsets[i].dfCount > 0) { @@ -3526,30 +3571,33 @@ int Internals::put_metadata(const std::vector &elemsets, bool count_onl if (elemsets[i].dfCount != elemsets[i].entityCount) { status = EX_FATAL; ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: # dist fact ({}) not equal to # elements ({}) " - "in element set {} file id {}", - elemsets[i].dfCount, elemsets[i].entityCount, elemsets[i].id, - exodusFilePtr); + std::string errmsg = fmt::format("Error: # dist fact ({}) not equal to # elements ({}) " + "in element set {} file id {}", + elemsets[i].dfCount, elemsets[i].entityCount, + elemsets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } // create variable for distribution factors status = nc_def_var(exodusFilePtr, VAR_FACT_ELS(cur_num_elem_sets + 1), - nc_flt_code(exodusFilePtr), 1, dims, &varid); + nc_flt_code(exodusFilePtr), 1, dims1, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: element set {} dist factors already exist in file id {}", - elemsets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: element set {} dist factors already exist in file id {}", + elemsets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create element set {} dist factors in file id {}", - elemsets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create element set {} dist factors in file id {}", + elemsets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 2); } if (elemsets[i].attributeCount > 0) { int numattrdim; @@ -3557,41 +3605,47 @@ int Internals::put_metadata(const std::vector &elemsets, bool count_onl elemsets[i].attributeCount, &numattrdim); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of attributes in elemset {}" - " in file id {}", - elemsets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of attributes in elemset {}" + " in file id {}", + elemsets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } - dims[0] = dimid; - dims[1] = numattrdim; - status = nc_def_var(exodusFilePtr, VAR_ESATTRIB(cur_num_elem_sets + 1), - nc_flt_code(exodusFilePtr), 2, dims, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attributes for element elemset {}" - " in file id {}", - elemsets[i].id, exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); + { + int dims[] = {dimid, numattrdim}; + status = nc_def_var(exodusFilePtr, VAR_ESATTRIB(cur_num_elem_sets + 1), + nc_flt_code(exodusFilePtr), 2, dims, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = + fmt::format("Error: failed to define attributes for element elemset {}" + " in file id {}", + elemsets[i].id, exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); + } + ex__compress_variable(exodusFilePtr, varid, 2); } - // Attribute name array... - dims[0] = numattrdim; - dims[1] = namestrdim; + { + // Attribute name array... + int dims[] = {numattrdim, namestrdim}; - status = nc_def_var(exodusFilePtr, VAR_NAME_ESATTRIB(cur_num_elem_sets + 1), NC_CHAR, 2, dims, - &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define attribute name array for elemset {}" - " in file id {}", - elemsets[i].id, exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); + status = nc_def_var(exodusFilePtr, VAR_NAME_ESATTRIB(cur_num_elem_sets + 1), NC_CHAR, 2, + dims, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = + fmt::format("Error: failed to define attribute name array for elemset {}" + " in file id {}", + elemsets[i].id, exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); + } + ex__set_compact_storage(exodusFilePtr, varid); } - ex__set_compact_storage(exodusFilePtr, varid); } } return (EX_NOERR); @@ -3737,25 +3791,20 @@ int Internals::put_metadata(const std::vector &sidesets, bool count_onl if (sidesets.empty()) { return (EX_NOERR); } - - std::string errmsg; - int dims[2]; - int bulk_type = get_type(exodusFilePtr, EX_BULK_INT64_DB); - int status = 0; // clear error code // Get number of side sets defined for this file int dimid; int num_side_sets = 0; - status = nc_inq_dimid(exodusFilePtr, DIM_NUM_SS, &dimid); + int status = nc_inq_dimid(exodusFilePtr, DIM_NUM_SS, &dimid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_EBADDIM) { - errmsg = fmt::format("Error: no side sets defined for file id {}", exodusFilePtr); + std::string errmsg = fmt::format("Error: no side sets defined for file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = + std::string errmsg = fmt::format("Error: failed to locate side sets defined in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } @@ -3782,13 +3831,14 @@ int Internals::put_metadata(const std::vector &sidesets, bool count_onl if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: side set {} already defined in file id {}", sidesets[i].id, - exodusFilePtr); + std::string errmsg = fmt::format("Error: side set {} already defined in file id {}", + sidesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to define number of sides for set {} in file id {}", - sidesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define number of sides for set {} in file id {}", + sidesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -3797,20 +3847,22 @@ int Internals::put_metadata(const std::vector &sidesets, bool count_onl continue; } - dims[0] = dimid; - int varid = 0; + int dims[1] = {dimid}; + int varid = 0; status = nc_def_var(exodusFilePtr, VAR_ELEM_SS(cur_num_side_sets + 1), bulk_type, 1, dims, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: side set {} element list already defined in file id {}", - sidesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: side set {} element list already defined in file id {}", + sidesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create side set {} element list in file id {}", - sidesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create side set {} element list in file id {}", + sidesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -3823,13 +3875,15 @@ int Internals::put_metadata(const std::vector &sidesets, bool count_onl if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: side list already exists for side set {} in file id {}", - sidesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: side list already exists for side set {} in file id {}", + sidesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create side list for side set {} in file id {}", - sidesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create side list for side set {} in file id {}", + sidesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -3843,13 +3897,15 @@ int Internals::put_metadata(const std::vector &sidesets, bool count_onl if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: side set df count {} already defined in file id {}", - sidesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: side set df count {} already defined in file id {}", + sidesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to define side set df count for set {} in file id {}", - sidesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to define side set df count for set {} in file id {}", + sidesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -3862,15 +3918,16 @@ int Internals::put_metadata(const std::vector &sidesets, bool count_onl if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { - errmsg = fmt::format("Error: dist factor list already exists for side set {}" - " in file id {}", - sidesets[i].id, exodusFilePtr); + std::string errmsg = fmt::format("Error: dist factor list already exists for side set {}" + " in file id {}", + sidesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } else { - errmsg = fmt::format("Error: failed to create dist factor list for side set {}" - " in file id {}", - sidesets[i].id, exodusFilePtr); + std::string errmsg = + fmt::format("Error: failed to create dist factor list for side set {}" + " in file id {}", + sidesets[i].id, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); } return (EX_FATAL); @@ -3952,17 +4009,13 @@ namespace { int conditional_define_variable(int exodusFilePtr, const char *var, int dimid, int *varid, nc_type type) { - int status; - - std::string errmsg; - - status = nc_inq_varid(exodusFilePtr, var, varid); + int status = nc_inq_varid(exodusFilePtr, var, varid); if (status != NC_NOERR) { status = nc_def_var(exodusFilePtr, var, type, 1, &dimid, varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: Failed to define variable \"{}\" in file ID {}", var, - exodusFilePtr); + std::string errmsg = fmt::format("Error: Failed to define variable \"{}\" in file ID {}", + var, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -3976,16 +4029,13 @@ namespace { { int dimid[1]; int varid; - int status; - - std::string errmsg; if (size > 0) { - status = nc_def_dim(exodusFilePtr, dim, size, &dimid[0]); + int status = nc_def_dim(exodusFilePtr, dim, size, &dimid[0]); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to dimension \"{}\" in file id {}", DIM_NUM_BOR_ELEMS, - exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to dimension \"{}\" in file id {}", + DIM_NUM_BOR_ELEMS, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -3993,8 +4043,8 @@ namespace { status = nc_def_var(exodusFilePtr, var, type, 1, dimid, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define variable \"{}\" in file ID {}", - VAR_ELEM_MAP_BOR, exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to define variable \"{}\" in file ID {}", + VAR_ELEM_MAP_BOR, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -4008,16 +4058,13 @@ namespace { { int dimid[1]; int varid; - int status; - - std::string errmsg; if (size > 0) { - status = nc_def_dim(exodusFilePtr, dim, size, &dimid[0]); + int status = nc_def_dim(exodusFilePtr, dim, size, &dimid[0]); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to dimension \"{}\" in file id {}", DIM_NUM_BOR_ELEMS, - exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to dimension \"{}\" in file id {}", + DIM_NUM_BOR_ELEMS, exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -4027,13 +4074,12 @@ namespace { status = nc_def_var(exodusFilePtr, var[i], types[i], 1, dimid, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define variable \"{}\" in file ID {}", - VAR_ELEM_MAP_BOR, exodusFilePtr); + std::string errmsg = fmt::format("Error: failed to define variable \"{}\" in file ID {}", + var[i], exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } ex__set_compact_storage(exodusFilePtr, varid); - ex__compress_variable(exodusFilePtr, varid, 1); i++; } } @@ -4042,14 +4088,12 @@ namespace { int put_int_array(int exoid, const char *var_type, const std::vector &array) { - std::string errmsg; - int var_id; - int status; + int var_id; - status = nc_inq_varid(exoid, var_type, &var_id); + int status = nc_inq_varid(exoid, var_type, &var_id); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to locate {} in file id {}", var_type, exoid); + std::string errmsg = fmt::format("Error: failed to locate {} in file id {}", var_type, exoid); ex_err_fn(exoid, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -4057,7 +4101,8 @@ namespace { status = nc_put_var_int(exoid, var_id, array.data()); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to write {} array in file id {}", var_type, exoid); + std::string errmsg = + fmt::format("Error: failed to write {} array in file id {}", var_type, exoid); ex_err_fn(exoid, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -4066,14 +4111,12 @@ namespace { int put_int_array(int exoid, const char *var_type, const std::vector &array) { - std::string errmsg; - int var_id; - int status; + int var_id; - status = nc_inq_varid(exoid, var_type, &var_id); + int status = nc_inq_varid(exoid, var_type, &var_id); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to locate {} in file id {}", var_type, exoid); + std::string errmsg = fmt::format("Error: failed to locate {} in file id {}", var_type, exoid); ex_err_fn(exoid, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -4081,7 +4124,8 @@ namespace { status = nc_put_var_longlong(exoid, var_id, (long long *)array.data()); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to write {} array in file id {}", var_type, exoid); + std::string errmsg = + fmt::format("Error: failed to write {} array in file id {}", var_type, exoid); ex_err_fn(exoid, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -4090,13 +4134,12 @@ namespace { int put_id_array(int exoid, const char *var_type, const std::vector &ids) { - std::string errmsg; - int var_id; + int var_id; int status = nc_inq_varid(exoid, var_type, &var_id); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to locate {} in file id {}", var_type, exoid); + std::string errmsg = fmt::format("Error: failed to locate {} in file id {}", var_type, exoid); ex_err_fn(exoid, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -4122,7 +4165,8 @@ namespace { if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to write {} array in file id {}", var_type, exoid); + std::string errmsg = + fmt::format("Error: failed to write {} array in file id {}", var_type, exoid); ex_err_fn(exoid, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -4132,77 +4176,58 @@ namespace { int define_coordinate_vars(int exodusFilePtr, int64_t nodes, int node_dim, int dimension, int dim_dim, int str_dim) { - std::string errmsg; - int status; - int dim[2]; - int varid; + int varid; if (nodes > 0) { - if (ex_large_model(exodusFilePtr) == 1) { - // node coordinate arrays -- separate storage... - dim[0] = node_dim; - if (dimension > 0) { - status = - nc_def_var(exodusFilePtr, VAR_COORD_X, nc_flt_code(exodusFilePtr), 1, dim, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define node x coordinate array in file id {}", - exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); - } - ex__compress_variable(exodusFilePtr, varid, 2); - } - - if (dimension > 1) { - status = - nc_def_var(exodusFilePtr, VAR_COORD_Y, nc_flt_code(exodusFilePtr), 1, dim, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define node y coordinate array in file id {}", - exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); - } - ex__compress_variable(exodusFilePtr, varid, 2); + // node coordinate arrays -- separate storage... + int dim[] = {node_dim}; + if (dimension > 0) { + int status = + nc_def_var(exodusFilePtr, VAR_COORD_X, nc_flt_code(exodusFilePtr), 1, dim, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = fmt::format( + "Error: failed to define node x coordinate array in file id {}", exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 2); + } - if (dimension > 2) { - status = - nc_def_var(exodusFilePtr, VAR_COORD_Z, nc_flt_code(exodusFilePtr), 1, dim, &varid); - if (status != NC_NOERR) { - ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define node z coordinate array in file id {}", - exodusFilePtr); - ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); - return (EX_FATAL); - } - ex__compress_variable(exodusFilePtr, varid, 2); + if (dimension > 1) { + int status = + nc_def_var(exodusFilePtr, VAR_COORD_Y, nc_flt_code(exodusFilePtr), 1, dim, &varid); + if (status != NC_NOERR) { + ex_opts(EX_VERBOSE); + std::string errmsg = fmt::format( + "Error: failed to define node y coordinate array in file id {}", exodusFilePtr); + ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); + return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 2); } - else { - // node coordinate arrays: -- all stored together (old method)2 - dim[0] = dim_dim; - dim[1] = node_dim; - status = nc_def_var(exodusFilePtr, VAR_COORD, nc_flt_code(exodusFilePtr), 2, dim, &varid); + + if (dimension > 2) { + int status = + nc_def_var(exodusFilePtr, VAR_COORD_Z, nc_flt_code(exodusFilePtr), 1, dim, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define node coordinate array in file id {}", - exodusFilePtr); + std::string errmsg = fmt::format( + "Error: failed to define node z coordinate array in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); } + ex__compress_variable(exodusFilePtr, varid, 2); } } // coordinate names array - dim[0] = dim_dim; - dim[1] = str_dim; + int dim[] = {dim_dim, str_dim}; - status = nc_def_var(exodusFilePtr, VAR_NAME_COOR, NC_CHAR, 2, dim, &varid); + int status = nc_def_var(exodusFilePtr, VAR_NAME_COOR, NC_CHAR, 2, dim, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = + std::string errmsg = fmt::format("Error: failed to define coordinate name array in file id {}", exodusFilePtr); ex_err_fn(exodusFilePtr, __func__, errmsg.c_str(), status); return (EX_FATAL); @@ -4218,16 +4243,14 @@ namespace { return EX_NOERR; } - size_t sixty_four_kb = 64 * 1024; - int dimid = 0; - int varid = 0; - int dim[2]; - int namestrdim = 0; - std::string errmsg; + size_t sixty_four_kb = 64 * 1024; + int dimid = 0; + int varid = 0; + int namestrdim = 0; int status = nc_inq_dimid(exoid, DIM_STR_NAME, &namestrdim); if (status != NC_NOERR) { - errmsg = fmt::format("Error: failed to get string length in file id {}", exoid); + std::string errmsg = fmt::format("Error: failed to get string length in file id {}", exoid); ex_err_fn(exoid, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -4235,17 +4258,19 @@ namespace { status = nc_def_dim(exoid, dim_num, count, &dimid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define number of {}s in file id {}", type, exoid); + std::string errmsg = + fmt::format("Error: failed to define number of {}s in file id {}", type, exoid); ex_err_fn(exoid, __func__, errmsg.c_str(), status); return (EX_FATAL); } // id status array: - dim[0] = dimid; - status = nc_def_var(exoid, stat_var, NC_INT, 1, dim, &varid); + int dim1[] = {dimid}; + status = nc_def_var(exoid, stat_var, NC_INT, 1, dim1, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define side {} status in file id {}", type, exoid); + std::string errmsg = + fmt::format("Error: failed to define side {} status in file id {}", type, exoid); ex_err_fn(exoid, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -4256,10 +4281,11 @@ namespace { // id array: int ids_type = get_type(exoid, EX_IDS_INT64_DB); int ids_size = ids_type == NC_INT ? 4 : 8; - status = nc_def_var(exoid, id_var, ids_type, 1, dim, &varid); + status = nc_def_var(exoid, id_var, ids_type, 1, dim1, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define {} property in file id {}", type, exoid); + std::string errmsg = + fmt::format("Error: failed to define {} property in file id {}", type, exoid); ex_err_fn(exoid, __func__, errmsg.c_str(), status); return (EX_FATAL); } @@ -4271,19 +4297,19 @@ namespace { status = nc_put_att_text(exoid, varid, ATT_PROP_NAME, 3, "ID"); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to store {} property name {} in file id {}", type, "ID", - exoid); + std::string errmsg = fmt::format("Error: failed to store {} property name {} in file id {}", + type, "ID", exoid); ex_err_fn(exoid, __func__, errmsg.c_str(), status); return (EX_FATAL); } if (name_var != nullptr) { - dim[0] = dimid; - dim[1] = namestrdim; - status = nc_def_var(exoid, name_var, NC_CHAR, 2, dim, &varid); + int dim[] = {dimid, namestrdim}; + status = nc_def_var(exoid, name_var, NC_CHAR, 2, dim, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); - errmsg = fmt::format("Error: failed to define {} name array in file id {}", type, exoid); + std::string errmsg = + fmt::format("Error: failed to define {} name array in file id {}", type, exoid); ex_err_fn(exoid, __func__, errmsg.c_str(), status); return (EX_FATAL); } diff --git a/packages/seacas/libraries/ioss/src/faodel/Iofaodel_Utils.C b/packages/seacas/libraries/ioss/src/faodel/Iofaodel_Utils.C index 99ae46fc0483..c60aeba90570 100644 --- a/packages/seacas/libraries/ioss/src/faodel/Iofaodel_Utils.C +++ b/packages/seacas/libraries/ioss/src/faodel/Iofaodel_Utils.C @@ -427,7 +427,7 @@ namespace Iofaodel { }; } - std::string get_entity_name(const kelpie::Key &k, std::string target) + std::string get_entity_name(const kelpie::Key &k, const std::string &target) { std::string name; @@ -440,7 +440,8 @@ namespace Iofaodel { return name; } - std::set get_entity_names(const std::vector &keys, std::string target) + std::set get_entity_names(const std::vector &keys, + const std::string & target) { std::set names; for (auto k : keys) { diff --git a/packages/seacas/libraries/ioss/src/generated/Iogn_GeneratedMesh.C b/packages/seacas/libraries/ioss/src/generated/Iogn_GeneratedMesh.C index bf3c904c9d22..6f5dbdfcc96a 100644 --- a/packages/seacas/libraries/ioss/src/generated/Iogn_GeneratedMesh.C +++ b/packages/seacas/libraries/ioss/src/generated/Iogn_GeneratedMesh.C @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -121,6 +122,7 @@ namespace Iogn { } void GeneratedMesh::create_tets(bool yesno) { createTets = yesno; } + void GeneratedMesh::create_pyramids(bool yesno) { createPyramids = yesno; } int64_t GeneratedMesh::add_shell_block(ShellLocation loc) { @@ -209,7 +211,7 @@ namespace Iogn { } } } - else if (option[0] == "nodeset") { + else if (option[0] == "nodeset" || option[0] == "nset") { // Option of the form "nodeset:xXyYzZ" // The argument specifies whether there is a nodeset // at the location. 'x' is minX, 'X' is maxX, etc. @@ -228,7 +230,7 @@ namespace Iogn { } } } - else if (option[0] == "sideset") { + else if (option[0] == "sideset" || option[0] == "sset") { // Option of the form "sideset:xXyYzZ" // The argument specifies whether there is a sideset // at the location. 'x' is minX, 'X' is maxX, etc. @@ -311,7 +313,7 @@ namespace Iogn { } } - else if (option[0] == "times") { + else if (option[0] == "times" || option[0] == "steps") { timestepCount = std::stoull(option[1]); } @@ -319,6 +321,16 @@ namespace Iogn { createTets = true; } + else if (option[0] == "pyramids") { + createPyramids = true; + if (processorCount > 1) { + std::ostringstream errmsg; + fmt::print(errmsg, "ERROR: Pyramid option can currently only be used in a serial run. " + "Parallel not supported yet.\n"); + IOSS_ERROR(errmsg); + } + } + else if (option[0] == "variables") { // Variables Option of the form "variables:global,10,element,100,..." auto tokens = Ioss::tokenize(option[1], ","); @@ -346,6 +358,7 @@ namespace Iogn { "\tnodeset:xXyYzZ (specifies which plane to apply nodeset)\n" "\tsideset:xXyYzZ (specifies which plane to apply sideset)\n" "\ttets (split each hex into 6 tets)\n" + "\tpyramids (split each hex into 6 pyramids)\n" "\tvariables:type,count,... " "type=global|element|node|nodal|nodeset|sideset|surface\n" "\ttimes:count (number of timesteps to generate)\n" @@ -398,9 +411,25 @@ namespace Iogn { } } - int64_t GeneratedMesh::node_count() const { return (numX + 1) * (numY + 1) * (numZ + 1); } + int64_t GeneratedMesh::node_count() const + { + auto count = (numX + 1) * (numY + 1) * (numZ + 1); + if (createPyramids) { + // Handle node at center of hex which is used as vertex of each pyramid + count += numX * numY * numZ; + } + return count; + } - int64_t GeneratedMesh::node_count_proc() const { return (numX + 1) * (numY + 1) * (myNumZ + 1); } + int64_t GeneratedMesh::node_count_proc() const + { + auto count = (numX + 1) * (numY + 1) * (myNumZ + 1); + if (createPyramids) { + // Handle node at center of hex which is used as vertex of each pyramid + count += numX * numY * myNumZ; + } + return count; + } int64_t GeneratedMesh::block_count() const { return shellBlocks.size() + 1; } @@ -431,7 +460,7 @@ namespace Iogn { assert(block_number <= block_count()); if (block_number == 1) { - int64_t mult = createTets ? 6 : 1; + int64_t mult = (createTets || createPyramids) ? 6 : 1; return mult * numX * numY * numZ; } ShellLocation loc = shellBlocks[block_number - 2]; @@ -457,7 +486,7 @@ namespace Iogn { assert(block_number <= block_count()); if (block_number == 1) { - int64_t mult = createTets ? 6 : 1; + int64_t mult = (createTets || createPyramids) ? 6 : 1; return mult * numX * numY * myNumZ; } ShellLocation loc = shellBlocks[block_number - 2]; @@ -591,6 +620,13 @@ namespace Iogn { return std::make_pair(std::string(Ioss::TriShell3::name), 3); } + if (createPyramids) { + if (block_number == 1) { + return std::make_pair(std::string(Ioss::Pyramid5::name), 5); + } + return std::make_pair(std::string(Ioss::Shell4::name), 4); + } + if (block_number == 1) { return std::make_pair(std::string(Ioss::Hex8::name), 8); } @@ -687,8 +723,8 @@ namespace Iogn { map.reserve(count); if (block_number == 1) { - // Hex/Tet block... - INT mult = createTets ? 6 : 1; + // Hex/Tet/Pyramid block... + INT mult = (createTets || createPyramids) ? 6 : 1; count = element_count_proc(1); INT offset = mult * myStartZ * numX * numY; for (INT i = 0; i < count; i++) { @@ -733,7 +769,7 @@ namespace Iogn { template void GeneratedMesh::raw_element_map(std::vector &map) const { - INT mult = createTets ? 6 : 1; + INT mult = (createTets || createPyramids) ? 6 : 1; INT count = element_count_proc(); map.reserve(count); @@ -861,6 +897,92 @@ namespace Iogn { break; } } + else if (createPyramids) { + // For pyramid elements + // NOT IMPLEMENTED AT ALL YET + assert(1 == 0); + switch (loc) { + case MX: + offset = myStartZ * numX * numY + 1; + for (size_t k = 0; k < myNumZ; ++k) { + for (size_t j = 0; j < numY; ++j) { + map[index++] = 6 * offset - 4; // 1-based elem id + map[index++] = 3; // 0-based local face id + map[index++] = 6 * offset - 3; + map[index++] = 3; // 0-based local face id + offset += numX; + } + } + break; + + case PX: + offset = myStartZ * numX * numY + numX; + for (size_t k = 0; k < myNumZ; ++k) { + for (size_t j = 0; j < numY; ++j) { + map[index++] = 6 * offset - 1; // 1-based elem id + map[index++] = 3; // 0-based local face id + map[index++] = 6 * offset; // 1-based elem id + map[index++] = 3; // 0-based local face id + offset += numX; + } + } + break; + + case MY: + offset = myStartZ * numX * numY + 1; + for (size_t k = 0; k < myNumZ; ++k) { + for (size_t i = 0; i < numX; ++i) { + map[index++] = 6 * offset - 2; // 1-based elem id + map[index++] = 0; // 0-based local face id + map[index++] = 6 * offset++ - 1; // 1-based elem id + map[index++] = 0; // 0-based local face id + } + offset += numX * (numY - 1); + } + break; + + case PY: + offset = myStartZ * numX * numY + numX * (numY - 1) + 1; + for (size_t k = 0; k < myNumZ; ++k) { + for (size_t i = 0; i < numX; ++i) { + map[index++] = 6 * offset - 5; + map[index++] = 1; // 0-based local face id + map[index++] = 6 * offset++ - 4; + map[index++] = 1; // 0-based local face id + } + offset += numX * (numY - 1); + } + break; + + case MZ: + if (myProcessor == 0) { + offset = 1; + for (size_t i = 0; i < numY; i++) { + for (size_t j = 0; j < numX; j++) { + map[index++] = 6 * offset - 5; + map[index++] = 3; + map[index++] = 6 * offset++; + map[index++] = 2; + } + } + } + break; + + case PZ: + if (myProcessor == processorCount - 1) { + offset = (numZ - 1) * numX * numY + 1; + for (size_t i = 0, k = 0; i < numY; i++) { + for (size_t j = 0; j < numX; j++, k++) { + map[index++] = 6 * offset - 3; + map[index++] = 1; + map[index++] = 6 * offset++ - 2; + map[index++] = 1; + } + } + } + break; + } + } else { // For hex elements switch (loc) { @@ -959,6 +1081,20 @@ namespace Iogn { } } + if (createPyramids) { + for (size_t m = myStartZ; m < myStartZ + myNumZ; m++) { + for (size_t i = 0; i < numY; i++) { + for (size_t j = 0; j < numX; j++) { + coord[k++] = sclX * static_cast(j) + 0.5 + offX; + coord[k++] = sclY * static_cast(i) + 0.5 + offY; + coord[k++] = sclZ * static_cast(m) + 0.5 + offZ; + } + } + } + } + + assert(k == 3 * node_count_proc()); + if (doRotation) { for (int64_t i = 0; i < count * 3; i += 3) { double xn = coord[i + 0]; @@ -1073,8 +1209,8 @@ namespace Iogn { void GeneratedMesh::connectivity(int64_t block_number, Ioss::Int64Vector &connect) const { - if (block_number == 1) { // HEX Element Block - int64_t npe = createTets ? 4 : 8; + if (block_number == 1) { // HEX/TET/PYR Element Block + int64_t npe = createTets ? 4 : createPyramids ? 5 : 8; connect.resize(element_count_proc(block_number) * npe); } else { @@ -1087,7 +1223,7 @@ namespace Iogn { void GeneratedMesh::connectivity(int64_t block_number, Ioss::IntVector &connect) const { if (block_number == 1) { // HEX Element Block - int64_t npe = createTets ? 4 : 8; + int64_t npe = createTets ? 4 : createPyramids ? 5 : 8; connect.resize(element_count_proc(block_number) * npe); } else { @@ -1149,6 +1285,41 @@ namespace Iogn { } } } + else if (createPyramids) { + INT pyr_vert[][5] = {{0, 1, 5, 4}, {1, 2, 6, 5}, {2, 3, 7, 6}, + {0, 4, 7, 3}, {0, 3, 2, 1}, {4, 5, 6, 7}}; + INT hex_vert[8]; + size_t cnt = 0; + INT offset = (numX + 1) * (numY + 1) * (myNumZ + 1); + + for (size_t m = myStartZ; m < myNumZ + myStartZ; m++) { + for (size_t i = 0, k = 0; i < numY; i++) { + for (size_t j = 0; j < numX; j++, k++) { + size_t base = (m * xp1yp1) + k + i + 1; + ++offset; + + hex_vert[0] = base; + hex_vert[1] = base + 1; + hex_vert[2] = base + numX + 2; + hex_vert[3] = base + numX + 1; + + hex_vert[4] = xp1yp1 + base; + hex_vert[5] = xp1yp1 + base + 1; + hex_vert[6] = xp1yp1 + base + numX + 2; + hex_vert[7] = xp1yp1 + base + numX + 1; + + for (auto &elem : pyr_vert) { + connect[cnt++] = hex_vert[elem[0]]; + connect[cnt++] = hex_vert[elem[1]]; + connect[cnt++] = hex_vert[elem[2]]; + connect[cnt++] = hex_vert[elem[3]]; + + connect[cnt++] = offset; + } + } + } + } + } else { // Hex elements size_t cnt = 0; diff --git a/packages/seacas/libraries/ioss/src/generated/Iogn_GeneratedMesh.h b/packages/seacas/libraries/ioss/src/generated/Iogn_GeneratedMesh.h index 4defec237dd5..9ebe70845e03 100644 --- a/packages/seacas/libraries/ioss/src/generated/Iogn_GeneratedMesh.h +++ b/packages/seacas/libraries/ioss/src/generated/Iogn_GeneratedMesh.h @@ -74,6 +74,9 @@ namespace Iogn { split into 6 tetrahedral elements. Cannot currently be used with shells or sidesets. + - pyramids -- no argument - specifies that each hex should be + split into 6 pyramidal elements. + - shell -- argument = xXyYzZ which specifies whether there is a shell block at that location. 'x' is minimum x face, 'X' is maximum x face, similarly for y and z. Note that the argument string is a single @@ -177,6 +180,12 @@ namespace Iogn { */ void create_tets(bool yesno); + /** + * Split each hexahedral element into 6 pyramidal elements. + * Cannot currently be used with sidesets or shells. + */ + void create_pyramids(bool yesno); + /** * Add a shell block along the specified face of the hex mesh. * The shell blocks will maintain the order of definition. The @@ -488,6 +497,7 @@ namespace Iogn { * sclY*i+offY, sclZ*i+offZ) */ bool doRotation{false}; bool createTets{false}; + bool createPyramids{false}; }; } // namespace Iogn #endif diff --git a/packages/seacas/libraries/ioss/src/init/CMakeLists.txt b/packages/seacas/libraries/ioss/src/init/CMakeLists.txt index 779598e54dfe..316909cd24d9 100644 --- a/packages/seacas/libraries/ioss/src/init/CMakeLists.txt +++ b/packages/seacas/libraries/ioss/src/init/CMakeLists.txt @@ -31,9 +31,11 @@ IF (TPL_ENABLE_ADIOS2) SET(ADIOS2LIB Ioad) ENDIF() -IF (NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows") - SET(IOVISLIB Iovs) -ENDIF() +SET(IOVISLIB Iovs) + +IF (TPL_ENABLE_Catalyst2) + SET(CATALYST2LIB Iocatalyst) +endif() TRIBITS_ADD_LIBRARY( Ionit @@ -41,7 +43,7 @@ TRIBITS_ADD_LIBRARY( SOURCES ${SOURCES} DEPLIBS Ioss ${EXODUSLIB} Iogn Iogs Iohb Iotr ${IOVISLIB} - ${PAMGENLIB} ${CGNSLIB} ${VISLIB} ${FAODELLIB} ${ADIOS2LIB} + ${PAMGENLIB} ${CGNSLIB} ${VISLIB} ${FAODELLIB} ${ADIOS2LIB} ${CATALYST2LIB} ) IF (BUILD_TESTING) diff --git a/packages/seacas/libraries/ioss/src/init/Ionit_Initializer.C b/packages/seacas/libraries/ioss/src/init/Ionit_Initializer.C index 3ad16bc6a453..3daea2db57a0 100644 --- a/packages/seacas/libraries/ioss/src/init/Ionit_Initializer.C +++ b/packages/seacas/libraries/ioss/src/init/Ionit_Initializer.C @@ -19,6 +19,10 @@ #include #endif +#if defined(SEACAS_HAVE_CATALYST2) +#include +#endif + #if defined(SEACAS_HAVE_PAMGEN) #include #endif @@ -34,10 +38,8 @@ #include #include #include -#ifndef _MSC_VER #include #include -#endif namespace { #if defined(IOSS_THREADSAFE) @@ -88,6 +90,9 @@ namespace Ioss { Iotr::Initializer(); #ifdef HAVE_SEACASIOSS_ADIOS2 Ioad::Initializer(); // ADIOS2 +#endif +#if defined(SEACAS_HAVE_CATALYST2) + Iocatalyst::Initializer(); // Catalyst 2 #endif } diff --git a/packages/seacas/libraries/ioss/src/main/cgns_decomp.C b/packages/seacas/libraries/ioss/src/main/cgns_decomp.C index 22ce6defc8bd..f288ad32c12e 100644 --- a/packages/seacas/libraries/ioss/src/main/cgns_decomp.C +++ b/packages/seacas/libraries/ioss/src/main/cgns_decomp.C @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -778,7 +779,7 @@ int main(int argc, char *argv[]) cleanup(zones); fmt::print(stderr, - "\nTotal Execution time = {:.5} seconds to decompose for {:L} processors. (decomp: " + "\nTotal Execution Time = {:.5} seconds to decompose for {:L} processors. (decomp: " "{:.5}, resolve_zgc: {:.5})\n", end2 - begin, interFace.proc_count, end1 - begin, end2 - end1); if (valid) { diff --git a/packages/seacas/libraries/ioss/src/main/io_info.C b/packages/seacas/libraries/ioss/src/main/io_info.C index 747b3954652f..cad14a4f7e9b 100644 --- a/packages/seacas/libraries/ioss/src/main/io_info.C +++ b/packages/seacas/libraries/ioss/src/main/io_info.C @@ -16,10 +16,6 @@ // ======================================================================== namespace { - - // Data space shared by most field input/output routines... - std::vector data; - void info_nodeblock(Ioss::Region ®ion, const Info::Interface &interFace); void info_edgeblock(Ioss::Region ®ion); void info_faceblock(Ioss::Region ®ion); @@ -195,8 +191,10 @@ namespace { if (interFace.compute_bbox()) { Ioss::AxisAlignedBoundingBox bbox = nb.get_bounding_box(); fmt::print("\tBounding Box: Minimum X,Y,Z = {:12.4e}\t{:12.4e}\t{:12.4e}\n" - "\t Maximum X,Y,Z = {:12.4e}\t{:12.4e}\t{:12.4e}\n", - bbox.xmin, bbox.ymin, bbox.zmin, bbox.xmax, bbox.ymax, bbox.zmax); + "\t Maximum X,Y,Z = {:12.4e}\t{:12.4e}\t{:12.4e}\n" + "\t Range X,Y,Z = {:12.4e}\t{:12.4e}\t{:12.4e}\n", + bbox.xmin, bbox.ymin, bbox.zmin, bbox.xmax, bbox.ymax, bbox.zmax, + bbox.xmax - bbox.xmin, bbox.ymax - bbox.ymin, bbox.zmax - bbox.zmin); } } @@ -263,8 +261,10 @@ namespace { if (interFace.compute_bbox()) { Ioss::AxisAlignedBoundingBox bbox = sb->get_bounding_box(); fmt::print("\tBounding Box: Minimum X,Y,Z = {:12.4e}\t{:12.4e}\t{:12.4e}\n" - "\t Maximum X,Y,Z = {:12.4e}\t{:12.4e}\t{:12.4e}\n", - bbox.xmin, bbox.ymin, bbox.zmin, bbox.xmax, bbox.ymax, bbox.zmax); + "\t Maximum X,Y,Z = {:12.4e}\t{:12.4e}\t{:12.4e}\n" + "\t Range X,Y,Z = {:12.4e}\t{:12.4e}\t{:12.4e}\n", + bbox.xmin, bbox.ymin, bbox.zmin, bbox.xmax, bbox.ymax, bbox.zmax, + bbox.xmax - bbox.xmin, bbox.ymax - bbox.ymin, bbox.zmax - bbox.zmin); } } } @@ -340,8 +340,10 @@ namespace { if (interFace.compute_bbox()) { Ioss::AxisAlignedBoundingBox bbox = eb->get_bounding_box(); fmt::print("\tBounding Box: Minimum X,Y,Z = {:12.4e}\t{:12.4e}\t{:12.4e}\n" - "\t Maximum X,Y,Z = {:12.4e}\t{:12.4e}\t{:12.4e}\n", - bbox.xmin, bbox.ymin, bbox.zmin, bbox.xmax, bbox.ymax, bbox.zmax); + "\t Maximum X,Y,Z = {:12.4e}\t{:12.4e}\t{:12.4e}\n" + "\t Range X,Y,Z = {:12.4e}\t{:12.4e}\t{:12.4e}\n", + bbox.xmin, bbox.ymin, bbox.zmin, bbox.xmax, bbox.ymax, bbox.zmax, + bbox.xmax - bbox.xmin, bbox.ymax - bbox.ymin, bbox.zmax - bbox.zmin); } } } diff --git a/packages/seacas/libraries/ioss/src/main/io_modify.C b/packages/seacas/libraries/ioss/src/main/io_modify.C index c249d8468a4c..5b3d9034a6eb 100644 --- a/packages/seacas/libraries/ioss/src/main/io_modify.C +++ b/packages/seacas/libraries/ioss/src/main/io_modify.C @@ -501,10 +501,8 @@ namespace { } } - void set_db_properties(const Modify::Interface &interFace, Ioss::DatabaseIO *dbi) + void set_db_properties(const Modify::Interface & /* interFace */, Ioss::DatabaseIO *dbi) { - std::string inpfile = interFace.filename(); - if (dbi == nullptr || !dbi->ok(true)) { std::exit(EXIT_FAILURE); } @@ -996,11 +994,8 @@ namespace { handle_list(tokens, region, true); return false; } - else { - fmt::print(stderr, fg(fmt::color::red), "ERROR: Unrecognized attribute command.\n"); - handle_help("attribute"); - return false; - } + fmt::print(stderr, fg(fmt::color::red), "ERROR: Unrecognized attribute command.\n"); + handle_help("attribute"); return false; } diff --git a/packages/seacas/libraries/ioss/src/main/io_shell.C b/packages/seacas/libraries/ioss/src/main/io_shell.C index 4530c39643f9..b4b70f33e621 100644 --- a/packages/seacas/libraries/ioss/src/main/io_shell.C +++ b/packages/seacas/libraries/ioss/src/main/io_shell.C @@ -141,11 +141,11 @@ int main(int argc, char *argv[]) if (rank == 0 && !interFace.quiet) { if (num_proc > 1) { - fmt::print(stderr, "\n\n\tTotal Execution time = {:.5} seconds on {} processors.\n", + fmt::print(stderr, "\n\n\tTotal Execution Time = {:.5} seconds on {} processors.\n", end - begin, num_proc); } else { - fmt::print(stderr, "\n\n\tTotal Execution time = {:.5} seconds.\n", end - begin); + fmt::print(stderr, "\n\n\tTotal Execution Time = {:.5} seconds.\n", end - begin); } } if (mem_stats) { @@ -407,7 +407,6 @@ namespace { void file_compare(IOShell::Interface &interFace, int rank) { Ioss::PropertyManager properties = set_properties(interFace); - int int_byte_size_api; for (const auto &inpfile : interFace.inputFile) { //======================================================================== @@ -466,7 +465,7 @@ namespace { // Get integer size being used on input file #1 and set it in // the interFace. - int_byte_size_api = dbi1->int_byte_size_api(); + int int_byte_size_api = dbi1->int_byte_size_api(); if (int_byte_size_api == 8) { interFace.ints_64_bit = true; } @@ -540,10 +539,10 @@ namespace { bool result = Ioss::Compare::compare_database(input_region1, input_region2, options); if (result) { - fmt::print(stderr, "\nDATABASES are EQUAL\n"); + fmt::print(stderr, "\n\nDATABASES are EQUAL"); } else { - fmt::print(stderr, "\nDATABASES are NOT equal\n"); + fmt::print(stderr, "\n\nDATABASES are NOT equal"); } } // loop over input files } diff --git a/packages/seacas/libraries/ioss/src/main/io_shell_ts.C b/packages/seacas/libraries/ioss/src/main/io_shell_ts.C index f851b92f793c..3e3c32be776e 100644 --- a/packages/seacas/libraries/ioss/src/main/io_shell_ts.C +++ b/packages/seacas/libraries/ioss/src/main/io_shell_ts.C @@ -168,11 +168,11 @@ int main(int argc, char *argv[]) if (rank == 0 && !interFace.quiet) { if (num_proc > 1) { - fmt::print(stderr, "\n\n\tTotal Execution time = {:.5} seconds on {} processors.\n", + fmt::print(stderr, "\n\n\tTotal Execution Time = {:.5} seconds on {} processors.\n", end - begin, num_proc); } else { - fmt::print(stderr, "\n\n\tTotal Execution time = {:.5} seconds.\n", end - begin); + fmt::print(stderr, "\n\n\tTotal Execution Time = {:.5} seconds.\n", end - begin); } } diff --git a/packages/seacas/libraries/ioss/src/main/skinner.C b/packages/seacas/libraries/ioss/src/main/skinner.C index aa66c5f23c7f..9910bfcfc967 100644 --- a/packages/seacas/libraries/ioss/src/main/skinner.C +++ b/packages/seacas/libraries/ioss/src/main/skinner.C @@ -119,7 +119,7 @@ int main(int argc, char *argv[]) double end = Ioss::Utils::timer(); if (my_rank == 0) { - fmt::print("\n\tTotal Execution time = {:.4} seconds\n", end - begin); + fmt::print("\n\tTotal Execution Time = {:.4} seconds\n", end - begin); fmt::print("\n{} execution successful.\n\n", codename); } return EXIT_SUCCESS; diff --git a/packages/seacas/libraries/ioss/src/pamgen/Iopg_DatabaseIO.C b/packages/seacas/libraries/ioss/src/pamgen/Iopg_DatabaseIO.C index 8118792688b3..3f812a665044 100644 --- a/packages/seacas/libraries/ioss/src/pamgen/Iopg_DatabaseIO.C +++ b/packages/seacas/libraries/ioss/src/pamgen/Iopg_DatabaseIO.C @@ -207,7 +207,7 @@ namespace Iopg { retval = Create_Pamgen_Mesh(mesh_description.c_str(), dimension, util().parallel_rank(), util().parallel_size(), INT_MAX); } - catch (const std::exception &x) { + catch (...) { error_detected = true; } diff --git a/packages/seacas/libraries/ioss/src/utest/CMakeLists.txt b/packages/seacas/libraries/ioss/src/utest/CMakeLists.txt index a464f8784bb8..04a5a2db2436 100644 --- a/packages/seacas/libraries/ioss/src/utest/CMakeLists.txt +++ b/packages/seacas/libraries/ioss/src/utest/CMakeLists.txt @@ -29,7 +29,8 @@ TRIBITS_ADD_EXECUTABLE( TRIBITS_ADD_TEST( Utst_blob_write_read NAME Utst_blob_write_read - NUM_MPI_PROCS 1 + NUM_MPI_PROCS ${NPROCS} + COMM mpi serial XHOSTTYPE Windows ) diff --git a/packages/seacas/libraries/ioss/src/visualization/utils/Iovs_Utils.C b/packages/seacas/libraries/ioss/src/visualization/utils/Iovs_Utils.C index b3b38b854e2c..45a6e3e94da6 100644 --- a/packages/seacas/libraries/ioss/src/visualization/utils/Iovs_Utils.C +++ b/packages/seacas/libraries/ioss/src/visualization/utils/Iovs_Utils.C @@ -15,14 +15,15 @@ #include #endif +#ifndef _WIN32 #include +#endif #include #ifdef _WIN32 #include #endif - namespace Iovs { std::string persistentLdLibraryPathEnvForCatalyst = ""; @@ -59,6 +60,7 @@ namespace Iovs { CatalystManagerBase *Utils::createCatalystManagerInstance() { +#ifdef IOSS_DLOPEN_ENABLED void *dlh = this->getDlHandle(); if (!dlh) { @@ -77,6 +79,9 @@ namespace Iovs { "'CreateCatalystManagerInstance'"); } return (*mkr)(); +#else + return nullptr; +#endif } std::unique_ptr @@ -385,7 +390,7 @@ namespace Iovs { #ifdef _WIN32 char *cbuf = _fullpath(nullptr, sierraInsDir.c_str(), _MAX_PATH); #else - char *cbuf = realpath(sierraInsDir.c_str(), nullptr); + char *cbuf = realpath(sierraInsDir.c_str(), nullptr); #endif std::string sierraInsPath = cbuf; free(cbuf); @@ -398,6 +403,13 @@ namespace Iovs { IOSS_ERROR(errmsg); } +#ifdef _WIN32 + { + std::ostringstream errmsg; + errmsg << "This code is not yet supported on windows...\n"; + IOSS_ERROR(errmsg); + } +#else char *cbase = strdup(sierraInsPath.c_str()); char *cdir = strdup(sierraInsPath.c_str()); char *bname = basename(cbase); @@ -414,6 +426,7 @@ namespace Iovs { free(cbase); free(cdir); +#endif return sierraInsPath + "/" + CATALYST_PLUGIN_PATH + "/" + sierraSystem + CATALYST_IOSS_CATALYST_PLUGIN_DIR; diff --git a/packages/seacas/libraries/mapvarlib/flgrad.f b/packages/seacas/libraries/mapvarlib/flgrad.f index b8385349ab66..e7758659a8e1 100644 --- a/packages/seacas/libraries/mapvarlib/flgrad.f +++ b/packages/seacas/libraries/mapvarlib/flgrad.f @@ -1,4 +1,4 @@ -C Copyright(C) 1999-2020 National Technology & Engineering Solutions +C Copyright(C) 1999-2021 National Technology & Engineering Solutions C of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with C NTESS, the U.S. Government retains certain rights in this software. C @@ -16,7 +16,7 @@ SUBROUTINE FLGRAD(IEL,ICOUNT,IELLST,CNTRA,SHLNRM,SOLEA,SOLGRA, C Start by translating into isoparametric coords (very helpful for C shells and makes some sense for continuum and is very useful for C subsequent interpolation. -C Then do constrained least sqares to f=a0+a1eta+a2ksi+a3phi to +C Then do constrained least squares to f=a0+a1eta+a2ksi+a3phi to C compute a1, a2, and a3 and stuff results into SOLGRA C Calls subroutines ERROR diff --git a/packages/seacas/libraries/supes/ext_lib/getline_int.c b/packages/seacas/libraries/supes/ext_lib/getline_int.c index 6618133dce26..5c8c9b025b57 100644 --- a/packages/seacas/libraries/supes/ext_lib/getline_int.c +++ b/packages/seacas/libraries/supes/ext_lib/getline_int.c @@ -1,6 +1,6 @@ /* - * Copyright(C) 1999-2020 National Technology & Engineering Solutions + * Copyright(C) 1999-2021 National Technology & Engineering Solutions * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with * NTESS, the U.S. Government retains certain rights in this software. * @@ -183,7 +183,7 @@ static void search_back(int new_search); /* look back for current string */ static void search_forw(int new_search); /* look forw for current string */ static void gl_beep(void); /* try to play a system beep sound */ -static int gl_do_tab_completion(char *buf, int *loc, size_t bufsize, int tabtab); +static int gl_do_tab_completion(char *buf, int *loc, size_t bufsize); static char *copy_string(char *dest, char const *source, long int elements) { @@ -357,17 +357,16 @@ static int gl_getcx(int tlen) * after tlen tenths of a second. */ { - int c, result; - char ch; - fd_set ss; - struct timeval tv; for (errno = 0;;) { + fd_set ss; FD_ZERO(&ss); FD_SET(0, &ss); /* set STDIN_FILENO */ + + struct timeval tv; tv.tv_sec = tlen / 10; tv.tv_usec = (tlen % 10) * 100000L; - result = select(1, &ss, NULL, NULL, &tv); + int result = select(1, &ss, NULL, NULL, &tv); if (result == 1) { /* ready */ break; @@ -382,7 +381,8 @@ static int gl_getcx(int tlen) } for (errno = 0;;) { - c = read(0, &ch, 1); + char ch; + int c = read(0, &ch, 1); if (c == 1) return ((int)ch); if (errno != EINTR) @@ -435,10 +435,8 @@ static void gl_putc(int c) static void gl_puts(const char *const buf) { - int len; - if (buf) { - len = strlen(buf); + int len = strlen(buf); write(1, buf, len); } } @@ -455,13 +453,10 @@ static void gl_error(const char *const buf) static void gl_init(void) /* set up variables and terminal */ { - const char *cp; - int w; - if (gl_init_done < 0) { /* -1 only on startup */ - cp = (const char *)getenv("COLUMNS"); + const char *cp = (const char *)getenv("COLUMNS"); if (cp != NULL) { - w = atoi(cp); + int w = atoi(cp); if (w > 20) gl_setwidth(w); } @@ -500,11 +495,9 @@ void gl_setwidth(int w) char *getline_int(char *prompt) { - int c, loc, tmp, lastch; - int vi_count, count; - int vi_delete; - char vi_countbuf[32]; - char *cp; + int c, lastch; + int count; + char vi_countbuf[32]; #ifdef __unix__ int sig; @@ -514,12 +507,12 @@ char *getline_int(char *prompt) * don't start in gl_vi_mode. They need to hit * ESC to go into vi command mode. */ - gl_vi_mode = 0; - vi_count = 0; - vi_delete = 0; + gl_vi_mode = 0; + int vi_count = 0; + int vi_delete = 0; if (gl_vi_preferred < 0) { gl_vi_preferred = 0; - cp = (char *)getenv("EDITOR"); + char *cp = (char *)getenv("EDITOR"); if (cp != NULL) gl_vi_preferred = (strstr(cp, "vi") != NULL); } @@ -628,7 +621,7 @@ char *getline_int(char *prompt) gl_kill(gl_pos); } else { - loc = (int)strlen(gl_buf); + int loc = (int)strlen(gl_buf); if (loc > 1) loc--; gl_fixup(gl_prompt, -1, loc); @@ -728,10 +721,11 @@ char *getline_int(char *prompt) gl_del(-1, 0); /* ^H and DEL */ break; case '\t': /* TAB */ + { + int tmp = gl_pos; if (gl_completion_proc) { - tmp = gl_pos; gl_buf[sizeof(gl_buf) - 1] = '\0'; - loc = gl_do_tab_completion(gl_buf, &tmp, sizeof(gl_buf), (lastch == '\t')); + int loc = gl_do_tab_completion(gl_buf, &tmp, sizeof(gl_buf)); gl_buf[sizeof(gl_buf) - 1] = '\0'; if (loc >= 0 || tmp != gl_pos) gl_fixup(gl_prompt, /* loc */ -2, tmp); @@ -741,14 +735,13 @@ char *getline_int(char *prompt) } } else if (gl_tab_hook) { - tmp = gl_pos; gl_buf[sizeof(gl_buf) - 1] = '\0'; - loc = gl_tab_hook(gl_buf, (int)gl_strlen(gl_prompt), &tmp, sizeof(gl_buf)); + int loc = gl_tab_hook(gl_buf, (int)gl_strlen(gl_prompt), &tmp, sizeof(gl_buf)); gl_buf[sizeof(gl_buf) - 1] = '\0'; if (loc >= 0 || tmp != gl_pos) gl_fixup(gl_prompt, loc, tmp); } - break; + } break; case '\013': gl_kill(gl_pos); /* ^K */ break; @@ -879,12 +872,10 @@ static void gl_addchar(int c) /* adds the character c to the input buffer at current location */ { - int i; - if (gl_cnt >= GL_BUF_SIZE - 1) gl_error("\n*** Error: getline(): input buffer overflow\n"); if (gl_overwrite == 0 || gl_pos == gl_cnt) { - for (i = gl_cnt; i >= gl_pos; i--) + for (int i = gl_cnt; i >= gl_pos; i--) gl_buf[i + 1] = gl_buf[i]; gl_buf[gl_pos] = (char)c; gl_fixup(gl_prompt, gl_pos, gl_pos + 1); @@ -899,16 +890,14 @@ static void gl_addchar(int c) static void gl_yank(void) /* adds the kill buffer to the input buffer at current location */ { - int i, len; - - len = strlen(gl_killbuf); + int len = strlen(gl_killbuf); if (len > 0) { if (gl_overwrite == 0) { if (gl_cnt + len >= GL_BUF_SIZE - 1) gl_error("\n*** Error: getline(): input buffer overflow\n"); - for (i = gl_cnt; i >= gl_pos; i--) + for (int i = gl_cnt; i >= gl_pos; i--) gl_buf[i + len] = gl_buf[i]; - for (i = 0; i < len; i++) + for (int i = 0; i < len; i++) gl_buf[gl_pos + i] = gl_killbuf[i]; gl_fixup(gl_prompt, gl_pos, gl_pos + len); } @@ -918,7 +907,7 @@ static void gl_yank(void) gl_error("\n*** Error: getline(): input buffer overflow\n"); gl_buf[gl_pos + len] = 0; } - for (i = 0; i < len; i++) + for (int i = 0; i < len; i++) gl_buf[gl_pos + i] = gl_killbuf[i]; gl_extent = len; gl_fixup(gl_prompt, gl_pos, gl_pos + len); @@ -931,10 +920,8 @@ static void gl_yank(void) static void gl_transpose(void) /* switch character under cursor and to left of cursor */ { - int c; - if (gl_pos > 0 && gl_cnt > gl_pos) { - c = gl_buf[gl_pos - 1]; + int c = gl_buf[gl_pos - 1]; gl_buf[gl_pos - 1] = gl_buf[gl_pos]; gl_buf[gl_pos] = (char)c; gl_extent = 2; @@ -1008,11 +995,7 @@ static void gl_kill(int pos) static void gl_killword(int direction) { - int pos = gl_pos; - int startpos = gl_pos; - int tmp; - int i; - + int pos = gl_pos; if (direction > 0) { /* forward */ while (pos < gl_cnt && !isspace(gl_buf[pos])) pos++; @@ -1029,8 +1012,9 @@ static void gl_killword(int direction) if (pos < gl_cnt && isspace(gl_buf[pos])) /* move onto word */ pos++; } + int startpos = gl_pos; if (pos < startpos) { - tmp = pos; + int tmp = pos; pos = startpos; startpos = tmp; } @@ -1039,7 +1023,7 @@ static void gl_killword(int direction) if (isspace(gl_killbuf[pos - startpos - 1])) gl_killbuf[pos - startpos - 1] = '\0'; gl_fixup(gl_prompt, -1, startpos); - for (i = 0, tmp = pos - startpos; i < tmp; i++) + for (int i = 0, tmp = pos - startpos; i < tmp; i++) gl_del(0, 0); } /* gl_killword */ @@ -1209,18 +1193,16 @@ static void gl_fixup(const char *prompt, int change, int cursor) static int gl_tab(char *buf, int offset, int *loc, size_t bufsize) /* default tab handler, acts like tabstops every 8 cols */ { - int i, count, len; - - len = strlen(buf); - count = 8 - (offset + *loc) % 8; - for (i = len; i >= *loc; i--) + int len = strlen(buf); + int count = 8 - (offset + *loc) % 8; + for (int i = len; i >= *loc; i--) if (i + count < (int)bufsize) buf[i + count] = buf[i]; - for (i = 0; i < count; i++) + for (int i = 0; i < count; i++) if (*loc + i < (int)bufsize) buf[*loc + i] = ' '; - i = *loc; - *loc = i + count; + int i = *loc; + *loc = i + count; return i; } @@ -1236,10 +1218,8 @@ static char hist_empty_elem[2] = ""; static void hist_init(void) { - int i; - hist_buf[0] = hist_empty_elem; - for (i = 1; i < HIST_SIZE; i++) + for (int i = 1; i < HIST_SIZE; i++) hist_buf[i] = (char *)0; } @@ -1247,7 +1227,6 @@ void gl_histadd(char *buf) { static char *prev = 0; char * p = buf; - int len; /* in case we call gl_histadd() before we call getline() */ if (gl_init_done < 0) { /* -1 only on startup */ @@ -1257,7 +1236,7 @@ void gl_histadd(char *buf) while (*p == ' ' || *p == '\t' || *p == '\n') p++; if (*p) { - len = strlen(buf); + int len = strlen(buf); if (strchr(p, '\n')) /* previously line already has NL stripped */ len--; if ((prev == 0) || ((int)strlen(prev) != len) || strncmp(prev, buf, (size_t)len) != 0) { @@ -1332,21 +1311,17 @@ static char *hist_save(char *p) void gl_histsavefile(const char *const path) { - FILE * fp; - const char *p; - int i, j; - - fp = fopen(path, + FILE *fp = fopen(path, #if defined(__windows__) || defined(MSDOS) - "wt" + "wt" #else - "w" + "w" #endif ); if (fp != NULL) { - for (i = 2; i < HIST_SIZE; i++) { - j = (hist_pos + i) % HIST_SIZE; - p = hist_buf[j]; + for (int i = 2; i < HIST_SIZE; i++) { + int j = (hist_pos + i) % HIST_SIZE; + char *p = hist_buf[j]; if ((p == NULL) || (*p == '\0')) continue; fprintf(fp, "%s\n", p); @@ -1357,17 +1332,15 @@ void gl_histsavefile(const char *const path) void gl_histloadfile(const char *const path) { - FILE *fp; - char line[256]; - - fp = fopen(path, + FILE *fp = fopen(path, #if defined(__windows__) || defined(MSDOS) - "rt" + "rt" #else - "r" + "r" #endif ); if (fp != NULL) { + char line[256]; memset(line, 0, sizeof(line)); while (fgets(line, sizeof(line) - 2, fp) != NULL) { gl_histadd(line); @@ -1461,7 +1434,7 @@ static void search_term(void) static void search_back(int new_search) { int found = 0; - char *p, *loc; + char *loc; search_forw_flg = 0; if (gl_search_mode == 0) { @@ -1473,7 +1446,7 @@ static void search_back(int new_search) } else if (search_pos > 0) { while (!found) { - p = hist_prev(); + char *p = hist_prev(); if (*p == 0) { /* not found, done looking */ gl_buf[0] = 0; gl_fixup(search_prompt, 0, 0); @@ -1537,31 +1510,25 @@ static void gl_beep(void) #endif } /* gl_beep */ -static int gl_do_tab_completion(char *buf, int *loc, size_t bufsize, int tabtab) +static int gl_do_tab_completion(char *buf, int *loc, size_t bufsize) { char * startp; size_t startoff, amt; - int c; - int qmode; - char * qstart; - char * lastspacestart; - char * cp; int ntoalloc, nused, nalloced, i; char **newgl_matchlist; char * strtoadd, *strtoadd1; int addquotes; - size_t llen, mlen, glen; + size_t mlen, glen; int allmatch; char * curposp; size_t lenaftercursor; - char * matchpfx; int wasateol; char ellipsessave[4]; /* Zero out the rest of the buffer, so we can move stuff around * and know we'll still be NUL-terminated. */ - llen = strlen(buf); + size_t llen = strlen(buf); memset(buf + llen, 0, bufsize - llen); bufsize -= 4; /* leave room for a NUL, space, and two quotes. */ curposp = buf + *loc; @@ -1574,14 +1541,14 @@ static int gl_do_tab_completion(char *buf, int *loc, size_t bufsize, int tabtab) memcpy(curposp, ellipsessave, (size_t)4); } - qmode = 0; - qstart = NULL; - lastspacestart = NULL; - matchpfx = NULL; + int qmode = 0; + char *qstart = NULL; + char *lastspacestart = NULL; + char *matchpfx = NULL; - cp = buf; + char *cp = buf; while (cp < curposp) { - c = (int)*cp++; + int c = (int)*cp++; if (c == '\0') break; if ((c == '"') || (c == '\'')) { @@ -2148,7 +2115,6 @@ char *gl_local_filename_completion_proc(const char *start, int idx) char *gl_win_getpass(const char *const prompt, char *const pass, int dsize) { char *cp; - int c; FlushConsoleInputBuffer(GetStdHandle(STD_INPUT_HANDLE)); ZeroMemory(pass, (DWORD)sizeof(dsize)); @@ -2158,7 +2124,7 @@ char *gl_win_getpass(const char *const prompt, char *const pass, int dsize) _cputs(prompt); for (cp = pass;;) { - c = (int)_getch(); + int c = (int)_getch(); if ((c == '\r') || (c == '\n')) break; if ((c == '\010') || (c == '\177')) { diff --git a/packages/seacas/libraries/suplib_c/add_to_log.c b/packages/seacas/libraries/suplib_c/add_to_log.c index ac83c4e3d3ea..94fff8309196 100644 --- a/packages/seacas/libraries/suplib_c/add_to_log.c +++ b/packages/seacas/libraries/suplib_c/add_to_log.c @@ -8,7 +8,7 @@ #include #include -#ifndef _MSC_VER +#if !defined(_WIN64) && !defined(WIN32) && !defined(_WINDOWS) && !defined(_MSC_VER) #include #include #endif @@ -22,7 +22,7 @@ void add_to_log(const char *my_name, double elapsed) { -#ifndef _MSC_VER +#if !defined(_WIN64) && !defined(WIN32) && !defined(_WINDOWS) && !defined(_MSC_VER) #define LEN 512 char time_string[LEN]; char log_string[LEN]; diff --git a/packages/seacas/libraries/suplib_c/adler.c b/packages/seacas/libraries/suplib_c/adler.c index 8f1fe1ef6c68..809f0852b84e 100644 --- a/packages/seacas/libraries/suplib_c/adler.c +++ b/packages/seacas/libraries/suplib_c/adler.c @@ -3,7 +3,7 @@ * zlib project The zlib source is subject to the following copyright * and license: * - * Copyright (C) 1995-2010 Jean-loup Gailly and Mark Adler + * Copyright (C) 1995-2010, 2021 Jean-loup Gailly and Mark Adler * * This software is provided 'as-is', without any express or implied * warranty. In no event will the authors be held liable for any damages @@ -55,14 +55,13 @@ size_t adler(size_t adler, const void *vbuf, size_t len) size_t s1 = adler & 0xffff; size_t s2 = (adler >> 16) & 0xffff; - int k = 0; if (buf == NULL || len == 0) { return 1L; } while (len > 0) { - k = len < NMAX ? len : NMAX; + int k = len < NMAX ? len : NMAX; len -= k; while (k >= 16) { DO16(buf); diff --git a/packages/seacas/libraries/suplib_cpp/CMakeLists.txt b/packages/seacas/libraries/suplib_cpp/CMakeLists.txt index 62bdc014d510..ff358eaf7ffd 100644 --- a/packages/seacas/libraries/suplib_cpp/CMakeLists.txt +++ b/packages/seacas/libraries/suplib_cpp/CMakeLists.txt @@ -12,6 +12,7 @@ INCLUDE_DIRECTORIES( ) SET(SOURCES + FileInfo.C GetLongOpt.C smart_assert.C iqsort.C diff --git a/packages/seacas/applications/exodiff/FileInfo.C b/packages/seacas/libraries/suplib_cpp/FileInfo.C similarity index 100% rename from packages/seacas/applications/exodiff/FileInfo.C rename to packages/seacas/libraries/suplib_cpp/FileInfo.C diff --git a/packages/seacas/applications/exodiff/FileInfo.h b/packages/seacas/libraries/suplib_cpp/FileInfo.h similarity index 100% rename from packages/seacas/applications/exodiff/FileInfo.h rename to packages/seacas/libraries/suplib_cpp/FileInfo.h diff --git a/packages/seacas/libraries/suplib_cpp/scopeguard.h b/packages/seacas/libraries/suplib_cpp/scopeguard.h index 4d7b32ddeded..092246d46c92 100644 --- a/packages/seacas/libraries/suplib_cpp/scopeguard.h +++ b/packages/seacas/libraries/suplib_cpp/scopeguard.h @@ -35,6 +35,7 @@ class ScopeGuardImplBase { other.Dismiss(); } + template static void SafeExecute(J &j) { if (!j.dismissed_) { @@ -53,7 +54,11 @@ class ScopeGuardImplBase void Dismiss() const { dismissed_ = true; } }; -using ScopeGuard = const ScopeGuardImplBase &; +// typedef const ScopeGuardImplBase& ScopeGuard; +#ifndef _MSC_VER +__attribute__((unused)) +#endif +typedef const ScopeGuardImplBase &ScopeGuard; template class ScopeGuardImpl0 : public ScopeGuardImplBase { diff --git a/packages/seacas/scripts/exodus3.in.py b/packages/seacas/scripts/exodus3.in.py index aa8b03ccd4f2..3fc5c45a6fa5 100644 --- a/packages/seacas/scripts/exodus3.in.py +++ b/packages/seacas/scripts/exodus3.in.py @@ -1,5 +1,5 @@ """ -exodus.py v 1.20.8 (seacas-py3) is a python wrapper of some of the exodus library +exodus.py v 1.20.9 (seacas-py3) is a python wrapper of some of the exodus library (Python 3 Version) Exodus is a common database for multiple application codes (mesh @@ -70,10 +70,10 @@ EXODUS_PY_COPYRIGHT_AND_LICENSE = __doc__ -EXODUS_PY_VERSION = "1.20.8 (seacas-py3)" +EXODUS_PY_VERSION = "1.20.9 (seacas-py3)" EXODUS_PY_COPYRIGHT = """ -You are using exodus.py v 1.20.8 (seacas-py3), a python wrapper of some of the exodus library. +You are using exodus.py v 1.20.9 (seacas-py3), a python wrapper of some of the exodus library. Copyright (c) 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 National Technology & Engineering Solutions of Sandia, LLC (NTESS). Under the terms of @@ -116,18 +116,19 @@ def getExodusVersion(): """ version_major = -1 version_minor = -1 - for line in open(ACCESS + "/include/exodusII.h"): - fields = line.split() - if (len(fields) == 3 and - fields[0] == '#define' and - fields[1] == 'EXODUS_VERSION_MAJOR'): - version_major = int(fields[2]) - if (len(fields) == 3 and - fields[0] == '#define' and - fields[1] == 'EXODUS_VERSION_MINOR'): - version_minor = int(fields[2]) - if (version_major > 0 and version_minor >= 0): - return 100 * version_major + version_minor + with open(ACCESS + "/include/exodusII.h") as header_file: + for line in header_file: + fields = line.split() + if (len(fields) == 3 and + fields[0] == '#define' and + fields[1] == 'EXODUS_VERSION_MAJOR'): + version_major = int(fields[2]) + if (len(fields) == 3 and + fields[0] == '#define' and + fields[1] == 'EXODUS_VERSION_MINOR'): + version_minor = int(fields[2]) + if (version_major > 0 and version_minor >= 0): + return 100 * version_major + version_minor return 0 @@ -2189,7 +2190,7 @@ def get_variable_values(self, objType, entityId, name, step): """ names = self.get_variable_names(objType) var_id = names.index(name) + 1 - numVals = 0; + numVals = 0 if objType == 'EX_NODAL': numVals = self.num_nodes() elif objType == 'EX_ELEM_BLOCK': @@ -2232,7 +2233,7 @@ def put_variable_values(self, objType, entityId, name, step, values): """ names = self.get_variable_names(objType) var_id = names.index(name) + 1 - numVals = 0; + numVals = 0 if objType == 'EX_NODAL': numVals = self.num_nodes() elif objType == 'EX_ELEM_BLOCK': @@ -5232,7 +5233,7 @@ def __ex_get_attributes(self, objType, objId): attributes[att[i].name.decode('utf8')] = tmp_att - return attributes; + return attributes # -------------------------------------------------------------------- diff --git a/packages/seacas/scripts/tests/test_exodus3.py b/packages/seacas/scripts/tests/test_exodus3.py index 0fa1f7005ca3..1fef6164beac 100644 --- a/packages/seacas/scripts/tests/test_exodus3.py +++ b/packages/seacas/scripts/tests/test_exodus3.py @@ -12,13 +12,14 @@ import os import tempfile import ctypes +from contextlib import contextmanager path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) sys.path.append(os.path.join(path, "lib")) import exodus as exo -class MyTestCase(unittest.TestCase): +class TestAssemblies(unittest.TestCase): def setUp(self): exofile = exo.exodus("test-assembly.exo", mode='r') @@ -86,7 +87,7 @@ def test_get_assemblies(self): [100, 200, 300, 400], [100, 222]] x.entity_list = entity_lists[i] - self.maxDiff=None + self.maxDiff = None self.assertEqual(str(expected), str(assemblies)) def test_put_assembly(self): @@ -145,13 +146,11 @@ def test_get_reduction_variable_values_assembly(self): def test_get_reduction_variable_values_assembly_no_values(self): temp_exofile = exo.exodus("test-assembly.exo", mode='r') - times = temp_exofile.get_times() assembly_ids = temp_exofile.get_ids("EX_ASSEMBLY") assemblies = [temp_exofile.get_assembly(assembly) for assembly in assembly_ids] values = temp_exofile.get_reduction_variable_values('EX_ASSEMBLY', assemblies[5].id, 1) self.assertListEqual([0.00, 0.00, 0.00, 0.00], list(values)) - def test_put_assemblies(self): new = exo.assembly(name='Unit_test', type='EX_ASSEMBLY', id=444) new.entity_list = [100, 222] @@ -165,7 +164,6 @@ def test_put_assemblies(self): self.assertEqual(new.id, assemblies[6].id) self.assertEqual(new.entity_list, assemblies[6].entity_list) - def test_put_assemblies_multiple_assemblies(self): new = exo.assembly(name='Unit_test1', type='EX_ASSEMBLY', id=444) new.entity_list = [100, 222] @@ -187,5 +185,43 @@ def test_put_assemblies_multiple_assemblies(self): self.assertEqual(new2.entity_list, assemblies[7].entity_list) +class TestExodusUtilities(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.tempdir.cleanup() + + def test_basename(self): + self.assertEqual("test", exo.basename("test.e")) + self.assertEqual("fake/path/to/test", exo.basename("fake/path/to/test.e")) + + def test_getExodusVersion(self): + include_path = os.path.join(self.tempdir.name, "include") + os.makedirs(include_path) + with open(os.path.join(include_path, "exodusII.h"), 'w') as fptr: + fptr.write("#define EXODUS_VERSION_MAJOR 1\n") + fptr.write("#define EXODUS_VERSION_MINOR 22\n") + with swap_ACCESS_value(self.tempdir.name): + self.assertEqual(122, exo.getExodusVersion()) + + def test_getExodusVersion_not_found(self): + include_path = os.path.join(self.tempdir.name, "include") + os.makedirs(include_path) + with open(os.path.join(include_path, "exodusII.h"), 'w') as fptr: + fptr.write("#define NOT_EXODUS_VERSION 1\n") + fptr.write("#define ALSO_NOT_EXODUS_VERSION 22\n") + with swap_ACCESS_value(self.tempdir.name): + self.assertEqual(0, exo.getExodusVersion()) + + +@contextmanager +def swap_ACCESS_value(new_access_value): + old_value = exo.ACCESS + exo.ACCESS = new_access_value + yield + exo.ACCESS = old_value + + if __name__ == '__main__': unittest.main() From afb7217820647f9d74f0582d5b8842543d9a0c04 Mon Sep 17 00:00:00 2001 From: Alan Williams Date: Mon, 30 Aug 2021 17:23:53 -0600 Subject: [PATCH 44/70] STK: Snapshot 08-30-21 17:23 --- packages/stk/hip_porting/build_env_hip.sh | 20 +++ packages/stk/hip_porting/hip_kokkos.patch | 13 ++ packages/stk/hip_porting/hip_seacas.patch | 12 ++ packages/stk/hip_porting/mpiexec | 4 + packages/stk/hip_porting/run_cmake_stk_hip.sh | 60 +++++++ .../stk_coupling/stk_coupling/Constants.hpp | 1 + .../stk_io/parameterTestUtils.hpp | 42 ++--- .../stk_doc_tests/stk_io/usingHeartbeat.cpp | 2 +- .../usingHeartbeatCSVChangePrecision.cpp | 12 +- .../usingHeartbeatOverrideSeparator.cpp | 8 +- .../stk_io/usingHeartbeatSpyhisFormat.cpp | 8 +- .../stk/stk_doc_tests/stk_io/usingHistory.cpp | 10 +- .../writingAndReadingGlobalParameters.cpp | 12 +- .../writingAndReadingGlobalParametersAuto.cpp | 13 +- .../stk_mesh/howToUseNgpFieldAsyncCopy.cpp | 2 +- .../stk_topology/element_topologies.cpp | 6 +- .../stk_doc_tests/stk_topology/equivalent.cpp | 5 +- .../stk_topology/how_to_use_stk_topology.cpp | 6 +- .../map_stk_topologies_to_ranks.cpp | 8 +- .../runtime_vs_compiletime_topology.cpp | 4 +- .../stk_topology/side_node_ordinals.cpp | 5 +- .../understanding_various_topologies.cpp | 18 +- .../stk/stk_doc_tests/stk_util/parameters.cpp | 59 ++----- packages/stk/stk_expreval/Jamfile | 1 + packages/stk/stk_io/example/io_mesh.cpp | 6 +- packages/stk/stk_io/stk_io/Heartbeat.cpp | 91 +++++++++- packages/stk/stk_io/stk_io/Heartbeat.hpp | 45 ++++- packages/stk/stk_io/stk_io/IOHelpers.cpp | 14 ++ packages/stk/stk_io/stk_io/IOHelpers.hpp | 8 + packages/stk/stk_io/stk_io/OutputFile.cpp | 39 ++++- packages/stk/stk_io/stk_io/OutputFile.hpp | 25 ++- packages/stk/stk_io/stk_io/StkIoUtils.hpp | 4 + .../stk/stk_io/stk_io/StkMeshIoBroker.cpp | 33 +++- .../stk/stk_io/stk_io/StkMeshIoBroker.hpp | 161 ++++++++++++++---- packages/stk/stk_mesh/Jamfile | 1 + .../stk/stk_mesh/stk_mesh/base/BulkData.cpp | 67 +++++--- .../stk/stk_mesh/stk_mesh/base/BulkData.hpp | 2 +- .../stk_mesh/stk_mesh/base/CellTopology.hpp | 13 -- .../stk_mesh/stk_mesh/base/DeviceField.hpp | 17 +- .../stk/stk_mesh/stk_mesh/base/DeviceMesh.cpp | 2 +- .../stk/stk_mesh/stk_mesh/base/DeviceMesh.hpp | 10 +- .../stk_mesh/base/FieldSyncDebugging.hpp | 8 +- .../stk_mesh/base/GetNgpExecutionSpace.hpp | 6 +- .../stk_mesh/stk_mesh/base/GetNgpField.hpp | 2 +- .../stk/stk_mesh/stk_mesh/base/HostField.hpp | 8 +- .../stk/stk_mesh/stk_mesh/base/HostMesh.hpp | 4 +- .../stk/stk_mesh/stk_mesh/base/MetaData.cpp | 15 +- .../stk/stk_mesh/stk_mesh/base/MetaData.hpp | 1 - .../stk/stk_mesh/stk_mesh/base/NgpAtomics.hpp | 2 +- .../stk_mesh/stk_mesh/base/NgpDynamicMesh.hpp | 38 +++-- .../stk_mesh/stk_mesh/base/NgpFieldBase.hpp | 8 +- .../stk_mesh/base/NgpFieldParallel.hpp | 4 +- .../stk_mesh/base/NgpFieldSyncDebugger.hpp | 4 +- .../stk_mesh/base/NgpForEachEntity.hpp | 4 +- .../stk/stk_mesh/stk_mesh/base/NgpMesh.hpp | 2 +- .../stk_mesh/base/NgpParallelComm.hpp | 8 +- .../stk_mesh/stk_mesh/base/NgpReductions.hpp | 4 +- .../stk/stk_mesh/stk_mesh/base/NgpSpaces.hpp | 46 +---- .../stk/stk_mesh/stk_mesh/base/NgpTypes.hpp | 32 ++-- .../stk/stk_mesh/stk_mesh/base/NgpUtils.hpp | 12 +- .../stk/stk_mesh/stk_mesh/base/Stencils.hpp | 25 --- .../stk_mesh/baseImpl/MeshImplUtils.cpp | 51 ++++++ .../stk_mesh/baseImpl/MeshImplUtils.hpp | 7 + .../stk_mesh/baseImpl/NgpFieldAux.hpp | 11 +- .../stk_mesh/baseImpl/check_comm_list.cpp | 6 +- .../stk_ngp_test/GlobalReporter.cpp | 2 +- .../stk_ngp_test/NgpTestDeviceMacros.hpp | 2 +- .../stk_ngp_test/stk_ngp_test/ngp_test.hpp | 15 +- .../stk_mesh/NgpFieldAsync.cpp | 18 +- .../stk_mesh/calculate_centroid.hpp | 2 +- .../stk_topology/stk_topology/topology.cpp | 9 +- .../stk_topology/stk_topology/topology.hpp | 4 +- .../stk_topology/topology_decl.hpp | 3 +- .../stk_topology/topology_defn.hpp | 1 + .../topology_detail/topology_data.cpp | 2 +- .../topology_detail/topology_data.hpp | 3 +- .../stk_topology/topology_type.hpp | 1 + .../stk_topology/topology_utils.hpp | 1 + ...geEntityOwnerKeepUnaffectedCustomGhost.cpp | 152 +++++++++++++++++ .../UnitTestRemoveNeededRecvGhost.cpp | 143 ++++++++++++++++ .../ngp/KokkosCentroidCalculation.hpp | 8 +- .../stk_mesh/ngp/KokkosNodalVolume.cpp | 6 +- .../stk_mesh/ngp/NgpFieldAsyncTest.cpp | 2 +- .../stk_mesh/ngp/NgpMeshTest.cpp | 4 +- .../stk_mesh/ngp/NgpParallelCommTest.cpp | 6 +- .../stk_mesh/ngp/NgpUnitTestUtils.hpp | 6 +- .../stk_unit_tests/stk_mesh/ngp/howToNgp.cpp | 28 +-- .../stk_mesh/ngp/howToNgpMeshMod.cpp | 4 +- .../stk_mesh/ngp/implNgpMeshMod.cpp | 4 +- .../stk_mesh/ngp/ngpFieldTest.cpp | 2 +- .../stk_mesh/ngp/ngpMultiStateFieldTests.cpp | 6 +- .../stk_topology/topology_test_utils.hpp | 6 +- .../stk_topology/unit_test_beam.cpp | 10 +- .../stk_topology/unit_test_hex.cpp | 14 +- .../stk_topology/unit_test_invalid.cpp | 8 +- .../stk_topology/unit_test_is_topology.cpp | 4 +- .../stk_topology/unit_test_line.cpp | 10 +- .../stk_topology/unit_test_node.cpp | 10 +- .../stk_topology/unit_test_particle.cpp | 10 +- .../stk_topology/unit_test_pyramid.cpp | 10 +- .../stk_topology/unit_test_quad.cpp | 12 +- .../stk_topology/unit_test_shell_line.cpp | 10 +- .../stk_topology/unit_test_spring.cpp | 10 +- .../stk_topology/unit_test_topology.cpp | 15 +- .../stk_topology/unit_test_triangle.cpp | 10 +- .../unit_test_validate_topology_data.cpp | 11 +- .../stk_topology/unit_test_wedge.cpp | 12 +- .../stk_util/util/UnitTestParameterList.cpp | 98 +++++++++++ .../stk_util/util/UnitTestThrowMacros.cpp | 14 +- packages/stk/stk_util/Jamfile | 18 +- packages/stk/stk_util/stk_util/CMakeLists.txt | 1 + .../stk/stk_util/stk_util/ngp/CMakeLists.txt | 57 +++++++ .../stk/stk_util/stk_util/ngp/NgpSpaces.hpp | 99 +++++++++++ .../stk_util/parallel/ParallelComm.cpp | 68 -------- .../stk_util/parallel/ParallelComm.hpp | 5 - .../stk_util/registry/ProductRegistry.cpp | 2 +- .../stk/stk_util/stk_util/stk_kokkos_macros.h | 2 +- .../stk_util/stk_util/util/ParameterList.hpp | 21 ++- .../stk_util/stk_util/util/ReportHandler.hpp | 29 ++++ .../stk_util/stk_util/util/StkNgpVector.hpp | 4 +- .../stk_util/stk_util/util/StridedArray.hpp | 2 +- 121 files changed, 1587 insertions(+), 606 deletions(-) create mode 100755 packages/stk/hip_porting/build_env_hip.sh create mode 100644 packages/stk/hip_porting/hip_kokkos.patch create mode 100644 packages/stk/hip_porting/hip_seacas.patch create mode 100755 packages/stk/hip_porting/mpiexec create mode 100755 packages/stk/hip_porting/run_cmake_stk_hip.sh create mode 100644 packages/stk/stk_unit_tests/stk_mesh/custom_ghosting/UnitTestChangeEntityOwnerKeepUnaffectedCustomGhost.cpp create mode 100644 packages/stk/stk_unit_tests/stk_mesh/custom_ghosting/UnitTestRemoveNeededRecvGhost.cpp create mode 100644 packages/stk/stk_unit_tests/stk_util/util/UnitTestParameterList.cpp create mode 100644 packages/stk/stk_util/stk_util/ngp/CMakeLists.txt create mode 100644 packages/stk/stk_util/stk_util/ngp/NgpSpaces.hpp diff --git a/packages/stk/hip_porting/build_env_hip.sh b/packages/stk/hip_porting/build_env_hip.sh new file mode 100755 index 000000000000..e99168bb92e2 --- /dev/null +++ b/packages/stk/hip_porting/build_env_hip.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +module load craype-accel-amd-gfx908 +module load PrgEnv-cray +module load rocm +module load boost +module load cray-hdf5 +module load cray-netcdf +module load cray-parallel-netcdf + +export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0 +export MPICH_GPU_SUPPORT_ENABLED=1 +export PATH=$HOME/Trilinos/builds:$PATH + +cd .. +git checkout -- packages +git apply builds/hip_kokkos.patch +git apply builds/hip_seacas.patch +git apply builds/hip_stk.patch +cd - diff --git a/packages/stk/hip_porting/hip_kokkos.patch b/packages/stk/hip_porting/hip_kokkos.patch new file mode 100644 index 000000000000..75d28c8d6010 --- /dev/null +++ b/packages/stk/hip_porting/hip_kokkos.patch @@ -0,0 +1,13 @@ +diff --git a/packages/kokkos/cmake/kokkos_tpls.cmake b/packages/kokkos/cmake/kokkos_tpls.cmake +index d8d044c9d75..3bd4ec7f3ed 100644 +--- a/packages/kokkos/cmake/kokkos_tpls.cmake ++++ b/packages/kokkos/cmake/kokkos_tpls.cmake +@@ -44,7 +44,7 @@ IF(KOKKOS_ENABLE_HIP AND NOT KOKKOS_CXX_COMPILER_ID STREQUAL HIPCC) + ELSE() + SET(ROCM_DEFAULT OFF) + ENDIF() +-KOKKOS_TPL_OPTION(ROCM ${ROCM_DEFAULT}) ++KOKKOS_TPL_OPTION(ROCM ${ROCM_DEFAULT} TRIBITS ROCM) + + IF (WIN32) + SET(LIBDL_DEFAULT Off) diff --git a/packages/stk/hip_porting/hip_seacas.patch b/packages/stk/hip_porting/hip_seacas.patch new file mode 100644 index 000000000000..f56b305c8126 --- /dev/null +++ b/packages/stk/hip_porting/hip_seacas.patch @@ -0,0 +1,12 @@ +diff --git a/packages/seacas/libraries/ioss/src/CMakeLists.txt b/packages/seacas/libraries/ioss/src/CMakeLists.txt +index dbef2ebb4d7..33dd362505d 100644 +--- a/packages/seacas/libraries/ioss/src/CMakeLists.txt ++++ b/packages/seacas/libraries/ioss/src/CMakeLists.txt +@@ -1,6 +1,7 @@ + IF (${PROJECT_NAME}_ENABLE_Kokkos) + SET(SEACAS_HAVE_KOKKOS ON) + ENDIF() ++SET(SEACAS_HAVE_KOKKOS OFF) + + IF (${PROJECT_NAME}_ENABLE_Pamgen) + SET(SEACAS_HAVE_PAMGEN ON) diff --git a/packages/stk/hip_porting/mpiexec b/packages/stk/hip_porting/mpiexec new file mode 100755 index 000000000000..80bd892a8fd1 --- /dev/null +++ b/packages/stk/hip_porting/mpiexec @@ -0,0 +1,4 @@ +#!/bin/sh + +ARGS=$(echo $@ | sed s/-np/-n/) +srun $ARGS diff --git a/packages/stk/hip_porting/run_cmake_stk_hip.sh b/packages/stk/hip_porting/run_cmake_stk_hip.sh new file mode 100755 index 000000000000..333262afe123 --- /dev/null +++ b/packages/stk/hip_porting/run_cmake_stk_hip.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +rm -rf hip +mkdir hip +cd hip + +trilinos_src_dir=${TRILINOS_DIR:-${PWD}/../..} +build_type=${CMAKE_BUILD_TYPE:-release} +fortran_macro=${FORTRAN_MACRO:-FORTRAN_ONE_UNDERSCORE} +cmake_cxx_flags=${CMAKE_CXX_FLAGS} +cmake_exe_linker_flags=${CMAKE_EXE_LINKER_FLAGS} + +printf "\nTRILINOS_DIR=${trilinos_src_dir}\n"; +printf "CMAKE_BUILD_TYPE=${build_type}\n"; +printf "CMAKE_EXE_LINKER_FLAGS=${cmake_exe_linker_flags}\n"; +printf "CMAKE_CXX_FLAGS=${cmake_cxx_flags}\n"; +printf "FORTRAN_MACRO=${fortran_macro}\n"; +printf "\nTo change these vars, set as env vars or pass to this script like 'VAR=value run_cmake_stk'\n\n"; + +HIP_FLAGS="-I${MPICH_DIR}/include -L${MPICH_DIR}/lib -lmpi ${PE_MPICH_GTL_DIR_amd_gfx908} -lmpi_gtl_hsa" + +cmake \ +-DCMAKE_BUILD_TYPE=${build_type^^} \ +-DCMAKE_C_COMPILER=hipcc \ +-DCMAKE_C_FLAGS:STRING="${HIP_FLAGS}" \ +-DCMAKE_CXX_COMPILER=hipcc \ +-DCMAKE_CXX_STANDARD="14" \ +-DCMAKE_CXX_FLAGS:STRING="-DNOT_HAVE_STK_SEACASAPREPRO_LIB -DSTK_NO_BOOST_STACKTRACE -D${fortran_macro} ${cmake_cxx_flags} ${HIP_FLAGS} -Wno-unused-command-line-argument -Werror=dangling-else" \ +-DCMAKE_EXE_LINKER_FLAGS="${cmake_exe_linker_flags}" \ +-DTPL_ENABLE_MPI=ON \ +-DMPI_BASE_DIR:PATH=${MPICH_DIR} \ +-DMPI_EXEC=mpiexec \ +-DTPL_ENABLE_BLAS=ON \ +-DTPL_BLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_cray_mpi.a \ +-DTPL_ENABLE_LAPACK=ON \ +-DTPL_LAPACK_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_cray_mpi.a \ +-DTPL_ENABLE_Boost=ON \ +-DTPL_ENABLE_HDF5:BOOL=ON \ +-DHDF5_INCLUDE_DIRS=${CRAY_HDF5_PREFIX}/include \ +-DHDF5_LIBRARY_DIRS=${CRAY_HDF5_PREFIX}/lib \ +-DTPL_ENABLE_Netcdf:BOOL=ON \ +-DNetcdf_INCLUDE_DIRS=${CRAY_NETCDF_PREFIX}/include \ +-DNetcdf_LIBRARY_DIRS=${CRAY_NETCDF_PREFIX}/lib \ +-DTPL_ENABLE_Pnetcdf:BOOL=ON \ +-DPnetcdf_INCLUDE_DIRS=${CRAY_PARALLEL_NETCDF_PREFIX}/include \ +-DPnetcdf_LIBRARY_DIRS=${CRAY_PARALLEL_NETCDF_PREFIX}/lib \ +-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON \ +-DTrilinos_ENABLE_TESTS:BOOL=OFF \ +-DTrilinos_ENABLE_ALL_OPTIONAL_PACKAGES=OFF \ +-DTrilinos_ALLOW_NO_PACKAGES:BOOL=OFF \ +-DTrilinos_ASSERT_MISSING_PACKAGES=OFF \ +-DTrilinos_ENABLE_Fortran:BOOL=OFF \ +-DTrilinos_ENABLE_Kokkos:BOOL=ON \ +-DKokkos_ENABLE_HIP:BOOL=ON \ +-DKokkos_ENABLE_ROCM:BOOL=ON \ +-DKokkos_ARCH_VEGA908:BOOL=ON \ +-DTrilinos_ENABLE_STK:BOOL=ON \ +-DTrilinos_ENABLE_STKBalance:BOOL=OFF \ +-DSTK_ENABLE_TESTS:BOOL=OFF \ +${trilinos_src_dir}/ diff --git a/packages/stk/stk_coupling/stk_coupling/Constants.hpp b/packages/stk/stk_coupling/stk_coupling/Constants.hpp index 6bcc77bd3956..9f1cdd561f4e 100644 --- a/packages/stk/stk_coupling/stk_coupling/Constants.hpp +++ b/packages/stk/stk_coupling/stk_coupling/Constants.hpp @@ -31,6 +31,7 @@ static const std::string CurrentTime = "Current Time"; static const std::string TimeStep = "Time Step"; static const std::string FinalTime = "Final Time"; static const std::string IsFinished = "Is Finished"; +static const std::string SuccessFlag = "Is Successful"; } } diff --git a/packages/stk/stk_doc_tests/stk_io/parameterTestUtils.hpp b/packages/stk/stk_doc_tests/stk_io/parameterTestUtils.hpp index dbda29312fbe..f2b7c3af1c36 100644 --- a/packages/stk/stk_doc_tests/stk_io/parameterTestUtils.hpp +++ b/packages/stk/stk_doc_tests/stk_io/parameterTestUtils.hpp @@ -43,40 +43,40 @@ inline void validate_parameters_equal_value(const stk::util::Parameter ¶mete { ASSERT_EQ(parameter.type, gold_parameter.type); switch(parameter.type) - { + { case stk::util::ParameterType::INTEGER: { - ASSERT_EQ(STK_ANY_NAMESPACE::any_cast(parameter.value), STK_ANY_NAMESPACE::any_cast(gold_parameter.value)); - break; + ASSERT_EQ(parameter.get_value(), gold_parameter.get_value()); + break; } case stk::util::ParameterType::DOUBLE: { - ASSERT_EQ(STK_ANY_NAMESPACE::any_cast(parameter.value), STK_ANY_NAMESPACE::any_cast(gold_parameter.value)); - break; + ASSERT_EQ(parameter.get_value(), gold_parameter.get_value()); + break; } case stk::util::ParameterType::DOUBLEVECTOR: { - std::vector vec = STK_ANY_NAMESPACE::any_cast >(parameter.value); - std::vector gvec = STK_ANY_NAMESPACE::any_cast >(gold_parameter.value); - ASSERT_EQ(vec.size(), gvec.size()); - for (size_t j = 0; j < vec.size(); ++j) { - ASSERT_EQ(vec[j], gvec[j]); - } - break; + std::vector vec = parameter.get_value >(); + std::vector gvec = gold_parameter.get_value >(); + ASSERT_EQ(vec.size(), gvec.size()); + for (size_t j = 0; j < vec.size(); ++j) { + ASSERT_EQ(vec[j], gvec[j]); + } + break; } case stk::util::ParameterType::INTEGERVECTOR: { - std::vector vec = STK_ANY_NAMESPACE::any_cast >(parameter.value); - std::vector gvec = STK_ANY_NAMESPACE::any_cast >(gold_parameter.value); - ASSERT_EQ(vec.size(), gvec.size()); - for (size_t j = 0; j < vec.size(); ++j) { - ASSERT_EQ(vec[j], gvec[j]); - } - break; + std::vector vec = parameter.get_value >(); + std::vector gvec = gold_parameter.get_value >(); + ASSERT_EQ(vec.size(), gvec.size()); + for (size_t j = 0; j < vec.size(); ++j) { + ASSERT_EQ(vec[j], gvec[j]); + } + break; } default: - ASSERT_EQ(1,0) << "Invalid type found in validate_parameters_equal_value"; - } + ASSERT_EQ(1,0) << "Invalid type found in validate_parameters_equal_value"; + } } #endif //include-guard diff --git a/packages/stk/stk_doc_tests/stk_io/usingHeartbeat.cpp b/packages/stk/stk_doc_tests/stk_io/usingHeartbeat.cpp index 20c8ca9bf5c2..daa3e741553f 100644 --- a/packages/stk/stk_doc_tests/stk_io/usingHeartbeat.cpp +++ b/packages/stk/stk_doc_tests/stk_io/usingHeartbeat.cpp @@ -100,7 +100,7 @@ namespace //+ Tell heartbeat which variables to output at each step... //+ NOTE: The address of the value to be output is needed since the //+ value is output in the process_heartbeat_output call. - stkIo.add_heartbeat_global(hb,paramName, ¶m.value, param.type); + stkIo.add_heartbeat_global(hb,paramName, param); }/*@\label{io:hb:end_add}*/ // Application's "Execution Loop" diff --git a/packages/stk/stk_doc_tests/stk_io/usingHeartbeatCSVChangePrecision.cpp b/packages/stk/stk_doc_tests/stk_io/usingHeartbeatCSVChangePrecision.cpp index dafeb30c1eb2..9bfb8485c20b 100644 --- a/packages/stk/stk_doc_tests/stk_io/usingHeartbeatCSVChangePrecision.cpp +++ b/packages/stk/stk_doc_tests/stk_io/usingHeartbeatCSVChangePrecision.cpp @@ -96,19 +96,17 @@ namespace hb_props.add(Ioss::Property("PRECISION", 10)); //+ Define the heartbeat output and the format (CSV) - size_t hb = - stkIo.add_heartbeat_output(file_name, stk::io::CSV, hb_props); + size_t hb = stkIo.add_heartbeat_output(file_name, stk::io::CSV, hb_props); //-END stk::util::ParameterMapType::const_iterator i = params.begin(); stk::util::ParameterMapType::const_iterator iend = params.end(); for (; i != iend; ++i) { - const std::string parameterName = (*i).first; - stk::util::Parameter ¶meter = params.get_param(parameterName); + const std::string parameterName = (*i).first; + stk::util::Parameter ¶meter = params.get_param(parameterName); - //+ Tell heartbeat which variables to output at each step... - stkIo.add_heartbeat_global(hb, parameterName, - ¶meter.value, parameter.type); + //+ Tell heartbeat which variables to output at each step... + stkIo.add_heartbeat_global(hb, parameterName, parameter); } int timestep_count = 1; diff --git a/packages/stk/stk_doc_tests/stk_io/usingHeartbeatOverrideSeparator.cpp b/packages/stk/stk_doc_tests/stk_io/usingHeartbeatOverrideSeparator.cpp index 3bb397a15c70..c61d672f8264 100644 --- a/packages/stk/stk_doc_tests/stk_io/usingHeartbeatOverrideSeparator.cpp +++ b/packages/stk/stk_doc_tests/stk_io/usingHeartbeatOverrideSeparator.cpp @@ -100,11 +100,11 @@ namespace stk::util::ParameterMapType::const_iterator i = parameters.begin(); stk::util::ParameterMapType::const_iterator iend = parameters.end(); for (; i != iend; ++i) { - const std::string parameterName = (*i).first; - stk::util::Parameter ¶meter = parameters.get_param(parameterName); + const std::string parameterName = (*i).first; + stk::util::Parameter ¶meter = parameters.get_param(parameterName); - // Tell heartbeat database which global variables should be output at each step... - stkIo.add_heartbeat_global(hb, parameterName, ¶meter.value, parameter.type); + // Tell heartbeat database which global variables should be output at each step... + stkIo.add_heartbeat_global(hb, parameterName, parameter); } // Now output the global variables... diff --git a/packages/stk/stk_doc_tests/stk_io/usingHeartbeatSpyhisFormat.cpp b/packages/stk/stk_doc_tests/stk_io/usingHeartbeatSpyhisFormat.cpp index 040c17c45a91..bf5f28bb6bb7 100644 --- a/packages/stk/stk_doc_tests/stk_io/usingHeartbeatSpyhisFormat.cpp +++ b/packages/stk/stk_doc_tests/stk_io/usingHeartbeatSpyhisFormat.cpp @@ -92,11 +92,11 @@ namespace stk::util::ParameterMapType::const_iterator i = parameters.begin(); stk::util::ParameterMapType::const_iterator iend = parameters.end(); for (; i != iend; ++i) { - const std::string parameterName = (*i).first; - stk::util::Parameter ¶meter = parameters.get_param(parameterName); + const std::string parameterName = (*i).first; + stk::util::Parameter ¶meter = parameters.get_param(parameterName); - // Tell heartbeat database which global variables should be output at each step... - stkIo.add_heartbeat_global(heartbeat_index, parameterName, ¶meter.value, parameter.type); + // Tell heartbeat database which global variables should be output at each step... + stkIo.add_heartbeat_global(heartbeat_index, parameterName, parameter); } // Now output the global variables... diff --git a/packages/stk/stk_doc_tests/stk_io/usingHistory.cpp b/packages/stk/stk_doc_tests/stk_io/usingHistory.cpp index fa2e4625563e..6e9bd28aec68 100644 --- a/packages/stk/stk_doc_tests/stk_io/usingHistory.cpp +++ b/packages/stk/stk_doc_tests/stk_io/usingHistory.cpp @@ -111,11 +111,11 @@ TEST(StkMeshIoBrokerHowTo, writeHistory) stk::util::ParameterMapType::const_iterator i = params.begin(); stk::util::ParameterMapType::const_iterator iend = params.end(); for (; i != iend; ++i) { - const std::string parameterName = (*i).first; - stk::util::Parameter ¶meter = params.get_param(parameterName); + const std::string parameterName = (*i).first; + stk::util::Parameter ¶meter = params.get_param(parameterName); - // Tell history database which global variables should be output at each step... - stkIo.add_heartbeat_global(hb, parameterName, ¶meter.value, parameter.type); + // Tell history database which global variables should be output at each step... + stkIo.add_heartbeat_global(hb, parameterName, parameter); } // Now output the global variables... @@ -344,7 +344,7 @@ TEST(StkMeshIoBrokerHowTo, writeEmptyHistory) stk::util::Parameter ¶meter = params.get_param(parameterName); // Tell history database which global variables should be output at each step... - stkIo.add_heartbeat_global(hb, parameterName, ¶meter.value, parameter.type); + stkIo.add_heartbeat_global(hb, parameterName, parameter); } stkIo.end_define_transient_for_heartbeat(hb); diff --git a/packages/stk/stk_doc_tests/stk_io/writingAndReadingGlobalParameters.cpp b/packages/stk/stk_doc_tests/stk_io/writingAndReadingGlobalParameters.cpp index ce8e9c2684da..d8cfd4dd1aa9 100644 --- a/packages/stk/stk_doc_tests/stk_io/writingAndReadingGlobalParameters.cpp +++ b/packages/stk/stk_doc_tests/stk_io/writingAndReadingGlobalParameters.cpp @@ -82,9 +82,9 @@ namespace stk::util::ParameterMapType::const_iterator i = params.begin(); stk::util::ParameterMapType::const_iterator ie = params.end(); for (; i != ie; ++i) { - const std::string parameterName = (*i).first; - stk::util::Parameter ¶m = params.get_param(parameterName); - stkIo.add_global(idx, parameterName, param.value, param.type); + const std::string parameterName = (*i).first; + stk::util::Parameter ¶m = params.get_param(parameterName); + stkIo.add_global(idx, parameterName, param); } stkIo.begin_output_step(idx, 0.0);/*@\label{io:global:write_begin}*/ @@ -112,9 +112,9 @@ namespace stk::util::ParameterMapType::const_iterator i = params.begin(); stk::util::ParameterMapType::const_iterator ie = params.end(); for (; i != ie; ++i) { - const std::string parameterName = (*i).first; - stk::util::Parameter ¶m = params.get_param(parameterName); - stkIo.get_global(parameterName, param.value, param.type); + const std::string parameterName = (*i).first; + stk::util::Parameter ¶m = params.get_param(parameterName); + stkIo.get_global(parameterName, param); } // ============================================================ diff --git a/packages/stk/stk_doc_tests/stk_io/writingAndReadingGlobalParametersAuto.cpp b/packages/stk/stk_doc_tests/stk_io/writingAndReadingGlobalParametersAuto.cpp index 5ba910306310..c1cbc46f8ce5 100644 --- a/packages/stk/stk_doc_tests/stk_io/writingAndReadingGlobalParametersAuto.cpp +++ b/packages/stk/stk_doc_tests/stk_io/writingAndReadingGlobalParametersAuto.cpp @@ -88,16 +88,15 @@ namespace stk::util::ParameterMapType::const_iterator i = params.begin(); stk::util::ParameterMapType::const_iterator iend = params.end(); for (; i != iend; ++i) { - const std::string paramName = (*i).first; - //+ NOTE: Need a reference to the parameter. - stk::util::Parameter ¶m = params.get_param(paramName); - //+ NOTE: Calling add_global_ref, passing address of value - stkIo.add_global_ref(idx, paramName, ¶m.value, param.type);/*@\label{io:global:autoparam}*/ + const std::string paramName = (*i).first; + //+ NOTE: Need a reference to the parameter. + stk::util::Parameter ¶m = params.get_param(paramName); + stkIo.add_global_ref(idx, paramName, param); } //+ All writing of the values is handled automatically, //+ do not need to call write_global - stkIo.process_output_request(idx, 0.0);/*@\label{io:global:autowrite}*/ + stkIo.process_output_request(idx, 0.0); } // ... Reading is the same as in previous example //-END @@ -126,7 +125,7 @@ namespace stk::util::Parameter ¶m = params.get_param(paramName); stk::util::Parameter &gold_param = gold_params.get_param(paramName); - stkIo.get_global(paramName, param.value, param.type); + stkIo.get_global(paramName, param); validate_parameters_equal_value(param, gold_param); } diff --git a/packages/stk/stk_doc_tests/stk_mesh/howToUseNgpFieldAsyncCopy.cpp b/packages/stk/stk_doc_tests/stk_mesh/howToUseNgpFieldAsyncCopy.cpp index fc8ef13f0d77..f2883bcde44d 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/howToUseNgpFieldAsyncCopy.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/howToUseNgpFieldAsyncCopy.cpp @@ -38,7 +38,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/packages/stk/stk_doc_tests/stk_topology/element_topologies.cpp b/packages/stk/stk_doc_tests/stk_topology/element_topologies.cpp index 6c66c0b7ad49..8a68e3c5a6cd 100644 --- a/packages/stk/stk_doc_tests/stk_topology/element_topologies.cpp +++ b/packages/stk/stk_doc_tests/stk_topology/element_topologies.cpp @@ -32,9 +32,9 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_TRUE +#include "stk_topology/topology.hpp" // for topology, topology::FACE_RANK, topology::EDGE_RANK +#include // for vector namespace { diff --git a/packages/stk/stk_doc_tests/stk_topology/equivalent.cpp b/packages/stk/stk_doc_tests/stk_topology/equivalent.cpp index 450cd0dfc535..7f5df40f9e8f 100644 --- a/packages/stk/stk_doc_tests/stk_topology/equivalent.cpp +++ b/packages/stk/stk_doc_tests/stk_topology/equivalent.cpp @@ -32,9 +32,8 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include +#include "gtest/gtest.h" // for AssertionResult, Test, Message, TestPartResult, EXP... +#include "stk_topology/topology.hpp" // for topology, EquivalentPermutation, topology::HEX_8 namespace { diff --git a/packages/stk/stk_doc_tests/stk_topology/how_to_use_stk_topology.cpp b/packages/stk/stk_doc_tests/stk_topology/how_to_use_stk_topology.cpp index 18b882f7e115..bffe57843902 100644 --- a/packages/stk/stk_doc_tests/stk_topology/how_to_use_stk_topology.cpp +++ b/packages/stk/stk_doc_tests/stk_topology/how_to_use_stk_topology.cpp @@ -32,9 +32,9 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_EQ +#include "stk_topology/topology.hpp" // for topology, create_superelement_topology, topology::I... +#include // for cout #ifndef __IBMCPP__ diff --git a/packages/stk/stk_doc_tests/stk_topology/map_stk_topologies_to_ranks.cpp b/packages/stk/stk_doc_tests/stk_topology/map_stk_topologies_to_ranks.cpp index d853b754c294..b6936f18f96c 100644 --- a/packages/stk/stk_doc_tests/stk_topology/map_stk_topologies_to_ranks.cpp +++ b/packages/stk/stk_doc_tests/stk_topology/map_stk_topologies_to_ranks.cpp @@ -32,9 +32,11 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, Test, EXP... +#include "stk_topology/topology.hpp" // for topology, create_superelement_topology, topology::Q... +#include // for size_t +#include // for allocator_traits<>::value_type +#include // for vector namespace { diff --git a/packages/stk/stk_doc_tests/stk_topology/runtime_vs_compiletime_topology.cpp b/packages/stk/stk_doc_tests/stk_topology/runtime_vs_compiletime_topology.cpp index a33ccddb230d..72c508d3d271 100644 --- a/packages/stk/stk_doc_tests/stk_topology/runtime_vs_compiletime_topology.cpp +++ b/packages/stk/stk_doc_tests/stk_topology/runtime_vs_compiletime_topology.cpp @@ -32,8 +32,8 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include +#include "gtest/gtest.h" // for Test, EXPECT_EQ, Message, SuiteApiResolver, TestInf... +#include "stk_topology/topology.hpp" // for topology, topology::HEX_8, topology::topology_type namespace { //-BEGIN diff --git a/packages/stk/stk_doc_tests/stk_topology/side_node_ordinals.cpp b/packages/stk/stk_doc_tests/stk_topology/side_node_ordinals.cpp index 00162508da69..11a8e878cf1f 100644 --- a/packages/stk/stk_doc_tests/stk_topology/side_node_ordinals.cpp +++ b/packages/stk/stk_doc_tests/stk_topology/side_node_ordinals.cpp @@ -32,9 +32,8 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include +#include "gtest/gtest.h" // for Test, EXPECT_EQ, Message, TestPartResult, SuiteApiR... +#include "stk_topology/topology.hpp" // for topology, topology::HEX_8 #ifndef __IBMCPP__ diff --git a/packages/stk/stk_doc_tests/stk_topology/understanding_various_topologies.cpp b/packages/stk/stk_doc_tests/stk_topology/understanding_various_topologies.cpp index 3cda53e20b63..90b6586f8342 100644 --- a/packages/stk/stk_doc_tests/stk_topology/understanding_various_topologies.cpp +++ b/packages/stk/stk_doc_tests/stk_topology/understanding_various_topologies.cpp @@ -32,19 +32,17 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include +#include "stk_util/stk_config.h" #ifdef STK_HAVE_STKIO -#include -#include -#include -#include -// #include -#include -#include -#include - +#include "gtest/gtest.h" +#include "Ioss_ElementTopology.h" +#include "Ioss_Utils.h" +#include "init/Ionit_Initializer.h" +#include "stk_io/IossBridge.hpp" +#include "stk_mesh/base/MetaData.hpp" +#include "stk_topology/topology.hpp" #include namespace { diff --git a/packages/stk/stk_doc_tests/stk_util/parameters.cpp b/packages/stk/stk_doc_tests/stk_util/parameters.cpp index 4d1d96db4283..0aeb1fe451e2 100644 --- a/packages/stk/stk_doc_tests/stk_util/parameters.cpp +++ b/packages/stk/stk_doc_tests/stk_util/parameters.cpp @@ -34,8 +34,6 @@ #include "stk_util/stk_config.h" -#ifdef STK_HAVE_BOOST - #include "gtest/gtest.h" #include "stk_util/util/ParameterList.hpp" // for ParameterList, Type, etc #include // for size_t @@ -151,53 +149,28 @@ namespace { //+ Access a variable of unknown type... - //+ The parameter uses STK_ANY_NAMESPACE::any to store the actual value. stk::util::Parameter ¶m = params.get_param("Answer"); double value_as_double = 0.0; switch (param.type) { - case stk::util::ParameterType::DOUBLE: - value_as_double = STK_ANY_NAMESPACE::any_cast(param.value); - break; - case stk::util::ParameterType::FLOAT: - value_as_double = static_cast(STK_ANY_NAMESPACE::any_cast(param.value)); - break; - case stk::util::ParameterType::INTEGER: - value_as_double = static_cast(STK_ANY_NAMESPACE::any_cast(param.value)); - break; - case stk::util::ParameterType::INT64: - value_as_double = static_cast(STK_ANY_NAMESPACE::any_cast(param.value)); - break; - default: - std::cerr << "ERROR: I can not convert 'Answers' value to a double\n"; - break; + case stk::util::ParameterType::DOUBLE: + value_as_double = param.get_value(); + break; + case stk::util::ParameterType::FLOAT: + value_as_double = static_cast(param.get_value()); + break; + case stk::util::ParameterType::INTEGER: + value_as_double = static_cast(param.get_value()); + break; + case stk::util::ParameterType::INT64: + value_as_double = static_cast(param.get_value()); + break; + default: + std::cerr << "ERROR: I can not convert 'Answers' value to a double\n"; + break; } EXPECT_EQ(static_cast(new_answer), value_as_double); } - { - //+ Access a variable of unknown type without using STK_ANY_NAMESPACE::any_cast - stk::util::Parameter ¶m = params.get_param("Answer"); - double value_as_double = 0.0; - switch (param.type) { - case stk::util::ParameterType::DOUBLE: - value_as_double = params.get_value("Answer"); - break; - case stk::util::ParameterType::FLOAT: - value_as_double = static_cast(params.get_value("Answer")); - break; - case stk::util::ParameterType::INTEGER: - value_as_double = static_cast(params.get_value("Answer")); - break; - case stk::util::ParameterType::INT64: - value_as_double = static_cast(params.get_value("Answer")); - break; - default: - std::cerr << "ERROR: I can not convert 'Answers' value to a double\n"; - break; - } - EXPECT_EQ(static_cast(new_answer), value_as_double); - } - //ENDParametersAccessingValues //BEGINParametersErrors @@ -259,5 +232,3 @@ namespace } } -#endif //STK_HAVE_BOOST - diff --git a/packages/stk/stk_expreval/Jamfile b/packages/stk/stk_expreval/Jamfile index 568a1a118688..6009ee4c538a 100644 --- a/packages/stk/stk_expreval/Jamfile +++ b/packages/stk/stk_expreval/Jamfile @@ -154,6 +154,7 @@ lib stk_expreval ] /sierra/stk_util//stk_util_environment /sierra/stk_util//stk_util_util + /sierra/stk_util//stk_util_ngp /mpi//mpi : [ ifuserbuild diff --git a/packages/stk/stk_io/example/io_mesh.cpp b/packages/stk/stk_io/example/io_mesh.cpp index c930a3276ba3..d760b653ae20 100644 --- a/packages/stk/stk_io/example/io_mesh.cpp +++ b/packages/stk/stk_io/example/io_mesh.cpp @@ -207,8 +207,8 @@ class IoMeshDriver mesh_data.add_global(results_index, input_field.get_name(), input_field.raw_storage()->name(), input_field.get_type()); if (hb_type != stk::io::NONE) { - stk::util::Parameter ¶m = parameters.get_param(input_field.get_name()); - mesh_data.add_heartbeat_global(heart, input_field.get_name(), ¶m.value, param.type); + stk::util::Parameter ¶m = parameters.get_param(input_field.get_name()); + mesh_data.add_heartbeat_global(heart, input_field.get_name(), param); } } @@ -254,7 +254,7 @@ class IoMeshDriver for (; i != iend; ++i) { const std::string parameterName = (*i).first; stk::util::Parameter ¶meter = parameters.get_param(parameterName); - mesh_data.get_global(parameterName, parameter.value, parameter.type); + mesh_data.get_global(parameterName, parameter); } for (i=parameters.begin(); i != iend; ++i) { diff --git a/packages/stk/stk_io/stk_io/Heartbeat.cpp b/packages/stk/stk_io/stk_io/Heartbeat.cpp index f0c0a8df39eb..45c4499d98d1 100644 --- a/packages/stk/stk_io/stk_io/Heartbeat.cpp +++ b/packages/stk/stk_io/stk_io/Heartbeat.cpp @@ -193,7 +193,7 @@ bool impl::Heartbeat::has_global(const std::string &name) return m_region->field_exists(name); } -void impl::Heartbeat::define_global_ref(const std::string &name, +void impl::Heartbeat::internal_define_global_ref(const std::string &name, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type, int copies, @@ -211,7 +211,44 @@ void impl::Heartbeat::define_global_ref(const std::string &name, } } +void impl::Heartbeat::define_global_ref(const std::string &name, + const stk::util::Parameter ¶m, + int copies, + Ioss::Field::RoleType role) +{ + internal_define_global_ref(name, ¶m.value, param.type, copies, role); +} + void impl::Heartbeat::add_global_ref(const std::string &name, + const stk::util::Parameter ¶m, + int copies, + Ioss::Field::RoleType role) +{ + if (m_processor == 0) { + ThrowErrorMsgIf (m_currentStep != 0, + "At least one output step has been written to the history/heartbeat file. " + "Variables cannot be added anymore."); + + Ioss::State currentState = m_region->get_state(); + if(currentState != Ioss::STATE_DEFINE_TRANSIENT) { + m_region->begin_mode(Ioss::STATE_DEFINE_TRANSIENT); + } + + internal_define_global_ref(name, ¶m.value, param.type, copies, role); + } +} + +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 +STK_DEPRECATED void impl::Heartbeat::define_global_ref(const std::string &name, + const STK_ANY_NAMESPACE::any *value, + stk::util::ParameterType::Type type, + int copies, + Ioss::Field::RoleType role) +{ + internal_define_global_ref(name, value, type, copies, role); +} + +STK_DEPRECATED void impl::Heartbeat::add_global_ref(const std::string &name, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type, int copies, @@ -227,11 +264,12 @@ void impl::Heartbeat::add_global_ref(const std::string &name, m_region->begin_mode(Ioss::STATE_DEFINE_TRANSIENT); } - define_global_ref(name, value, type, copies, role); + internal_define_global_ref(name, value, type, copies, role); } } +#endif -void impl::Heartbeat::define_global_ref(const std::string &name, +void impl::Heartbeat::internal_define_global_ref(const std::string &name, const STK_ANY_NAMESPACE::any *value, const std::string &storage, Ioss::Field::BasicType dataType, @@ -253,7 +291,51 @@ void impl::Heartbeat::define_global_ref(const std::string &name, } } +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 +STK_DEPRECATED void impl::Heartbeat::define_global_ref(const std::string &name, + const STK_ANY_NAMESPACE::any *value, + const std::string &storage, + Ioss::Field::BasicType dataType, + int copies, + Ioss::Field::RoleType role) +{ + internal_define_global_ref(name, value, storage, dataType, copies, role); +} +#endif + +void impl::Heartbeat::define_global_ref(const std::string &name, + const stk::util::Parameter ¶m, + const std::string &storage, + Ioss::Field::BasicType dataType, + int copies, + Ioss::Field::RoleType role) +{ + internal_define_global_ref(name, ¶m.value, storage, dataType, copies, role); +} + void impl::Heartbeat::add_global_ref(const std::string &name, + const stk::util::Parameter ¶m, + const std::string &storage, + Ioss::Field::BasicType dataType, + int copies, + Ioss::Field::RoleType role) +{ + if (m_processor == 0) { + ThrowErrorMsgIf (m_currentStep != 0, + "At least one output step has been written to the history/heartbeat file. " + "Variables cannot be added anymore."); + + Ioss::State currentState = m_region->get_state(); + if(currentState != Ioss::STATE_DEFINE_TRANSIENT) { + m_region->begin_mode(Ioss::STATE_DEFINE_TRANSIENT); + } + + internal_define_global_ref(name, ¶m.value, storage, dataType, copies, role); + } +} + +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 +STK_DEPRECATED void impl::Heartbeat::add_global_ref(const std::string &name, const STK_ANY_NAMESPACE::any *value, const std::string &storage, Ioss::Field::BasicType dataType, @@ -270,9 +352,10 @@ void impl::Heartbeat::add_global_ref(const std::string &name, m_region->begin_mode(Ioss::STATE_DEFINE_TRANSIENT); } - define_global_ref(name, value, storage, dataType, copies, role); + internal_define_global_ref(name, value, storage, dataType, copies, role); } } +#endif void impl::Heartbeat::process_output_pre_write(int step, double time) { diff --git a/packages/stk/stk_io/stk_io/Heartbeat.hpp b/packages/stk/stk_io/stk_io/Heartbeat.hpp index 9966b6d46c78..83a6f61865fb 100644 --- a/packages/stk/stk_io/stk_io/Heartbeat.hpp +++ b/packages/stk/stk_io/stk_io/Heartbeat.hpp @@ -89,30 +89,56 @@ class Heartbeat { ~Heartbeat() {}; void define_global_ref(const std::string &variableName, + const stk::util::Parameter ¶m, + int copies = 1, + Ioss::Field::RoleType role = Ioss::Field::TRANSIENT); + + void define_global_ref(const std::string &name, + const stk::util::Parameter ¶m, + const std::string &storage, + Ioss::Field::BasicType dataType, + int copies = 1, + Ioss::Field::RoleType role = Ioss::Field::TRANSIENT); + + void add_global_ref(const std::string &variableName, + const stk::util::Parameter ¶m, + int copies = 1, + Ioss::Field::RoleType role = Ioss::Field::TRANSIENT); + + void add_global_ref(const std::string &name, + const stk::util::Parameter ¶m, + const std::string &storage, + Ioss::Field::BasicType dataType, + int copies = 1, + Ioss::Field::RoleType role = Ioss::Field::TRANSIENT); + +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 + STK_DEPRECATED void define_global_ref(const std::string &variableName, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type, int copies = 1, Ioss::Field::RoleType role = Ioss::Field::TRANSIENT); - void define_global_ref(const std::string &name, + STK_DEPRECATED void define_global_ref(const std::string &name, const STK_ANY_NAMESPACE::any *value, const std::string &storage, Ioss::Field::BasicType dataType, int copies = 1, Ioss::Field::RoleType role = Ioss::Field::TRANSIENT); - void add_global_ref(const std::string &variableName, + STK_DEPRECATED void add_global_ref(const std::string &variableName, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type, int copies = 1, Ioss::Field::RoleType role = Ioss::Field::TRANSIENT); - void add_global_ref(const std::string &name, + STK_DEPRECATED void add_global_ref(const std::string &name, const STK_ANY_NAMESPACE::any *value, const std::string &storage, Ioss::Field::BasicType dataType, int copies = 1, Ioss::Field::RoleType role = Ioss::Field::TRANSIENT); +#endif void process_output(int step, double time); void process_output_pre_write(int step, double time); @@ -130,6 +156,19 @@ class Heartbeat { bool has_global(const std::string &name); private: + void internal_define_global_ref(const std::string &variableName, + const STK_ANY_NAMESPACE::any *value, + stk::util::ParameterType::Type type, + int copies = 1, + Ioss::Field::RoleType role = Ioss::Field::TRANSIENT); + + void internal_define_global_ref(const std::string &name, + const STK_ANY_NAMESPACE::any *value, + const std::string &storage, + Ioss::Field::BasicType dataType, + int copies = 1, + Ioss::Field::RoleType role = Ioss::Field::TRANSIENT); + std::vector m_fields; Teuchos::RCP m_region; diff --git a/packages/stk/stk_io/stk_io/IOHelpers.cpp b/packages/stk/stk_io/stk_io/IOHelpers.cpp index dd8825676010..693cf09bc93b 100644 --- a/packages/stk/stk_io/stk_io/IOHelpers.cpp +++ b/packages/stk/stk_io/stk_io/IOHelpers.cpp @@ -243,6 +243,12 @@ bool internal_read_global(Teuchos::RCP input_region, const std::st bool abort_if_not_found); +void internal_write_parameter(Teuchos::RCP output_region, + const std::string &name, const stk::util::Parameter ¶m) +{ + internal_write_parameter(output_region, name, param.value, param.type); +} + void internal_write_parameter(Teuchos::RCP output_region, const std::string &name, const STK_ANY_NAMESPACE::any &any_value, stk::util::ParameterType::Type type) @@ -311,6 +317,14 @@ void write_defined_global_any_fields(Teuchos::RCP region, } } +bool internal_read_parameter(Teuchos::RCP input_region, + const std::string &globalVarName, + stk::util::Parameter& param, + bool abort_if_not_found) +{ + return internal_read_parameter(input_region, globalVarName, param.value, param.type, abort_if_not_found); +} + bool internal_read_parameter(Teuchos::RCP input_region, const std::string &globalVarName, STK_ANY_NAMESPACE::any &any_value, stk::util::ParameterType::Type type, diff --git a/packages/stk/stk_io/stk_io/IOHelpers.hpp b/packages/stk/stk_io/stk_io/IOHelpers.hpp index cf1a8cb03cef..cb7564587ccc 100644 --- a/packages/stk/stk_io/stk_io/IOHelpers.hpp +++ b/packages/stk/stk_io/stk_io/IOHelpers.hpp @@ -96,6 +96,9 @@ void internal_write_parameter(Teuchos::RCP output_region, const std::string &name, const STK_ANY_NAMESPACE::any &any_value, stk::util::ParameterType::Type type); +void internal_write_parameter(Teuchos::RCP output_region, + const std::string &name, const stk::util::Parameter ¶m); + void write_defined_global_any_fields(Teuchos::RCP region, std::vector &global_any_fields); @@ -104,6 +107,11 @@ bool internal_read_parameter(Teuchos::RCP input_region, STK_ANY_NAMESPACE::any &any_value, stk::util::ParameterType::Type type, bool abort_if_not_found); +bool internal_read_parameter(Teuchos::RCP input_region, + const std::string &globalVarName, + stk::util::Parameter& param, + bool abort_if_not_found); + void internal_add_global(Teuchos::RCP region, const std::string &globalVarName, const std::string &storage, diff --git a/packages/stk/stk_io/stk_io/OutputFile.cpp b/packages/stk/stk_io/stk_io/OutputFile.cpp index 7a411254d59d..b56c87708b4e 100644 --- a/packages/stk/stk_io/stk_io/OutputFile.cpp +++ b/packages/stk/stk_io/stk_io/OutputFile.cpp @@ -385,7 +385,17 @@ void OutputFile::add_user_data(const std::vector& partNames, const } -void OutputFile::add_global_ref(const std::string &name, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type) +void OutputFile::add_global_ref(const std::string &name, const stk::util::Parameter ¶m) +{ + ThrowErrorMsgIf (m_fieldsDefined, + "On region named " << m_region->name() << + " Attempting to add global variable after data has already been written to the database."); + std::pair parameter_type = get_io_parameter_size_and_type(param.type, param.value); + internal_add_global(m_region, name, parameter_type.first, parameter_type.second); + m_globalAnyFields.emplace_back(name, ¶m.value, param.type); +} +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 +STK_DEPRECATED void OutputFile::add_global_ref(const std::string &name, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type) { ThrowErrorMsgIf (m_fieldsDefined, "On region named " << m_region->name() << @@ -394,13 +404,25 @@ void OutputFile::add_global_ref(const std::string &name, const STK_ANY_NAMESPACE internal_add_global(m_region, name, parameter_type.first, parameter_type.second); m_globalAnyFields.emplace_back(name, value, type); } +#endif bool OutputFile::has_global(const std::string &globalVarName) const { return m_region->field_exists(globalVarName); } -void OutputFile::add_global(const std::string &name, const STK_ANY_NAMESPACE::any &value, stk::util::ParameterType::Type type) +void OutputFile::add_global(const std::string &name, const stk::util::Parameter ¶m) +{ + ThrowErrorMsgIf (m_fieldsDefined, + "On region named " << m_region->name() << + " Attempting to add global variable after data has already been written to the database."); + std::pair parameter_type = get_io_parameter_size_and_type(param.type, param.value); + m_anyGlobalVariablesDefined = true; // This output file has at least 1 global variable. + internal_add_global(m_region, name, parameter_type.first, parameter_type.second); +} + +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 +STK_DEPRECATED void OutputFile::add_global(const std::string &name, const STK_ANY_NAMESPACE::any &value, stk::util::ParameterType::Type type) { ThrowErrorMsgIf (m_fieldsDefined, "On region named " << m_region->name() << @@ -409,6 +431,7 @@ void OutputFile::add_global(const std::string &name, const STK_ANY_NAMESPACE::an m_anyGlobalVariablesDefined = true; // This output file has at least 1 global variable. internal_add_global(m_region, name, parameter_type.first, parameter_type.second); } +#endif void OutputFile::add_global(const std::string &globalVarName, Ioss::Field::BasicType dataType) { @@ -437,11 +460,19 @@ void OutputFile::add_global(const std::string &globalVarName, const std::string internal_add_global(m_region, globalVarName, storage, dataType); } -void OutputFile::write_global(const std::string &globalVarName, - const STK_ANY_NAMESPACE::any &value, stk::util::ParameterType::Type type) +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 +STK_DEPRECATED void OutputFile::write_global(const std::string &globalVarName, + const STK_ANY_NAMESPACE::any &value, stk::util::ParameterType::Type type) { internal_write_parameter(m_region, globalVarName, value, type); } +#endif + +void OutputFile::write_global(const std::string &globalVarName, + const stk::util::Parameter ¶m) +{ + internal_write_parameter(m_region, globalVarName, param); +} void OutputFile::write_global(const std::string &globalVarName, std::vector& globalVarData) { diff --git a/packages/stk/stk_io/stk_io/OutputFile.hpp b/packages/stk/stk_io/stk_io/OutputFile.hpp index 989317b12e17..47e98528de26 100644 --- a/packages/stk/stk_io/stk_io/OutputFile.hpp +++ b/packages/stk/stk_io/stk_io/OutputFile.hpp @@ -45,6 +45,7 @@ #include #include #include // for MeshField, etc +#include #include // for BulkData #include // for Selector #include // for ParallelMachine @@ -149,14 +150,34 @@ class OutputFile void add_attribute_field(stk::mesh::FieldBase &field, const OutputVariableParams &var); void add_user_data(const std::vector& userData, const std::string &alternate_name, stk::io::DataLocation loc); bool has_global(const std::string &globalVarName) const; - void add_global(const std::string &variableName, const STK_ANY_NAMESPACE::any &value, stk::util::ParameterType::Type type); - void add_global_ref(const std::string &variableName, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type); + void add_global(const std::string &variableName, const stk::util::Parameter &value); + + template + void add_global(const std::string &variableName, const T& value, stk::util::ParameterType::Type type) + { + STK_ANY_NAMESPACE::any anyValue(value); + std::pair parameter_type = get_io_parameter_size_and_type(type, anyValue); + m_anyGlobalVariablesDefined = true; + internal_add_global(m_region, variableName, parameter_type.first, parameter_type.second); + } + +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 + STK_DEPRECATED void add_global(const std::string &variableName, const STK_ANY_NAMESPACE::any &value, stk::util::ParameterType::Type type); +#endif + void add_global_ref(const std::string &variableName, const stk::util::Parameter &value); +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 + STK_DEPRECATED void add_global_ref(const std::string &variableName, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type); +#endif void add_global(const std::string &variableName, Ioss::Field::BasicType dataType); void add_global(const std::string &variableName, const std::string &type, Ioss::Field::BasicType dataType); void add_global(const std::string &variableName, int component_count, Ioss::Field::BasicType dataType); void write_global(const std::string &variableName, + const stk::util::Parameter ¶m); +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 + STK_DEPRECATED void write_global(const std::string &variableName, const STK_ANY_NAMESPACE::any &value, stk::util::ParameterType::Type type); +#endif void write_global(const std::string &variableName, double globalVarData); void write_global(const std::string &variableName, int globalVarData); void write_global(const std::string &variableName, std::vector& globalVarData); diff --git a/packages/stk/stk_io/stk_io/StkIoUtils.hpp b/packages/stk/stk_io/stk_io/StkIoUtils.hpp index a1530d2364c5..8588fc5cb334 100644 --- a/packages/stk/stk_io/stk_io/StkIoUtils.hpp +++ b/packages/stk/stk_io/stk_io/StkIoUtils.hpp @@ -47,6 +47,10 @@ #include "stk_util/util/ParameterList.hpp" #include "Ioss_Field.h" +#ifdef STK_HAVE_BOOST +#include +#endif + namespace stk { namespace io { class StkMeshIoBroker; } } namespace stk { namespace mesh { class BulkData; } } namespace stk { namespace mesh { class MetaData; } } diff --git a/packages/stk/stk_io/stk_io/StkMeshIoBroker.cpp b/packages/stk/stk_io/stk_io/StkMeshIoBroker.cpp index aa46e45a81fb..51f19c9b20c4 100644 --- a/packages/stk/stk_io/stk_io/StkMeshIoBroker.cpp +++ b/packages/stk/stk_io/stk_io/StkMeshIoBroker.cpp @@ -913,6 +913,16 @@ void StkMeshIoBroker::get_global_variable_names(std::vector &names) } bool StkMeshIoBroker::get_global(const std::string &globalVarName, + stk::util::Parameter ¶m, + bool abort_if_not_found) const +{ + validate_input_file_index(m_activeMeshIndex); + auto region = m_inputFiles[m_activeMeshIndex]->get_input_io_region(); + return internal_read_parameter(region, globalVarName, param.value, param.type, abort_if_not_found); +} + +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 +STK_DEPRECATED bool StkMeshIoBroker::get_global(const std::string &globalVarName, STK_ANY_NAMESPACE::any &value, stk::util::ParameterType::Type type, bool abort_if_not_found) const { @@ -920,6 +930,7 @@ bool StkMeshIoBroker::get_global(const std::string &globalVarName, auto region = m_inputFiles[m_activeMeshIndex]->get_input_io_region(); return internal_read_parameter(region, globalVarName, value, type, abort_if_not_found); } +#endif size_t StkMeshIoBroker::get_global_variable_length(const std::string& globalVarName) const { @@ -977,18 +988,36 @@ bool StkMeshIoBroker::has_global(size_t output_file_index, const std::string &gl } void StkMeshIoBroker::add_global(size_t output_file_index, const std::string &name, + const stk::util::Parameter ¶m) +{ + validate_output_file_index(output_file_index); + m_outputFiles[output_file_index]->add_global(name, param); +} + +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 +STK_DEPRECATED void StkMeshIoBroker::add_global(size_t output_file_index, const std::string &name, const STK_ANY_NAMESPACE::any &value, stk::util::ParameterType::Type type) { validate_output_file_index(output_file_index); m_outputFiles[output_file_index]->add_global(name, value, type); } +#endif void StkMeshIoBroker::add_global_ref(size_t output_file_index, const std::string &name, + const stk::util::Parameter ¶m) +{ + validate_output_file_index(output_file_index); + m_outputFiles[output_file_index]->add_global_ref(name, param); +} + +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 +STK_DEPRECATED void StkMeshIoBroker::add_global_ref(size_t output_file_index, const std::string &name, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type) { validate_output_file_index(output_file_index); m_outputFiles[output_file_index]->add_global_ref(name, value, type); } +#endif void StkMeshIoBroker::add_global(size_t output_file_index, const std::string &globalVarName, Ioss::Field::BasicType dataType) { @@ -1008,12 +1037,14 @@ void StkMeshIoBroker::add_global(size_t output_file_index, const std::string &gl m_outputFiles[output_file_index]->add_global(globalVarName, storage, dataType); } -void StkMeshIoBroker::write_global(size_t output_file_index, const std::string &globalVarName, +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 +STK_DEPRECATED void StkMeshIoBroker::write_global(size_t output_file_index, const std::string &globalVarName, const STK_ANY_NAMESPACE::any &value, stk::util::ParameterType::Type type) const { validate_output_file_index(output_file_index); m_outputFiles[output_file_index]->write_global(globalVarName, value, type); } +#endif void StkMeshIoBroker::write_global(size_t output_file_index, const std::string &globalVarName, double globalVarData) const { diff --git a/packages/stk/stk_io/stk_io/StkMeshIoBroker.hpp b/packages/stk/stk_io/stk_io/StkMeshIoBroker.hpp index 8603720dfa57..d40b8f839ab0 100644 --- a/packages/stk/stk_io/stk_io/StkMeshIoBroker.hpp +++ b/packages/stk/stk_io/stk_io/StkMeshIoBroker.hpp @@ -144,7 +144,7 @@ namespace stk { // (typically locally owned part) used to associate entities // when generating the output database. void set_subset_selector(size_t output_file_index, Teuchos::RCP my_selector); - void set_subset_selector(size_t output_file_index, stk::mesh::Selector &my_selector); + void set_subset_selector(size_t output_file_index, const stk::mesh::Selector &my_selector); void set_skin_mesh_selector(size_t output_file_index, Teuchos::RCP my_selector); void set_skin_mesh_selector(size_t output_file_index, stk::mesh::Selector &my_selector); @@ -206,13 +206,6 @@ namespace stk { m_autoLoadDistributionFactorPerNodeSet = shouldAutoLoad; } -#ifndef STK_HIDE_DEPRECATED_CODE // Delete after November 2020 - STK_DEPRECATED void enable_edge_io() - { - m_enableEdgeIO = true; - } -#endif - // Create the Ioss::DatabaseIO associated with the specified filename // and type (exodus by default). The routine checks that the // file exists and is readable and will throw an exception if not. @@ -394,21 +387,26 @@ namespace stk { size_t get_global_variable_length(const std::string& name) const; bool get_global(const std::string &variableName, - STK_ANY_NAMESPACE::any &value, - stk::util::ParameterType::Type type, - bool abort_if_not_found=true) const; + stk::util::Parameter ¶m, + bool abort_if_not_found=true) const; +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 + STK_DEPRECATED bool get_global(const std::string &variableName, + STK_ANY_NAMESPACE::any &value, + stk::util::ParameterType::Type type, + bool abort_if_not_found=true) const; +#endif bool get_global(const std::string &variableName, - int &globalVar, - bool abort_if_not_found=true) const; + int &globalVar, + bool abort_if_not_found=true) const; bool get_global(const std::string &variableName, - double &globalVar, - bool abort_if_not_found=true) const; + double &globalVar, + bool abort_if_not_found=true) const; bool get_global(const std::string &variableName, - std::vector &globalVar, - bool abort_if_not_found=true) const; + std::vector &globalVar, + bool abort_if_not_found=true) const; bool get_global(const std::string &variableName, - std::vector &globalVar, - bool abort_if_not_found=true) const; + std::vector &globalVar, + bool abort_if_not_found=true) const; bool has_input_global(const std::string &globalVarName) const; void add_input_field(const stk::io::MeshField &mesh_field); @@ -476,24 +474,31 @@ namespace stk { stk::io::DataLocation loc); bool has_global(size_t output_file_index, const std::string &globalVarName) const; + void add_global_ref(size_t output_file_index, + const std::string &variableName, + const stk::util::Parameter ¶m); + void add_global(size_t output_file_index, + const std::string &variableName, + const stk::util::Parameter ¶m); template void add_global(size_t output_file_index, const std::string& variableName, const T& value, stk::util::ParameterType::Type type) - { + { validate_output_file_index(output_file_index); - STK_ANY_NAMESPACE::any anyValue(value); - m_outputFiles[output_file_index]->add_global(variableName, anyValue, type); + m_outputFiles[output_file_index]->add_global(variableName, value, type); } - void add_global_ref(size_t output_file_index, +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 + STK_DEPRECATED void add_global_ref(size_t output_file_index, const std::string &variableName, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type); - void add_global(size_t output_file_index, + STK_DEPRECATED void add_global(size_t output_file_index, const std::string &variableName, const STK_ANY_NAMESPACE::any &value, stk::util::ParameterType::Type type); +#endif void add_global(size_t output_file_index, const std::string &variableName, Ioss::Field::BasicType dataType); @@ -542,14 +547,18 @@ namespace stk { const T &value, stk::util::ParameterType::Type type) const { validate_output_file_index(output_file_index); - STK_ANY_NAMESPACE::any anyValue(value); - m_outputFiles[output_file_index]->write_global(globalVarName, anyValue, type); + stk::util::Parameter p; + p.value = value; + p.type = type; + m_outputFiles[output_file_index]->write_global(globalVarName, p); } - void write_global(size_t output_file_index, - const std::string &variableName, - const STK_ANY_NAMESPACE::any &value, - stk::util::ParameterType::Type type) const; +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 + STK_DEPRECATED void write_global(size_t output_file_index, + const std::string &variableName, + const STK_ANY_NAMESPACE::any &value, + stk::util::ParameterType::Type type) const; +#endif void write_global(size_t output_file_index, const std::string &variableName, double data) const; @@ -570,13 +579,42 @@ namespace stk { bool openFileImmediately = true); void define_heartbeat_global(size_t index, + const std::string &name, + const stk::util::Parameter ¶m, + int copies = 1, + Ioss::Field::RoleType role = Ioss::Field::REDUCTION); + + void define_heartbeat_global(size_t index, + const std::string &globalVarName, + const stk::util::Parameter ¶m, + const std::string &storage, + Ioss::Field::BasicType dataType, + int copies = 1, + Ioss::Field::RoleType role = Ioss::Field::REDUCTION); + + void add_heartbeat_global(size_t index, + const std::string &name, + const stk::util::Parameter ¶m, + int copies = 1, + Ioss::Field::RoleType role = Ioss::Field::REDUCTION); + + void add_heartbeat_global(size_t index, + const std::string &globalVarName, + const stk::util::Parameter ¶m, + const std::string &storage, + Ioss::Field::BasicType dataType, + int copies = 1, + Ioss::Field::RoleType role = Ioss::Field::REDUCTION); + +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 + STK_DEPRECATED void define_heartbeat_global(size_t index, const std::string &name, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type, int copies = 1, Ioss::Field::RoleType role = Ioss::Field::REDUCTION); - void define_heartbeat_global(size_t index, + STK_DEPRECATED void define_heartbeat_global(size_t index, const std::string &globalVarName, const STK_ANY_NAMESPACE::any *value, const std::string &storage, @@ -584,20 +622,21 @@ namespace stk { int copies = 1, Ioss::Field::RoleType role = Ioss::Field::REDUCTION); - void add_heartbeat_global(size_t index, + STK_DEPRECATED void add_heartbeat_global(size_t index, const std::string &name, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type, int copies = 1, Ioss::Field::RoleType role = Ioss::Field::REDUCTION); - void add_heartbeat_global(size_t index, + STK_DEPRECATED void add_heartbeat_global(size_t index, const std::string &globalVarName, const STK_ANY_NAMESPACE::any *value, const std::string &storage, Ioss::Field::BasicType dataType, int copies = 1, Ioss::Field::RoleType role = Ioss::Field::REDUCTION); +#endif bool has_heartbeat_global(size_t output_file_index, const std::string &globalVarName) const; @@ -743,6 +782,7 @@ namespace stk { stk::mesh::Selector m_activeSelector; + stk::mesh::Selector m_subsetSelector; Teuchos::RCP m_deprecatedSelector; protected: @@ -793,9 +833,10 @@ namespace stk { } inline void StkMeshIoBroker::set_subset_selector(size_t output_file_index, - stk::mesh::Selector &my_selector) { + const stk::mesh::Selector &my_selector) { + m_subsetSelector = my_selector; validate_output_file_index(output_file_index); - m_outputFiles[output_file_index]->set_subset_selector(Teuchos::rcpFromRef(my_selector)); + m_outputFiles[output_file_index]->set_subset_selector(Teuchos::rcpFromRef(m_subsetSelector)); } inline void StkMeshIoBroker::set_skin_mesh_selector(size_t output_file_index, @@ -874,6 +915,51 @@ namespace stk { { replace_bulk_data(Teuchos::rcpFromRef(arg_bulk_data));} inline void StkMeshIoBroker::define_heartbeat_global(size_t index, + const std::string &name, + const stk::util::Parameter ¶m, + int copies, + Ioss::Field::RoleType role) + { + STKIORequire(index < m_heartbeat.size()); + m_heartbeat[index]->define_global_ref(name, param, copies, role); + } + + inline void StkMeshIoBroker::define_heartbeat_global(size_t index, + const std::string &globalVarName, + const stk::util::Parameter ¶m, + const std::string &storage, + Ioss::Field::BasicType dataType, + int copies, + Ioss::Field::RoleType role) + { + STKIORequire(index < m_heartbeat.size()); + m_heartbeat[index]->define_global_ref(globalVarName, param, storage, dataType, copies, role); + } + + inline void StkMeshIoBroker::add_heartbeat_global(size_t index, + const std::string &name, + const stk::util::Parameter ¶m, + int copies, + Ioss::Field::RoleType role) + { + STKIORequire(index < m_heartbeat.size()); + m_heartbeat[index]->add_global_ref(name, param, copies, role); + } + + inline void StkMeshIoBroker::add_heartbeat_global(size_t index, + const std::string &globalVarName, + const stk::util::Parameter ¶m, + const std::string &storage, + Ioss::Field::BasicType dataType, + int copies, + Ioss::Field::RoleType role) + { + STKIORequire(index < m_heartbeat.size()); + m_heartbeat[index]->add_global_ref(globalVarName, param, storage, dataType, copies, role); + } + +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after September 2021 + inline STK_DEPRECATED void StkMeshIoBroker::define_heartbeat_global(size_t index, const std::string &name, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type, @@ -896,7 +982,7 @@ namespace stk { m_heartbeat[index]->define_global_ref(globalVarName, value, storage, dataType, copies, role); } - inline void StkMeshIoBroker::add_heartbeat_global(size_t index, + inline STK_DEPRECATED void StkMeshIoBroker::add_heartbeat_global(size_t index, const std::string &name, const STK_ANY_NAMESPACE::any *value, stk::util::ParameterType::Type type, @@ -907,7 +993,7 @@ namespace stk { m_heartbeat[index]->add_global_ref(name, value, type, copies, role); } - inline void StkMeshIoBroker::add_heartbeat_global(size_t index, + inline STK_DEPRECATED void StkMeshIoBroker::add_heartbeat_global(size_t index, const std::string &globalVarName, const STK_ANY_NAMESPACE::any *value, const std::string &storage, @@ -918,6 +1004,7 @@ namespace stk { STKIORequire(index < m_heartbeat.size()); m_heartbeat[index]->add_global_ref(globalVarName, value, storage, dataType, copies, role); } +#endif inline void StkMeshIoBroker::process_heartbeat_output(size_t index, int step, double time) { diff --git a/packages/stk/stk_mesh/Jamfile b/packages/stk/stk_mesh/Jamfile index 5b661939f90e..fe51fdefb5ca 100644 --- a/packages/stk/stk_mesh/Jamfile +++ b/packages/stk/stk_mesh/Jamfile @@ -197,6 +197,7 @@ lib stk_mesh_base /sierra/stk_util//stk_util_parallel /sierra/stk_util//stk_util_util /sierra/stk_util//stk_util_diag + /sierra/stk_util//stk_util_ngp /sierra/stk_topology//stk_topology /tpl/trilinos//shards /tpl/lapack//lapack_interface diff --git a/packages/stk/stk_mesh/stk_mesh/base/BulkData.cpp b/packages/stk/stk_mesh/stk_mesh/base/BulkData.cpp index 1500f08cdeef..45f3dd959aa6 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/BulkData.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/BulkData.cpp @@ -2804,6 +2804,14 @@ void BulkData::internal_change_entity_owner( const std::vector & arg OrdinalVector scratchOrdinalVec, scratchSpace; PartVector parts ; std::vector relations ; + + OrdinalVector removeCustomGhostParts; + const std::vector& ghostingObjs = ghostings(); + const unsigned firstCustomGhosting = 2; + for(unsigned i=firstCustomGhosting; i & arg partOrdinals.push_back(part->mesh_meta_data_ordinal()); } - internal_change_entity_parts( entity , partOrdinals , OrdinalVector(), scratchOrdinalVec, scratchSpace ); + internal_change_entity_parts( entity , partOrdinals , removeCustomGhostParts, scratchOrdinalVec, scratchSpace ); + for(unsigned i=firstCustomGhosting; iordinal(), p)); + } log_created_parallel_copy( entity ); @@ -2872,13 +2883,17 @@ void BulkData::internal_change_entity_owner( const std::vector & arg for ( EntityVector::reverse_iterator i = unique_list_of_send_closure.rbegin() ; i != unique_list_of_send_closure.rend() ; ++i) { stk::mesh::Entity entity = *i; if ( ! this->owned_closure(entity) ) { + for(unsigned ig=firstCustomGhosting; igmesh_meta_data_ordinal()); OrdinalVector scratchOrdinalVec, scratchSpace; + for (std::set::reverse_iterator reverseIterator = removeSendGhosts.rbegin(); reverseIterator != removeSendGhosts.rend(); ++reverseIterator) { + const EntityKey key = reverseIterator->first; + const int proc = reverseIterator->second; + if (impl::has_upward_send_ghost_connectivity(*this, ghosting, proc, get_entity(key))) { + continue; + } + + entity_comm_map_erase(key, EntityCommInfo(ghosting.ordinal(), proc)); + internal_change_entity_parts(get_entity(key), {}, removeGhostingPart, scratchOrdinalVec, scratchSpace); + } + + for(EntityCommListInfo& info : m_entity_comm_list) { + if (info.entity_comm == nullptr || info.entity_comm->comm_map.empty()) { + info.key = EntityKey(); + } + } + delete_unneeded_entries_from_the_comm_list(); + stk::util::sort_and_unique(removeRecvGhosts, EntityLess(*this)); for (unsigned i=0; i 1) { all_reduce( parallel() , ReduceMin<1>( & verified_ok ) ); } diff --git a/packages/stk/stk_mesh/stk_mesh/base/BulkData.hpp b/packages/stk/stk_mesh/stk_mesh/base/BulkData.hpp index eb3611eca18c..a9aba76307aa 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/BulkData.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/BulkData.hpp @@ -52,7 +52,7 @@ #include // for Selector #include // for MeshIndex, EntityRank, etc #include -#include "stk_mesh/base/NgpSpaces.hpp" +#include "stk_util/ngp/NgpSpaces.hpp" #include // for BucketRepository #include // for ParallelMachine #include // for char_traits, string diff --git a/packages/stk/stk_mesh/stk_mesh/base/CellTopology.hpp b/packages/stk/stk_mesh/stk_mesh/base/CellTopology.hpp index 0018ecd9b6ab..4e3ec70fbba3 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/CellTopology.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/CellTopology.hpp @@ -51,19 +51,6 @@ namespace mesh { typedef shards::CellTopology CellTopology; -#ifndef STK_HIDE_DEPRECATED_CODE // Delete after 2019-07-18 -template< typename id_type > -STK_DEPRECATED -int findPermutation( const CellTopology top , - const id_type * const expected_node , - const id_type * const actual_node ) -{ - return shards::findPermutation( *top.getCellTopologyData() , expected_node , actual_node ); -} -#endif - -/** \} */ - } // namespace mesh } // namespace stk diff --git a/packages/stk/stk_mesh/stk_mesh/base/DeviceField.hpp b/packages/stk/stk_mesh/stk_mesh/base/DeviceField.hpp index f6845ce45ac7..ed8b1e640a15 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/DeviceField.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/DeviceField.hpp @@ -60,7 +60,7 @@ template class NgpDebugger> class DeviceField : public NgpFieldBase { private: - using FieldDataDeviceUnmanagedViewType = Kokkos::View>; + using FieldDataDeviceUnmanagedViewType = Kokkos::View>; using StkDebugger = typename NgpDebugger::StkFieldSyncDebuggerType; public: @@ -112,7 +112,7 @@ class DeviceField : public NgpFieldBase } DeviceField(const stk::mesh::BulkData& bulk, const stk::mesh::FieldBase &stkField, - const ExecSpace& execSpace, bool isFromGetUpdatedNgpField = false) + const stk::ngp::ExecSpace& execSpace, bool isFromGetUpdatedNgpField = false) : NgpFieldBase(), rank(stkField.entity_rank()), ordinal(stkField.mesh_meta_data_ordinal()), @@ -141,8 +141,8 @@ class DeviceField : public NgpFieldBase hostField->template make_field_sync_debugger(); fieldSyncDebugger = NgpDebugger(&hostField->get_field_sync_debugger()); - needSyncToHost = Kokkos::View("needSyncToHost"); - needSyncToDevice = Kokkos::View("needSyncToDevice"); + needSyncToHost = Kokkos::View("needSyncToHost"); + needSyncToDevice = Kokkos::View("needSyncToDevice"); const int maxStates = static_cast(stk::mesh::MaximumFieldStates); for (int s=0; s nonconst_device_mesh_indices(Kokkos::view_alloc(Kokkos::WithoutInitializing, "tmp_dev_mesh_indices"), length); + Kokkos::View nonconst_device_mesh_indices(Kokkos::view_alloc(Kokkos::WithoutInitializing, "tmp_dev_mesh_indices"), length); Kokkos::deep_copy(nonconst_device_mesh_indices, hostMeshIndices); deviceMeshIndices = nonconst_device_mesh_indices; } diff --git a/packages/stk/stk_mesh/stk_mesh/base/DeviceMesh.hpp b/packages/stk/stk_mesh/stk_mesh/base/DeviceMesh.hpp index 726bec911f20..82b7bb337434 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/DeviceMesh.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/DeviceMesh.hpp @@ -48,7 +48,7 @@ #include #include -#include +#include #include #include @@ -166,7 +166,7 @@ struct DeviceMeshIndex class DeviceMesh : public NgpMeshBase { public: - using MeshExecSpace = stk::mesh::ExecSpace; + using MeshExecSpace = stk::ngp::ExecSpace; using ConnectedNodes = DeviceBucket::ConnectedNodes; using ConnectedEntities = DeviceBucket::ConnectedEntities; using ConnectedOrdinals = DeviceBucket::ConnectedOrdinals; @@ -490,7 +490,7 @@ class DeviceMesh : public NgpMeshBase void copy_volatile_fast_shared_comm_map_to_device(); - using BucketView = Kokkos::View; + using BucketView = Kokkos::View; const stk::mesh::BulkData *bulk; unsigned spatial_dimension; unsigned synchronizedCount; @@ -504,8 +504,8 @@ class DeviceMesh : public NgpMeshBase HostMeshIndexType hostMeshIndices; MeshIndexType deviceMeshIndices; - Kokkos::View bucketEntityOffsets[stk::topology::NUM_RANKS]; - Kokkos::View::HostMirror hostBucketEntityOffsets[stk::topology::NUM_RANKS]; + Kokkos::View bucketEntityOffsets[stk::topology::NUM_RANKS]; + Kokkos::View::HostMirror hostBucketEntityOffsets[stk::topology::NUM_RANKS]; UnsignedViewType entityConnectivityOffset[stk::topology::NUM_RANKS][stk::topology::NUM_RANKS]; UnsignedViewType::HostMirror hostEntityConnectivityOffset[stk::topology::NUM_RANKS][stk::topology::NUM_RANKS]; diff --git a/packages/stk/stk_mesh/stk_mesh/base/FieldSyncDebugging.hpp b/packages/stk/stk_mesh/stk_mesh/base/FieldSyncDebugging.hpp index fa9159534e58..3802081e39e8 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/FieldSyncDebugging.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/FieldSyncDebugging.hpp @@ -2,7 +2,7 @@ #define FIELDSYNCDEBUGGING_HPP #include "Kokkos_Core.hpp" -#include "stk_mesh/base/NgpSpaces.hpp" +#include "stk_util/ngp/NgpSpaces.hpp" #ifndef __has_builtin #define __has_builtin(x) 0 // Compatibility with non-clang compilers. @@ -29,7 +29,7 @@ #define HOST_DEBUG_LINE_NUMBER -1 #endif -#if defined(HAS_LOCATION_BUILTINS) && !defined(KOKKOS_ENABLE_CUDA) +#if defined(HAS_LOCATION_BUILTINS) && !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) #define DEVICE_USE_LOCATION_BUILTINS #define DEVICE_DEBUG_FILE_NAME __builtin_FILE() #define DEVICE_DEBUG_LINE_NUMBER __builtin_LINE() @@ -48,10 +48,10 @@ enum LastModLocation : uint8_t { HOST_OR_DEVICE = HOST | DEVICE }; -using LastFieldModLocationType = Kokkos::View; +using LastFieldModLocationType = Kokkos::View; template -using ScalarUvmType = Kokkos::View; +using ScalarUvmType = Kokkos::View; struct DummyOverload { diff --git a/packages/stk/stk_mesh/stk_mesh/base/GetNgpExecutionSpace.hpp b/packages/stk/stk_mesh/stk_mesh/base/GetNgpExecutionSpace.hpp index a9984a5eb97c..ee818681ce7a 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/GetNgpExecutionSpace.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/GetNgpExecutionSpace.hpp @@ -37,12 +37,12 @@ #include #include -#include +#include namespace stk { namespace mesh { -template +template class ExecSpaceWrapper { public: operator const ExecSpaceType&() { return space; } @@ -73,7 +73,7 @@ struct ExecSpaceAndCudaStreamDeleter { streamPtr = deleter.streamPtr; } - void operator()(ExecSpace* e) const { + void operator()(stk::ngp::ExecSpace* e) const { cudaStreamDestroy(*streamPtr); delete streamPtr; delete e; diff --git a/packages/stk/stk_mesh/stk_mesh/base/GetNgpField.hpp b/packages/stk/stk_mesh/stk_mesh/base/GetNgpField.hpp index 6004cba4e1f2..4ed3d86017a6 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/GetNgpField.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/GetNgpField.hpp @@ -42,7 +42,7 @@ namespace stk { namespace mesh { template class NgpDebugger = DefaultNgpFieldSyncDebugger> -NgpField & get_updated_ngp_field_async(const FieldBase & stkField, const ExecSpace& execSpace) +NgpField & get_updated_ngp_field_async(const FieldBase & stkField, const stk::ngp::ExecSpace& execSpace) { NgpFieldBase * ngpField = impl::get_ngp_field(stkField); diff --git a/packages/stk/stk_mesh/stk_mesh/base/HostField.hpp b/packages/stk/stk_mesh/stk_mesh/base/HostField.hpp index 8da929cdfacc..6bc15adb0307 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/HostField.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/HostField.hpp @@ -81,7 +81,7 @@ class HostField : public NgpFieldBase } HostField(const stk::mesh::BulkData& b, const stk::mesh::FieldBase& f, - const ExecSpace& execSpace, bool isFromGetUpdatedNgpField = false) + const stk::ngp::ExecSpace& execSpace, bool isFromGetUpdatedNgpField = false) : HostField(b, f, isFromGetUpdatedNgpField) { asyncCopyState.set_state(execSpace, impl::INVALID); @@ -98,7 +98,7 @@ class HostField : public NgpFieldBase void set_field_states(HostField* fields[]) {} - void set_execution_space(const ExecSpace& execSpace) override { asyncCopyState.set_execution_space(execSpace); } + void set_execution_space(const stk::ngp::ExecSpace& execSpace) override { asyncCopyState.set_execution_space(execSpace); } size_t num_syncs_to_host() const override { return field->num_syncs_to_host(); } size_t num_syncs_to_device() const override { return field->num_syncs_to_device(); } @@ -204,7 +204,7 @@ class HostField : public NgpFieldBase Kokkos::fence(); } - void sync_to_host(const ExecSpace& execSpace) override + void sync_to_host(const stk::ngp::ExecSpace& execSpace) override { if (need_sync_to_host()) { field->increment_num_syncs_to_host(); @@ -218,7 +218,7 @@ class HostField : public NgpFieldBase Kokkos::fence(); } - void sync_to_device(const ExecSpace& execSpace) override + void sync_to_device(const stk::ngp::ExecSpace& execSpace) override { if (need_sync_to_device()) { field->increment_num_syncs_to_device(); diff --git a/packages/stk/stk_mesh/stk_mesh/base/HostMesh.hpp b/packages/stk/stk_mesh/stk_mesh/base/HostMesh.hpp index fe62042caa0f..43d4499b355f 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/HostMesh.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/HostMesh.hpp @@ -49,7 +49,7 @@ #include #include -#include +#include #include #include @@ -65,7 +65,7 @@ struct HostMeshIndex class HostMesh : public NgpMeshBase { public: - using MeshExecSpace = stk::mesh::HostExecSpace; + using MeshExecSpace = stk::ngp::HostExecSpace; using MeshIndex = HostMeshIndex; using BucketType = stk::mesh::Bucket; using ConnectedNodes = util::StridedArray; diff --git a/packages/stk/stk_mesh/stk_mesh/base/MetaData.cpp b/packages/stk/stk_mesh/stk_mesh/base/MetaData.cpp index 2008cde171d8..c6f178a13d96 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/MetaData.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/MetaData.cpp @@ -502,20 +502,7 @@ void MetaData::commit() #endif } -MetaData::~MetaData() -{ - // Destroy the properties, used 'new' to allocate so now use 'delete' - - try { - std::vector::iterator i = m_created_topologies.begin(); - for ( ; i != m_created_topologies.end(); ++i) { - delete *i; - } - } catch(...) {} - - // PartRepository is member data - // FieldRepository is member data -} +MetaData::~MetaData() {} void MetaData::internal_declare_known_cell_topology_parts() { diff --git a/packages/stk/stk_mesh/stk_mesh/base/MetaData.hpp b/packages/stk/stk_mesh/stk_mesh/base/MetaData.hpp index 3f802910d116..bdab63f5037e 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/MetaData.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/MetaData.hpp @@ -615,7 +615,6 @@ class MetaData { mutable FieldBase* m_coord_field; std::vector< std::string > m_entity_rank_names ; - std::vector m_created_topologies; // Delete after 2019-07-18 unsigned m_spatial_dimension; SurfaceBlockMap m_surfaceToBlock; diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpAtomics.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpAtomics.hpp index bf700e149daa..fcfe2dc56ab9 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpAtomics.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpAtomics.hpp @@ -43,7 +43,7 @@ namespace mesh { template STK_FUNCTION void atomic_add(T *dest, const T src) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_OPENMP) +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_OPENMP) || defined(KOKKOS_ENABLE_HIP) Kokkos::atomic_add(dest, src); #else *dest += src; diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpDynamicMesh.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpDynamicMesh.hpp index 248238abcc64..75c596591a1e 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpDynamicMesh.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpDynamicMesh.hpp @@ -49,13 +49,15 @@ #include "stk_mesh/base/Bucket.hpp" #include "stk_mesh/base/Field.hpp" -#include "stk_mesh/base/NgpSpaces.hpp" +#include "stk_util/ngp/NgpSpaces.hpp" #include "stk_mesh/base/NgpUtils.hpp" #include "stk_mesh/base/NgpMesh.hpp" #include "stk_util/util/StkNgpVector.hpp" #ifdef KOKKOS_ENABLE_CUDA typedef Kokkos::Cuda Device; +#elif defined(KOKKOS_ENABLE_HIP) +typedef Kokkos::Experimental::HIP Device; #else typedef Kokkos::Serial Device; #endif @@ -64,14 +66,14 @@ namespace stk { namespace mesh { -typedef Kokkos::View UnsignedViewType; -typedef Kokkos::View RankViewType; -typedef Kokkos::View PartOrdViewType; -typedef Kokkos::View UnmanagedPartOrdViewType; -typedef Kokkos::View EntityViewType; -typedef Kokkos::View UnmanagedEntityViewType; -typedef Kokkos::View BktConnectivityType; -typedef Kokkos::View UnmanagedBktConnectivityType; +typedef Kokkos::View UnsignedViewType; +typedef Kokkos::View RankViewType; +typedef Kokkos::View PartOrdViewType; +typedef Kokkos::View UnmanagedPartOrdViewType; +typedef Kokkos::View EntityViewType; +typedef Kokkos::View UnmanagedEntityViewType; +typedef Kokkos::View BktConnectivityType; +typedef Kokkos::View UnmanagedBktConnectivityType; STK_FUNCTION inline @@ -207,7 +209,7 @@ struct DynamicBucket { class DynamicMesh { public: - typedef stk::mesh::ExecSpace MeshExecSpace; + typedef stk::ngp::ExecSpace MeshExecSpace; typedef DynamicBucket::ConnectedNodes ConnectedNodes; typedef DynamicBucket BucketType; @@ -234,7 +236,7 @@ class DynamicMesh { indexOfFirstEmptyBucket = UnsignedViewType("indexOfFirstEmptyBucket",stk::topology::NUM_RANKS); hostIndexOfFirstEmptyBucket = Kokkos::create_mirror_view(indexOfFirstEmptyBucket); - hostMeshIndices = Kokkos::View::HostMirror("host_mesh_indices", bulk->get_size_of_entity_index_space()); + hostMeshIndices = Kokkos::View::HostMirror("host_mesh_indices", bulk->get_size_of_entity_index_space()); hostEntityRanks = RankViewType::HostMirror("host_entity_ranks", bulk->get_size_of_entity_index_space()); stk::mesh::EntityRank endRank = static_cast(bulk->mesh_meta_data().entity_rank_count()); @@ -470,9 +472,9 @@ class DynamicMesh Kokkos::deep_copy(buckets[rank], hostBuckets[rank]); - Kokkos::parallel_for(Kokkos::RangePolicy(0,numStkBuckets), *this); - Kokkos::parallel_for(Kokkos::RangePolicy(0,numStkBuckets), *this); - Kokkos::parallel_for(Kokkos::RangePolicy(0,numStkBuckets), *this); + Kokkos::parallel_for(Kokkos::RangePolicy(0,numStkBuckets), *this); + Kokkos::parallel_for(Kokkos::RangePolicy(0,numStkBuckets), *this); + Kokkos::parallel_for(Kokkos::RangePolicy(0,numStkBuckets), *this); Kokkos::deep_copy(indexOfFirstEmptyBucket, hostIndexOfFirstEmptyBucket); Kokkos::deep_copy(hostBuckets[rank], buckets[rank]); @@ -538,7 +540,7 @@ class DynamicMesh void copy_mesh_indices_and_ranks_to_device() { unsigned length = hostMeshIndices.size(); - Kokkos::View tmp_device_mesh_indices("tmp_dev_mesh_indices", length); + Kokkos::View tmp_device_mesh_indices("tmp_dev_mesh_indices", length); Kokkos::deep_copy(tmp_device_mesh_indices, hostMeshIndices); deviceMeshIndices = tmp_device_mesh_indices; RankViewType tmp_device_entity_ranks("tmp_dev_ent_ranks", length); @@ -552,7 +554,7 @@ class DynamicMesh } - typedef Kokkos::View BucketView; + typedef Kokkos::View BucketView; const stk::mesh::BulkData *bulk; unsigned maxNumParts; @@ -577,8 +579,8 @@ class DynamicMesh UnsignedViewType indexOfFirstEmptyBucket; UnsignedViewType::HostMirror hostIndexOfFirstEmptyBucket; BucketView::HostMirror hostBuckets[stk::topology::NUM_RANKS]; - Kokkos::View::HostMirror hostMeshIndices; - Kokkos::View deviceMeshIndices; + Kokkos::View::HostMirror hostMeshIndices; + Kokkos::View deviceMeshIndices; RankViewType deviceEntityRanks; RankViewType::HostMirror hostEntityRanks; }; diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpFieldBase.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpFieldBase.hpp index b2b7f6454913..0f25de211dc0 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpFieldBase.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpFieldBase.hpp @@ -3,7 +3,7 @@ #include "stk_util/stk_config.h" #include "stk_mesh/base/Selector.hpp" -#include "stk_mesh/base/NgpSpaces.hpp" +#include "stk_util/ngp/NgpSpaces.hpp" #include "stk_mesh/base/NgpTypes.hpp" #include @@ -29,13 +29,13 @@ class NgpFieldBase virtual void clear_host_sync_state() = 0; virtual void clear_device_sync_state() = 0; virtual void sync_to_host() = 0; - virtual void sync_to_host(const ExecSpace& execSpace) = 0; + virtual void sync_to_host(const stk::ngp::ExecSpace& execSpace) = 0; virtual void sync_to_device() = 0; - virtual void sync_to_device(const ExecSpace& execSpace) = 0; + virtual void sync_to_device(const stk::ngp::ExecSpace& execSpace) = 0; virtual size_t synchronized_count() const = 0; virtual size_t num_syncs_to_host() const = 0; virtual size_t num_syncs_to_device() const = 0; - virtual void set_execution_space(const ExecSpace& execSpace) = 0; + virtual void set_execution_space(const stk::ngp::ExecSpace& execSpace) = 0; virtual void fence() = 0; virtual void debug_modification_begin() = 0; diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpFieldParallel.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpFieldParallel.hpp index a2d3a0291e61..91a1c4984d08 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpFieldParallel.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpFieldParallel.hpp @@ -160,10 +160,10 @@ struct NgpFieldInfo }; template -using FieldDataViewType = Kokkos::View; +using FieldDataViewType = Kokkos::View; template -using FieldView = Kokkos::View*, MemSpace>; +using FieldView = Kokkos::View*, stk::ngp::MemSpace>; template class ParallelSumDataExchangeSymPackUnpackHandler diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpFieldSyncDebugger.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpFieldSyncDebugger.hpp index 9c798111558b..e3644bb73211 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpFieldSyncDebugger.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpFieldSyncDebugger.hpp @@ -38,7 +38,7 @@ #include "stk_mesh/base/BulkData.hpp" #include "stk_mesh/base/Types.hpp" #include "stk_mesh/base/Entity.hpp" -#include "stk_mesh/base/NgpSpaces.hpp" +#include "stk_util/ngp/NgpSpaces.hpp" #include "stk_mesh/base/FieldBase.hpp" #include "stk_mesh/base/DeviceMesh.hpp" #include "stk_mesh/base/NgpFieldBase.hpp" @@ -535,7 +535,7 @@ class NgpFieldSyncDebugger } - using FieldNameType = Kokkos::View; + using FieldNameType = Kokkos::View; template using HostArrayType = Kokkos::View; StkFieldSyncDebuggerType* m_stkFieldSyncDebugger; diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpForEachEntity.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpForEachEntity.hpp index dad6c3bb5dc2..c1e6cb5030af 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpForEachEntity.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpForEachEntity.hpp @@ -37,7 +37,7 @@ #include #include #include -#include +#include #include namespace stk { @@ -70,7 +70,7 @@ struct TeamFunctor bucketIds(b), functor(f) {} - typedef typename Kokkos::TeamPolicy::member_type TeamHandleType; + typedef typename Kokkos::TeamPolicy::member_type TeamHandleType; STK_FUNCTION void operator()(const TeamHandleType& team) const { diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpMesh.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpMesh.hpp index ea4f91caa036..3540d35c2117 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpMesh.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpMesh.hpp @@ -47,7 +47,7 @@ #include #include -#include +#include #include #include diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpParallelComm.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpParallelComm.hpp index 96502dba975a..10fa57db928a 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpParallelComm.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpParallelComm.hpp @@ -35,18 +35,18 @@ #define STK_MESH_NGPPARALLELCOMM_HPP #include "stk_util/parallel/Parallel.hpp" // for ParallelMachine -#include "stk_mesh/base/NgpSpaces.hpp" +#include "stk_util/ngp/NgpSpaces.hpp" #include "Kokkos_Core.hpp" namespace stk { namespace mesh { -using CommProcsViewType = Kokkos::View; +using CommProcsViewType = Kokkos::View; -using OffsetViewType = Kokkos::View; +using OffsetViewType = Kokkos::View; template -using BufferViewType = Kokkos::View; +using BufferViewType = Kokkos::View; template void ngp_parallel_data_exchange_sym_pack_unpack(MPI_Comm mpi_communicator, diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpReductions.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpReductions.hpp index 55877620d608..92182e3e26e2 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpReductions.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpReductions.hpp @@ -38,7 +38,7 @@ #include #include #include -#include +#include namespace stk { namespace mesh { @@ -146,7 +146,7 @@ struct ReductionTeamFunctor ReductionTeamFunctor(const Mesh m, stk::NgpVector b, Accessor a) : mesh(m), bucketIds(b), accessor(a) {} - using TeamHandleType = typename Kokkos::TeamPolicy::member_type; + using TeamHandleType = typename Kokkos::TeamPolicy::member_type; STK_FUNCTION void join(value_type& dest, const value_type& src) const { accessor.reduction.join(dest,src); diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpSpaces.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpSpaces.hpp index a61f381ed7e9..bf8d776ff83e 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpSpaces.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpSpaces.hpp @@ -35,6 +35,7 @@ #define STK_MESH_NGPSPACES_HPP #include +#include #ifdef KOKKOS_ENABLE_OPENMP #include @@ -43,45 +44,12 @@ namespace stk { namespace mesh { -#ifdef KOKKOS_ENABLE_CUDA -using ExecSpace = Kokkos::Cuda; -#elif defined(KOKKOS_ENABLE_OPENMP) -using ExecSpace = Kokkos::OpenMP; -#else -using ExecSpace = Kokkos::Serial; -#endif - -#ifdef KOKKOS_ENABLE_CUDA -using HostExecSpace = Kokkos::Serial; -#elif defined(KOKKOS_ENABLE_OPENMP) -using HostExecSpace = Kokkos::OpenMP; -#else -using HostExecSpace = Kokkos::Serial; -#endif - -#ifdef KOKKOS_ENABLE_CUDA -using MemSpace = Kokkos::CudaSpace; -#elif defined(KOKKOS_ENABLE_OPENMP) -using MemSpace = Kokkos::OpenMP; -#else -using MemSpace = Kokkos::HostSpace; -#endif - -#ifdef KOKKOS_ENABLE_CUDA -using UVMMemSpace = Kokkos::CudaUVMSpace; -#elif defined(KOKKOS_ENABLE_OPENMP) -using UVMMemSpace = Kokkos::OpenMP; -#else -using UVMMemSpace = Kokkos::HostSpace; -#endif - -#ifdef KOKKOS_ENABLE_CUDA -using HostPinnedSpace = Kokkos::CudaHostPinnedSpace; -#else -using HostPinnedSpace = MemSpace; -#endif - -using ScheduleType = Kokkos::Schedule; +using ExecSpace [[deprecated("Please use stk_util/ngp/NgpSpaces.hpp instead of stk_mesh/base/NgpSpaces.hpp")]] = stk::ngp::ExecSpace; +using HostExecSpace [[deprecated("Please use stk_util/ngp/NgpSpaces.hpp instead of stk_mesh/base/NgpSpaces.hpp")]] = stk::ngp::HostExecSpace; +using MemSpace [[deprecated("Please use stk_util/ngp/NgpSpaces.hpp instead of stk_mesh/base/NgpSpaces.hpp")]] = stk::ngp::MemSpace; +using UVMMemSpace [[deprecated("Please use stk_util/ngp/NgpSpaces.hpp instead of stk_mesh/base/NgpSpaces.hpp")]] = stk::ngp::UVMMemSpace; +using HostPinnedSpace [[deprecated("Please use stk_util/ngp/NgpSpaces.hpp instead of stk_mesh/base/NgpSpaces.hpp")]] = stk::ngp::HostPinnedSpace; +using ScheduleType [[deprecated("Please use stk_util/ngp/NgpSpaces.hpp instead of stk_mesh/base/NgpSpaces.hpp")]] = stk::ngp::ScheduleType; } } diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpTypes.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpTypes.hpp index 203fd1f498d8..9d08fc6815bf 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpTypes.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpTypes.hpp @@ -35,33 +35,33 @@ #ifndef NGPTYPES_HPP #define NGPTYPES_HPP -#include +#include #include "stk_mesh/base/Types.hpp" #include namespace stk { namespace mesh { -using DeviceCommMapIndices = Kokkos::View; -using EntityKeyViewType = Kokkos::View; -using EntityViewType = Kokkos::View; -using BucketConnectivityType = Kokkos::View; -using UnsignedViewType = Kokkos::View; -using BoolViewType = Kokkos::View; -using OrdinalViewType = Kokkos::View; -using PartOrdinalViewType = Kokkos::View; -using PermutationViewType = Kokkos::View; -using FastSharedCommMapViewType = Kokkos::View; +using DeviceCommMapIndices = Kokkos::View; +using EntityKeyViewType = Kokkos::View; +using EntityViewType = Kokkos::View; +using BucketConnectivityType = Kokkos::View; +using UnsignedViewType = Kokkos::View; +using BoolViewType = Kokkos::View; +using OrdinalViewType = Kokkos::View; +using PartOrdinalViewType = Kokkos::View; +using PermutationViewType = Kokkos::View; +using FastSharedCommMapViewType = Kokkos::View; using HostMeshIndexType = Kokkos::View::HostMirror; -using MeshIndexType = Kokkos::View>; +using MeshIndexType = Kokkos::View>; -template using FieldDataDeviceViewType = Kokkos::View; -template using FieldDataHostViewType = Kokkos::View; +template using FieldDataDeviceViewType = Kokkos::View; +template using FieldDataHostViewType = Kokkos::View; template using UnmanagedHostInnerView = Kokkos::View>; -template using UnmanagedDevInnerView = Kokkos::View>; +template using UnmanagedDevInnerView = Kokkos::View>; -#ifdef KOKKOS_ENABLE_CUDA +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) #define ORDER_INDICES(i,j) j,i #else #define ORDER_INDICES(i,j) i,j diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpUtils.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpUtils.hpp index babdfefa713a..8cf4a8cbbe34 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpUtils.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpUtils.hpp @@ -88,10 +88,10 @@ inline stk::NgpVector get_bucket_ids(const stk::mesh::BulkData &bulk, } template -void transpose_contiguous_device_data_into_buffer(stk::mesh::ExecSpace & execSpace, unsigned numEntitiesInBlock, unsigned numPerEntity, +void transpose_contiguous_device_data_into_buffer(stk::ngp::ExecSpace & execSpace, unsigned numEntitiesInBlock, unsigned numPerEntity, ViewType & deviceView, ViewType & bufferView) { - const auto& rangePolicy = Kokkos::RangePolicy(execSpace, 0, numEntitiesInBlock); + const auto& rangePolicy = Kokkos::RangePolicy(execSpace, 0, numEntitiesInBlock); Kokkos::parallel_for("transpose_contiguous_device_data_into_buffer", rangePolicy, KOKKOS_LAMBDA(const int& entityIdx) { @@ -103,10 +103,10 @@ void transpose_contiguous_device_data_into_buffer(stk::mesh::ExecSpace & execSpa } template -void transpose_buffer_into_contiguous_device_data(stk::mesh::ExecSpace & execSpace, unsigned numEntitiesInBlock, unsigned numPerEntity, +void transpose_buffer_into_contiguous_device_data(stk::ngp::ExecSpace & execSpace, unsigned numEntitiesInBlock, unsigned numPerEntity, ViewType & bufferView, ViewType & deviceView) { - const auto& rangePolicy = Kokkos::RangePolicy(execSpace, 0, numEntitiesInBlock); + const auto& rangePolicy = Kokkos::RangePolicy(execSpace, 0, numEntitiesInBlock); Kokkos::parallel_for("transpose_buffer_into_contiguous_device_data", rangePolicy, KOKKOS_LAMBDA(const int& entityIdx) { @@ -128,7 +128,7 @@ void transpose_all_device_data_into_buffer(ExecSpaceType & execSpace, stk::mesh::Selector selector = stk::mesh::selectField(stkField); size_t numBuckets = bucketSizes.extent(0); - typedef typename Kokkos::TeamPolicy::member_type TeamHandleType; + typedef typename Kokkos::TeamPolicy::member_type TeamHandleType; const auto& teamPolicy = Kokkos::TeamPolicy(execSpace, numBuckets, Kokkos::AUTO); Kokkos::parallel_for("transpose_all_device_data_into_buffer", teamPolicy, KOKKOS_LAMBDA(const TeamHandleType & team) { @@ -156,7 +156,7 @@ void transpose_buffer_into_all_device_data(ExecSpaceType & execSpace, stk::mesh::Selector selector = stk::mesh::selectField(stkField); size_t numBuckets = bucketSizes.extent(0); - typedef typename Kokkos::TeamPolicy::member_type TeamHandleType; + typedef typename Kokkos::TeamPolicy::member_type TeamHandleType; const auto& teamPolicy = Kokkos::TeamPolicy(execSpace, numBuckets, Kokkos::AUTO); Kokkos::parallel_for("transpose_buffer_into_all_device_data", teamPolicy, KOKKOS_LAMBDA(const TeamHandleType & team) { diff --git a/packages/stk/stk_mesh/stk_mesh/base/Stencils.hpp b/packages/stk/stk_mesh/stk_mesh/base/Stencils.hpp index f416684fb80a..2b31bcc03169 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/Stencils.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/Stencils.hpp @@ -44,31 +44,6 @@ namespace stk { namespace mesh { -#ifndef STK_HIDE_DEPRECATED_CODE // Delete after April 2021 and delete this header after trilinos/percept is updated - -STK_DEPRECATED relation_stencil_ptr get_element_node_stencil(size_t spatial_dimension); - -template -STK_DEPRECATED int element_node_stencil( EntityRank , EntityRank , unsigned ); - - -template -STK_DEPRECATED int element_node_stencil( EntityRank from_type , EntityRank to_type , unsigned identifier ) -{ - enum { number_node = TopologyTraits::node_count }; - - int ordinal = -1 ; - - if ( element_rank == from_type && - stk::topology::NODE_RANK == to_type && - identifier < number_node ) { - ordinal = static_cast(identifier); - } - - return ordinal ; -} - -#endif // STK_HIDE_DEPRECATED_CODE } // namespace mesh } // namespace stk diff --git a/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshImplUtils.cpp b/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshImplUtils.cpp index 15ccb7f0e178..c19ada76591c 100644 --- a/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshImplUtils.cpp +++ b/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshImplUtils.cpp @@ -1678,6 +1678,57 @@ void connect_face_to_elements(stk::mesh::BulkData& bulk, stk::mesh::Entity face) "Face with id: " << bulk.identifier(face) << " has no valid connectivity to elements"); } +bool has_upward_recv_ghost_connectivity(const stk::mesh::BulkData &bulk, + const stk::mesh::Ghosting& ghosting, + stk::mesh::Entity entity) +{ + if(!bulk.is_valid(entity)) + return false; + + const stk::mesh::EntityRank entityRank = bulk.entity_rank(entity); + const stk::mesh::EntityRank endRank = static_cast(bulk.mesh_meta_data().entity_rank_count()); + for(stk::mesh::EntityRank conRank = static_cast(entityRank + 1); conRank <= endRank; ++conRank) + { + unsigned numConnected = bulk.num_connectivity(entity, conRank); + if(numConnected > 0) { + const stk::mesh::Entity* conn = bulk.begin(entity, conRank); + for(unsigned i=0; i(bulk.mesh_meta_data().entity_rank_count()); + for(stk::mesh::EntityRank conRank = static_cast(entityRank + 1); conRank <= endRank; ++conRank) + { + unsigned numConnected = bulk.num_connectivity(entity, conRank); + if(numConnected > 0) { + const stk::mesh::Entity* conn = bulk.begin(entity, conRank); + for(unsigned i=0; i namespace stk { @@ -51,6 +51,7 @@ enum NgpFieldSyncMode { struct AsyncCopyState { + KOKKOS_FUNCTION AsyncCopyState() : execSpace(Kokkos::DefaultExecutionSpace()), syncMode(INVALID) @@ -62,13 +63,13 @@ struct AsyncCopyState { syncMode(state.syncMode) {} - void set_state(const ExecSpace& space, NgpFieldSyncMode mode) + void set_state(const stk::ngp::ExecSpace& space, NgpFieldSyncMode mode) { execSpace = space; syncMode = mode; } - void set_execution_space(const ExecSpace& space) + void set_execution_space(const stk::ngp::ExecSpace& space) { execSpace = space; } @@ -79,11 +80,11 @@ struct AsyncCopyState { syncMode = INVALID; } - ExecSpace execSpace; + stk::ngp::ExecSpace execSpace; NgpFieldSyncMode syncMode; }; } } } -#endif \ No newline at end of file +#endif diff --git a/packages/stk/stk_mesh/stk_mesh/baseImpl/check_comm_list.cpp b/packages/stk/stk_mesh/stk_mesh/baseImpl/check_comm_list.cpp index 190b449e2cc4..8ebb68f6d9a5 100644 --- a/packages/stk/stk_mesh/stk_mesh/baseImpl/check_comm_list.cpp +++ b/packages/stk/stk_mesh/stk_mesh/baseImpl/check_comm_list.cpp @@ -101,12 +101,12 @@ void check_recvd_key_and_ghost_id_against_expected_recv_data(stk::CommBuffer& bu } } -void check_for_expected_recv_data_that_failed_to_arrive(const std::vector& expected_recv_data, std::ostream& os) +void check_for_expected_recv_data_that_failed_to_arrive(const std::vector& expected_recv_data, int localProc, std::ostream& os) { if (!expected_recv_data.empty()) { for(const KeyProcGhostId& data : expected_recv_data) { if (!data.alreadyFound) { - os << "\tFailed to recv {"<= (b)) #define NGP_ASSERT_GE(a, b) NGP_ASSERT_TRUE((a) >= (b)) +#ifdef __HIP_DEVICE_COMPILE__ +//FIXME: unsupported indirect call to function on HIP-Clang +#define NGP_EXPECT_NEAR(a, b, tolerance) +#define NGP_ASSERT_NEAR(a, b, tolerance) + +#else #define NGP_EXPECT_NEAR(a, b, tolerance) \ do { \ if (!::ngp_testing::internal::expect_near(a, b, tolerance)) { \ @@ -107,7 +120,7 @@ bool expect_near(const T a, const T b, const T tolerance) { return; \ } \ } while (false) - +#endif #include "ngp_test.cpp" diff --git a/packages/stk/stk_performance_tests/stk_mesh/NgpFieldAsync.cpp b/packages/stk/stk_performance_tests/stk_mesh/NgpFieldAsync.cpp index d68249a16507..12a8a0a2a699 100644 --- a/packages/stk/stk_performance_tests/stk_mesh/NgpFieldAsync.cpp +++ b/packages/stk/stk_performance_tests/stk_mesh/NgpFieldAsync.cpp @@ -142,10 +142,10 @@ class NgpFieldAsyncTest : public stk::unit_test_util::MeshFixture stk::performance_tests::move_elements_to_other_blocks(get_bulk(), m_numElemsPerDim); } - void pass_time_on_device(const stk::mesh::ExecSpace& space, unsigned iterationSpent = 100) + void pass_time_on_device(const stk::ngp::ExecSpace& space, unsigned iterationSpent = 100) { - typedef typename Kokkos::TeamPolicy::member_type TeamHandleType; - const auto& teamPolicy = Kokkos::TeamPolicy(space, 10, Kokkos::AUTO); + typedef typename Kokkos::TeamPolicy::member_type TeamHandleType; + const auto& teamPolicy = Kokkos::TeamPolicy(space, 10, Kokkos::AUTO); Kokkos::parallel_for("run_with_team_policy", teamPolicy, KOKKOS_LAMBDA(const TeamHandleType & team) { @@ -350,7 +350,7 @@ TEST_F(NgpFieldAsyncTest, SyncToDeviceAsyncTiming) reset_fields_values_on_host(fields); update_fields_values_on_host(fields); - std::vector> spaces; + std::vector> spaces; for(unsigned i = 0; i < numStreams; i++) { auto space = stk::mesh::get_execution_space_with_stream(); spaces.push_back(space); @@ -431,7 +431,7 @@ TEST_F(NgpFieldAsyncTest, SyncToHostAsyncTiming) set_fields_values_on_device(fields, initialValue); set_fields_values_on_device(fields, setValue); - std::vector> spaces; + std::vector> spaces; for(unsigned i = 0; i < numStreams; i++) { auto space = stk::mesh::get_execution_space_with_stream(); spaces.push_back(space); @@ -510,7 +510,7 @@ TEST_F(NgpFieldAsyncTest, SyncAsyncTiming) reset_fields_values_on_host(fields); update_fields_values_on_host(fields); - std::vector> spaces; + std::vector> spaces; for(unsigned i = 0; i < numStreams; i++) { auto space = stk::mesh::get_execution_space_with_stream(); spaces.push_back(space); @@ -602,7 +602,7 @@ TEST_F(NgpFieldAsyncTest, PartialSyncToDeviceAsyncTiming) reset_fields_values_on_host(fields); update_fields_values_on_host(fields); - std::vector> spaces; + std::vector> spaces; for(unsigned i = 0; i < numStreams; i++) { auto space = stk::mesh::get_execution_space_with_stream(); spaces.push_back(space); @@ -695,7 +695,7 @@ TEST_F(NgpFieldAsyncTest, PartialSyncToHostAsyncTiming) set_fields_values_on_device(fields, initialValue); set_fields_values_on_device(fields, setValue); - std::vector> spaces; + std::vector> spaces; for(unsigned i = 0; i < numStreams; i++) { auto space = stk::mesh::get_execution_space_with_stream(); spaces.push_back(space); @@ -756,7 +756,7 @@ TEST_F(NgpFieldAsyncTest, AsyncDeepCopyTiming) ngpFields.push_back(&ngpIntField); } - std::vector> spaces; + std::vector> spaces; for(unsigned i = 0; i < numStreams; i++) { auto space = stk::mesh::get_execution_space_with_stream(); spaces.push_back(space); diff --git a/packages/stk/stk_performance_tests/stk_mesh/calculate_centroid.hpp b/packages/stk/stk_performance_tests/stk_mesh/calculate_centroid.hpp index 9bbe498cc6e0..710455b3994e 100644 --- a/packages/stk/stk_performance_tests/stk_mesh/calculate_centroid.hpp +++ b/packages/stk/stk_performance_tests/stk_mesh/calculate_centroid.hpp @@ -178,7 +178,7 @@ std::vector get_centroid_average_from_device(stk::mesh::BulkData &bulk, stk::mesh::NgpField& ngpField = stk::mesh::get_updated_ngp_field(centroid); stk::mesh::NgpMesh& ngpMesh = stk::mesh::get_updated_ngp_mesh(bulk); - typedef Kokkos::View DeviceAverageView; + typedef Kokkos::View DeviceAverageView; typedef typename DeviceAverageView::HostMirror HostAverageView; DeviceAverageView deviceAverageView("averageVew", 4); diff --git a/packages/stk/stk_topology/stk_topology/topology.cpp b/packages/stk/stk_topology/stk_topology/topology.cpp index d9c494b19036..3c33899afa62 100644 --- a/packages/stk/stk_topology/stk_topology/topology.cpp +++ b/packages/stk/stk_topology/stk_topology/topology.cpp @@ -32,10 +32,11 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include -#include +#include "stk_topology/topology.hpp" +#include "stk_util/stk_kokkos_macros.h" // for STK_FUNCTION +#include // for operator<<, setw +#include // for operator<<, ostream, basic_ostream, endl, basic_... +#include // for operator<<, string diff --git a/packages/stk/stk_topology/stk_topology/topology.hpp b/packages/stk/stk_topology/stk_topology/topology.hpp index 4623569298d0..dd8db1420a50 100644 --- a/packages/stk/stk_topology/stk_topology/topology.hpp +++ b/packages/stk/stk_topology/stk_topology/topology.hpp @@ -35,7 +35,7 @@ #ifndef STKTOPOLOGY_TOPOLOGY_HPP #define STKTOPOLOGY_TOPOLOGY_HPP -#include -#include +#include "stk_topology/topology_decl.hpp" +#include "stk_topology/topology_defn.hpp" #endif //STKTOPOLOGY_TOPOLOGY_HPP diff --git a/packages/stk/stk_topology/stk_topology/topology_decl.hpp b/packages/stk/stk_topology/stk_topology/topology_decl.hpp index aa98b404004c..414e412a2707 100644 --- a/packages/stk/stk_topology/stk_topology/topology_decl.hpp +++ b/packages/stk/stk_topology/stk_topology/topology_decl.hpp @@ -1,7 +1,8 @@ #ifndef TOPOLOGY_DECL_HPP #define TOPOLOGY_DECL_HPP -#include +// IWYU pragma: private, include "stk_topology/topology.hpp" +#include "stk_util/stk_config.h" #include namespace stk { diff --git a/packages/stk/stk_topology/stk_topology/topology_defn.hpp b/packages/stk/stk_topology/stk_topology/topology_defn.hpp index 5c6faf5eb9a0..6e9a9f9e4b9c 100644 --- a/packages/stk/stk_topology/stk_topology/topology_defn.hpp +++ b/packages/stk/stk_topology/stk_topology/topology_defn.hpp @@ -1,6 +1,7 @@ #ifndef TOPOLOGY_DEFN_HPP #define TOPOLOGY_DEFN_HPP +// IWYU pragma: private, include "stk_topology/topology.hpp" #include "stk_topology/topology_decl.hpp" #include "stk_topology/topology_utils.hpp" #include "stk_topology/apply_functor.hpp" diff --git a/packages/stk/stk_topology/stk_topology/topology_detail/topology_data.cpp b/packages/stk/stk_topology/stk_topology/topology_detail/topology_data.cpp index 9af17be5aba6..d076070e09c5 100644 --- a/packages/stk/stk_topology/stk_topology/topology_detail/topology_data.cpp +++ b/packages/stk/stk_topology/stk_topology/topology_detail/topology_data.cpp @@ -32,7 +32,7 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include +#include "stk_topology/topology_detail/topology_data.hpp" namespace stk { namespace topology_detail { diff --git a/packages/stk/stk_topology/stk_topology/topology_detail/topology_data.hpp b/packages/stk/stk_topology/stk_topology/topology_detail/topology_data.hpp index 3495601e32af..7c453f287e48 100644 --- a/packages/stk/stk_topology/stk_topology/topology_detail/topology_data.hpp +++ b/packages/stk/stk_topology/stk_topology/topology_detail/topology_data.hpp @@ -35,7 +35,8 @@ #ifndef STKTOPOLOGY_DETAIL_TOPOLOGY_DATA_HPP #define STKTOPOLOGY_DETAIL_TOPOLOGY_DATA_HPP -#include +// IWYU pragma: private, include "stk_topology/topology.hpp" +#include "stk_topology/topology_decl.hpp" #include //TODO implement permutations for tets, pyramids, wedges and hexes diff --git a/packages/stk/stk_topology/stk_topology/topology_type.hpp b/packages/stk/stk_topology/stk_topology/topology_type.hpp index 9c83e7ed8e8d..d76bf8bbee8c 100644 --- a/packages/stk/stk_topology/stk_topology/topology_type.hpp +++ b/packages/stk/stk_topology/stk_topology/topology_type.hpp @@ -34,6 +34,7 @@ #ifndef STKTOPOLOGY_TOPOLOGY_TYPE_HPP #define STKTOPOLOGY_TOPOLOGY_TYPE_HPP +// IWYU pragma: private, include "stk_topology/topology.hpp" #include "stk_topology/topology_decl.hpp" #include "stk_topology/topology_utils.hpp" #include "stk_topology/topology_detail/topology_data.hpp" diff --git a/packages/stk/stk_topology/stk_topology/topology_utils.hpp b/packages/stk/stk_topology/stk_topology/topology_utils.hpp index 4f62193c7bc4..13afdaf62d8d 100644 --- a/packages/stk/stk_topology/stk_topology/topology_utils.hpp +++ b/packages/stk/stk_topology/stk_topology/topology_utils.hpp @@ -35,6 +35,7 @@ #ifndef STKTOPOLOGY_TOPOLOGY_TCC #define STKTOPOLOGY_TOPOLOGY_TCC +// IWYU pragma: private, include "stk_topology/topology.hpp" #include "stk_topology/topology_decl.hpp" #include "stk_util/stk_config.h" diff --git a/packages/stk/stk_unit_tests/stk_mesh/custom_ghosting/UnitTestChangeEntityOwnerKeepUnaffectedCustomGhost.cpp b/packages/stk/stk_unit_tests/stk_mesh/custom_ghosting/UnitTestChangeEntityOwnerKeepUnaffectedCustomGhost.cpp new file mode 100644 index 000000000000..3949321bd449 --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_mesh/custom_ghosting/UnitTestChangeEntityOwnerKeepUnaffectedCustomGhost.cpp @@ -0,0 +1,152 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include +#include +#include +#include +#include +#include + + +class ChangeEntityOwnerTest : public ::testing::Test +{ +public: + ChangeEntityOwnerTest() + : meta(nullptr), bulk(nullptr), customGhosting(nullptr) + { + } + + virtual ~ChangeEntityOwnerTest() + { + delete bulk; + delete meta; + } + + void setup_mesh(stk::ParallelMachine pm) + { + meta = new stk::mesh::MetaData(2); + bulk = new stk::mesh::BulkData(*meta, pm, stk::mesh::BulkData::NO_AUTO_AURA); + create_1_beam_per_proc(); + custom_ghost_node2_to_proc2(); + } + + void create_1_beam_per_proc() + { + const int myProc = stk::parallel_machine_rank(bulk->parallel()); + stk::mesh::Part& beamPart = meta->get_topology_root_part(stk::topology::BEAM_2); + + bulk->modification_begin(); + + stk::mesh::EntityId firstNodeId = myProc + 1; + stk::mesh::Entity node1 = bulk->declare_node(firstNodeId); + stk::mesh::Entity node2 = bulk->declare_node(firstNodeId+1); + if (bulk->parallel_rank()==0) { + int otherProc = 1; + bulk->add_node_sharing(node2, otherProc); + } + if (bulk->parallel_rank()==1) { + int otherProc = 0; + bulk->add_node_sharing(node1, otherProc); + otherProc = 2; + bulk->add_node_sharing(node2, otherProc); + } + if (bulk->parallel_rank()==2) { + int otherProc = 1; + bulk->add_node_sharing(node1, otherProc); + } + + stk::mesh::EntityId beamId = myProc + 1; + + stk::mesh::Entity beam = bulk->declare_element(beamId, stk::mesh::ConstPartVector{&beamPart}); + bulk->declare_relation(beam, node1, 0); + bulk->declare_relation(beam, node2, 1); + + bulk->modification_end(); + } + + void custom_ghost_node2_to_proc2() + { + bulk->modification_begin(); + customGhosting = &bulk->create_ghosting("customGhosting"); + std::vector sendGhost; + if (bulk->parallel_rank() == 0) { + stk::mesh::EntityId nodeId = 2; + const int otherProc = 2; + sendGhost.push_back(stk::mesh::EntityProc(bulk->get_entity(stk::topology::NODE_RANK, nodeId), otherProc)); + } + bulk->change_ghosting(*customGhosting, sendGhost); + bulk->modification_end(); + } + + void move_beam2_to_proc0() + { + std::vector elemToMove; + + if (bulk->parallel_rank()==1) { + const int otherProc = 0; + stk::mesh::EntityId beamId = 2; + elemToMove.push_back(stk::mesh::EntityProc(bulk->get_entity(stk::topology::ELEM_RANK, beamId), otherProc)); + } + + bulk->change_entity_owner(elemToMove); + } + +protected: + stk::mesh::MetaData* meta; + stk::mesh::BulkData* bulk; + stk::mesh::Ghosting * customGhosting; +}; + +TEST_F(ChangeEntityOwnerTest, keepUnaffectedCustomGhost) +{ + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 3) { GTEST_SKIP(); } + + setup_mesh(MPI_COMM_WORLD); + + stk::mesh::EntityKey node2key(stk::topology::NODE_RANK, 2); + stk::mesh::Entity node2 = bulk->get_entity(node2key); + EXPECT_TRUE(bulk->is_valid(node2)); + + if (bulk->parallel_rank() == 0) { + EXPECT_TRUE(bulk->in_send_ghost(*customGhosting, node2key, 2)); + } + + move_beam2_to_proc0(); + + if (bulk->parallel_rank() == 0) { + EXPECT_TRUE(bulk->in_send_ghost(*customGhosting, node2key, 2)); + } +} + diff --git a/packages/stk/stk_unit_tests/stk_mesh/custom_ghosting/UnitTestRemoveNeededRecvGhost.cpp b/packages/stk/stk_unit_tests/stk_mesh/custom_ghosting/UnitTestRemoveNeededRecvGhost.cpp new file mode 100644 index 000000000000..05272968e1a6 --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_mesh/custom_ghosting/UnitTestRemoveNeededRecvGhost.cpp @@ -0,0 +1,143 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include +#include +#include +#include +#include +#include + + +class CustomGhostTest : public ::testing::Test +{ +public: + CustomGhostTest() + : meta(nullptr), bulk(nullptr), customGhosting(nullptr) + { + } + + virtual ~CustomGhostTest() + { + delete bulk; + delete meta; + } + + void setup_mesh(stk::ParallelMachine pm) + { + meta = new stk::mesh::MetaData(2); + bulk = new stk::mesh::BulkData(*meta, pm, stk::mesh::BulkData::NO_AUTO_AURA); + create_2_beams_per_proc(); + custom_ghost_beam1_to_proc1(); + } + + void create_2_beams_per_proc() + { + const int myProc = stk::parallel_machine_rank(bulk->parallel()); + stk::mesh::Part& beamPart = meta->get_topology_root_part(stk::topology::BEAM_2); + + bulk->modification_begin(); + + const unsigned numLocalNodes = 3; + stk::mesh::EntityId firstNodeId = myProc*numLocalNodes + 1; + stk::mesh::Entity node1 = bulk->declare_node(firstNodeId); + stk::mesh::Entity node2 = bulk->declare_node(firstNodeId+1); + stk::mesh::Entity node3 = bulk->declare_node(firstNodeId+2); + + const unsigned numLocalBeams = 2; + stk::mesh::EntityId firstBeamId = myProc*numLocalBeams + 1; + + stk::mesh::Entity beam1 = bulk->declare_element(firstBeamId, stk::mesh::ConstPartVector{&beamPart}); + bulk->declare_relation(beam1, node1, 0); + bulk->declare_relation(beam1, node2, 1); + + stk::mesh::Entity beam2 = bulk->declare_element(firstBeamId+1, stk::mesh::ConstPartVector{&beamPart}); + bulk->declare_relation(beam2, node2, 0); + bulk->declare_relation(beam2, node3, 1); + + bulk->modification_end(); + } + + void custom_ghost_beam1_to_proc1() + { + bulk->modification_begin(); + customGhosting = &bulk->create_ghosting("customGhosting"); + std::vector sendGhost; + if (bulk->parallel_rank() == 0) { + const int otherProc = 1; + stk::mesh::EntityId beamId = 1; + sendGhost.push_back(stk::mesh::EntityProc(bulk->get_entity(stk::topology::ELEM_RANK, beamId), otherProc)); + } + bulk->change_ghosting(*customGhosting, sendGhost); + bulk->modification_end(); + } + + void custom_ghost_rm_beam1_add_beam2() + { + std::vector sendGhost; + std::vector rmRecvGhost; + + if (bulk->parallel_rank()==0) { + const int otherProc = 1; + stk::mesh::EntityId beamId = 2; + sendGhost.push_back(stk::mesh::EntityProc(bulk->get_entity(stk::topology::ELEM_RANK, beamId), otherProc)); + } + else { + stk::mesh::EntityId beamId = 1; + rmRecvGhost.push_back(stk::mesh::EntityKey(stk::topology::ELEM_RANK, beamId)); + stk::mesh::EntityId nodeId = 1; + rmRecvGhost.push_back(stk::mesh::EntityKey(stk::topology::NODE_RANK, nodeId)); + rmRecvGhost.push_back(stk::mesh::EntityKey(stk::topology::NODE_RANK, nodeId+1)); + } + + bulk->modification_begin(); + bulk->change_ghosting(*customGhosting, sendGhost, rmRecvGhost); + bulk->modification_end(); + } + +private: + stk::mesh::MetaData* meta; + stk::mesh::BulkData* bulk; + stk::mesh::Ghosting * customGhosting; +}; + +TEST_F(CustomGhostTest, removeNeededRecvGhost) +{ + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 2) { GTEST_SKIP(); } + + setup_mesh(MPI_COMM_WORLD); + + EXPECT_NO_THROW(custom_ghost_rm_beam1_add_beam2()); +} + diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/KokkosCentroidCalculation.hpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/KokkosCentroidCalculation.hpp index 0ae921237d03..fe3d67a331ba 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/KokkosCentroidCalculation.hpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/KokkosCentroidCalculation.hpp @@ -53,7 +53,7 @@ namespace { -#if defined(KOKKOS_ENABLE_CUDA) +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) typedef double my_double; #else typedef long double my_double; @@ -63,6 +63,8 @@ typedef long double my_double; typedef Kokkos::OpenMP ExecSpace ; #elif defined(KOKKOS_ENABLE_CUDA) typedef Kokkos::Cuda ExecSpace ; +#elif defined(KOKKOS_ENABLE_HIP) +typedef Kokkos::Experimental::HIP ExecSpace ; #else typedef Kokkos::Serial ExecSpace ; #endif @@ -79,6 +81,8 @@ typedef Kokkos::HostSpace MemSpace; typedef Kokkos::OpenMP UVMMemSpace; #elif defined(KOKKOS_ENABLE_CUDA) typedef Kokkos::CudaUVMSpace UVMMemSpace; +#elif defined(KOKKOS_ENABLE_HIP) +typedef Kokkos::Experimental::HIPHostPinnedSpace UVMMemSpace; #else typedef Kokkos::HostSpace UVMMemSpace; #endif @@ -92,7 +96,7 @@ typedef Kokkos::View HostViewVectorType; typedef Kokkos::TeamPolicy team_policy ; typedef Kokkos::TeamPolicy::member_type member_type ; -#if defined(KOKKOS_ENABLE_CUDA) +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) typedef Kokkos::LayoutLeft Layout ; #else typedef Kokkos::LayoutRight Layout ; diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/KokkosNodalVolume.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/KokkosNodalVolume.cpp index 4a8a81029362..000f4f891f1c 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/KokkosNodalVolume.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/KokkosNodalVolume.cpp @@ -95,7 +95,7 @@ void calculate_nodal_volume_given_elem_nodes_stkmesh(const stk::mesh::Entity* el } } -typedef Kokkos::TeamPolicy HostTeamPolicyType; +typedef Kokkos::TeamPolicy HostTeamPolicyType; typedef HostTeamPolicyType::member_type HostTeamHandleType; #ifndef KOKKOS_ENABLE_CUDA @@ -108,7 +108,7 @@ void calculate_nodal_volume_stkmesh(stk::mesh::BulkData& mesh, stk::mesh::Field< unsigned numBuckets = elemBuckets.size(); for(int n=0; n(numBuckets, Kokkos::AUTO), + Kokkos::parallel_for(Kokkos::TeamPolicy< stk::ngp::HostExecSpace >(numBuckets, Kokkos::AUTO), [&] (const HostTeamHandleType& team) { const int elementBucketIndex = team.league_rank(); @@ -259,7 +259,7 @@ void assemble_nodal_volume_from_elem_volume_bucket_field_access(stk::mesh::NgpMe const stk::mesh::NgpField &elemVolumePerNode, stk::mesh::NgpField &staticNodalVolume) { - typedef Kokkos::TeamPolicy::member_type TeamHandleType; + typedef Kokkos::TeamPolicy::member_type TeamHandleType; unsigned numBuckets = ngpMesh.num_buckets(stk::topology::ELEM_RANK); Kokkos::parallel_for(Kokkos::TeamPolicy(numBuckets, Kokkos::AUTO), KOKKOS_LAMBDA (const TeamHandleType& team) diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpFieldAsyncTest.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpFieldAsyncTest.cpp index 6d297f1a325e..6922b0db02b7 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpFieldAsyncTest.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpFieldAsyncTest.cpp @@ -51,7 +51,7 @@ #include #include #include -#include +#include #include #include #include "NgpUnitTestUtils.hpp" diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpMeshTest.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpMeshTest.cpp index 0f6e3950e32d..4a77fab77c33 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpMeshTest.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpMeshTest.cpp @@ -44,7 +44,7 @@ #include "stk_ngp_test/ngp_test.hpp" #include #include -#include +#include #include #include #include @@ -171,7 +171,7 @@ void check_volatile_fast_shared_comm_map_values_on_device(const stk::mesh::NgpMe }); } -using HostCommMapIndices = Kokkos::View; +using HostCommMapIndices = Kokkos::View; NGP_TEST_F(NgpMeshTest, volatileFastSharedCommMap) { diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpParallelCommTest.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpParallelCommTest.cpp index d1c7944255d7..a76ce098af36 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpParallelCommTest.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpParallelCommTest.cpp @@ -4,7 +4,7 @@ #include "Kokkos_Core.hpp" #include #include "stk_mesh/base/NgpParallelComm.hpp" -#include "stk_mesh/base/NgpSpaces.hpp" +#include "stk_util/ngp/NgpSpaces.hpp" namespace { @@ -16,8 +16,8 @@ class NgpParallelComm : public ::ngp_testing::Test } }; -using NeighborCrsViewType = Kokkos::StaticCrsGraph; -using ValueViewType = Kokkos::View; +using NeighborCrsViewType = Kokkos::StaticCrsGraph; +using ValueViewType = Kokkos::View; class ParallelDataExchangeSymPackUnpackHandler { diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpUnitTestUtils.hpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpUnitTestUtils.hpp index baf3b1404e4a..8c87cd7c4596 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpUnitTestUtils.hpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpUnitTestUtils.hpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include @@ -84,7 +84,7 @@ inline void check_bucket_layout(const stk::mesh::BulkData& bulk, const std::vect } } - using BucketPartOrdinalType = Kokkos::View; + using BucketPartOrdinalType = Kokkos::View; BucketPartOrdinalType bucketPartOrdinals("bucketPartOrdinals", numBuckets); BucketPartOrdinalType::HostMirror hostBucketPartOrdinals = Kokkos::create_mirror_view(bucketPartOrdinals); for (size_t i = 0; i < buckets.size(); ++i) { @@ -100,7 +100,7 @@ inline void check_bucket_layout(const stk::mesh::BulkData& bulk, const std::vect } }); - using BucketEntitiesType = Kokkos::View; + using BucketEntitiesType = Kokkos::View; BucketEntitiesType bucketEntities("bucketEntities", numElemsAcrossBuckets+numBuckets); BucketEntitiesType::HostMirror hostBucketEntities = Kokkos::create_mirror_view(bucketEntities); size_t index = 0; diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/howToNgp.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/howToNgp.cpp index fa878fa700d7..669117606bf4 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/howToNgp.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/howToNgp.cpp @@ -25,7 +25,7 @@ namespace { -using IntDualViewType = Kokkos::DualView; +using IntDualViewType = Kokkos::DualView; void set_field_on_device_and_copy_back(stk::mesh::BulkData &bulk, stk::mesh::EntityRank rank, @@ -116,7 +116,7 @@ void test_field_on_subset_of_mesh(const stk::mesh::BulkData& bulk, const FieldTy stk::mesh::PartOrdinal partThatDoesntHaveField) { stk::mesh::NgpMesh & ngpMesh = stk::mesh::get_updated_ngp_mesh(bulk); - typedef Kokkos::TeamPolicy::member_type TeamHandleType; + typedef Kokkos::TeamPolicy::member_type TeamHandleType; const auto& teamPolicy = Kokkos::TeamPolicy(ngpMesh.num_buckets(stk::topology::ELEM_RANK), Kokkos::AUTO); @@ -213,7 +213,7 @@ void run_connected_node_test(const stk::mesh::BulkData& bulk) const stk::mesh::NgpMesh & ngpMesh = stk::mesh::get_updated_ngp_mesh(bulk); - typedef Kokkos::TeamPolicy::member_type TeamHandleType; + typedef Kokkos::TeamPolicy::member_type TeamHandleType; const auto& teamPolicy = Kokkos::TeamPolicy(ngpMesh.num_buckets(stk::topology::ELEM_RANK), Kokkos::AUTO); @@ -279,7 +279,7 @@ void run_id_test(const stk::mesh::BulkData& bulk) { const stk::mesh::NgpMesh & ngpMesh = stk::mesh::get_updated_ngp_mesh(bulk); - typedef Kokkos::TeamPolicy::member_type TeamHandleType; + typedef Kokkos::TeamPolicy::member_type TeamHandleType; const auto& teamPolicy = Kokkos::TeamPolicy(ngpMesh.num_buckets(stk::topology::ELEM_RANK), Kokkos::AUTO); @@ -339,7 +339,7 @@ void run_connected_face_test(const stk::mesh::BulkData& bulk) const stk::mesh::NgpMesh & ngpMesh = stk::mesh::get_updated_ngp_mesh(bulk); - typedef Kokkos::TeamPolicy::member_type TeamHandleType; + typedef Kokkos::TeamPolicy::member_type TeamHandleType; const auto& teamPolicy = Kokkos::TeamPolicy(ngpMesh.num_buckets(stk::topology::ELEM_RANK), Kokkos::AUTO); @@ -406,7 +406,7 @@ void run_connected_face_ordinal_test(const stk::mesh::BulkData& bulk) const stk::mesh::NgpMesh & ngpMesh = stk::mesh::get_updated_ngp_mesh(bulk); - typedef Kokkos::TeamPolicy::member_type TeamHandleType; + typedef Kokkos::TeamPolicy::member_type TeamHandleType; const auto& teamPolicy = Kokkos::TeamPolicy(ngpMesh.num_buckets(stk::topology::ELEM_RANK), Kokkos::AUTO); @@ -484,7 +484,7 @@ void run_connected_face_permutation_test(const stk::mesh::BulkData& bulk) const stk::mesh::NgpMesh & ngpMesh = stk::mesh::get_updated_ngp_mesh(bulk); - typedef Kokkos::TeamPolicy::member_type TeamHandleType; + typedef Kokkos::TeamPolicy::member_type TeamHandleType; const auto& teamPolicy = Kokkos::TeamPolicy(ngpMesh.num_buckets(stk::topology::ELEM_RANK), Kokkos::AUTO); @@ -554,7 +554,7 @@ void run_another_connected_face_test(const stk::mesh::BulkData& bulk) const stk::mesh::NgpMesh & ngpMesh = stk::mesh::get_updated_ngp_mesh(bulk); - typedef Kokkos::TeamPolicy::member_type TeamHandleType; + typedef Kokkos::TeamPolicy::member_type TeamHandleType; const auto& teamPolicy = Kokkos::TeamPolicy(ngpMesh.num_buckets(stk::topology::ELEM_RANK), Kokkos::AUTO); Kokkos::parallel_for(teamPolicy, @@ -611,7 +611,7 @@ void test_view_of_fields(const stk::mesh::BulkData& bulk, stk::mesh::Field& field1, stk::mesh::Field& field2) { - using FieldViewType = Kokkos::View*,stk::mesh::MemSpace>; + using FieldViewType = Kokkos::View*,stk::ngp::MemSpace>; FieldViewType fields(Kokkos::ViewAllocateWithoutInitializing("fields"),2); FieldViewType::HostMirror hostFields = Kokkos::create_mirror_view(fields); @@ -707,14 +707,14 @@ unsigned count_num_elems(stk::mesh::NgpMesh ngpMesh, stk::mesh::EntityRank rank, stk::mesh::Part &part) { - Kokkos::View numElems("numElems", 1); + Kokkos::View numElems("numElems", 1); stk::mesh::for_each_entity_run(ngpMesh, rank, part, KOKKOS_LAMBDA(const stk::mesh::FastMeshIndex& entity) { unsigned fieldValue = static_cast(ngpField(entity, 0)); Kokkos::atomic_add(&numElems(0), fieldValue); }); - Kokkos::View::HostMirror numElemsHost = + Kokkos::View::HostMirror numElemsHost = Kokkos::create_mirror_view(numElems); Kokkos::deep_copy(numElemsHost, numElems); return numElemsHost(0); @@ -1124,7 +1124,7 @@ void run_part_membership_test(const stk::mesh::BulkData& bulk, stk::mesh::PartOr { const stk::mesh::NgpMesh & ngpMesh = stk::mesh::get_updated_ngp_mesh(bulk); - typedef Kokkos::TeamPolicy::member_type TeamHandleType; + typedef Kokkos::TeamPolicy::member_type TeamHandleType; const auto& teamPolicy = Kokkos::TeamPolicy(ngpMesh.num_buckets(stk::topology::ELEM_RANK), Kokkos::AUTO); @@ -1259,7 +1259,7 @@ class FakeEntity { size_t m_value; }; -using FakeEntityType = Kokkos::View; +using FakeEntityType = Kokkos::View; class FakeBucket { public: @@ -1301,7 +1301,7 @@ class FakeBucket { FakeEntityType m_innerView; }; -using FakeBuckets = Kokkos::View; +using FakeBuckets = Kokkos::View; class FakeMesh { diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/howToNgpMeshMod.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/howToNgpMeshMod.cpp index 12c7315986be..7de3cc2446f7 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/howToNgpMeshMod.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/howToNgpMeshMod.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -15,7 +15,7 @@ #include #include "NgpUnitTestUtils.hpp" -typedef Kokkos::DualView IntViewType; +typedef Kokkos::DualView IntViewType; class NgpMeshModHowTo : public stk::unit_test_util::MeshFixture {}; diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/implNgpMeshMod.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/implNgpMeshMod.cpp index 557d29d81405..519843e83533 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/implNgpMeshMod.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/implNgpMeshMod.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -14,7 +14,7 @@ #include #include "NgpUnitTestUtils.hpp" -typedef Kokkos::DualView IntViewType; +typedef Kokkos::DualView IntViewType; class NgpMeshModImpl : public stk::unit_test_util::MeshFixture { diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpFieldTest.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpFieldTest.cpp index fddad1825844..b4a466c84e3f 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpFieldTest.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpFieldTest.cpp @@ -269,7 +269,7 @@ class NgpFieldFixture : public stk::unit_test_util::MeshFixture stk::mesh::NgpField& ngpField, stk::mesh::Field& stkField, Func&& checkFunc) { - using FieldData = Kokkos::View; + using FieldData = Kokkos::View; unsigned numElems = elements.size(); unsigned numPerEntity = stkField.max_size(rank); diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpMultiStateFieldTests.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpMultiStateFieldTests.cpp index 625fb8bc4141..89cc02fed60b 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpMultiStateFieldTests.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpMultiStateFieldTests.cpp @@ -78,7 +78,7 @@ class ClassWithNgpField stk::mesh::NgpField m_ngpField; }; -#ifdef KOKKOS_ENABLE_CUDA +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) #define MY_LAMBDA KOKKOS_LAMBDA #else #define MY_LAMBDA [&] @@ -87,7 +87,7 @@ class ClassWithNgpField ClassWithNgpField* create_class_on_device(const stk::mesh::NgpField& ngpField) { ClassWithNgpField* devicePtr = static_cast( - Kokkos::kokkos_malloc("device class memory", sizeof(ClassWithNgpField))); + Kokkos::kokkos_malloc("device class memory", sizeof(ClassWithNgpField))); Kokkos::parallel_for("construct class on device", 1, MY_LAMBDA(const int i) { new (devicePtr) ClassWithNgpField(ngpField); } ); @@ -101,7 +101,7 @@ void delete_class_on_device(ClassWithNgpField* devicePtr) KOKKOS_LAMBDA(const int i) { devicePtr->~ClassWithNgpField(); } ); Kokkos::fence(); - Kokkos::kokkos_free(static_cast(devicePtr)); + Kokkos::kokkos_free(static_cast(devicePtr)); } class NgpMultiStateFieldTest : public stk::mesh::fixtures::TestHexFixture diff --git a/packages/stk/stk_unit_tests/stk_topology/topology_test_utils.hpp b/packages/stk/stk_unit_tests/stk_topology/topology_test_utils.hpp index d9557aefafd6..5b192a606336 100644 --- a/packages/stk/stk_unit_tests/stk_topology/topology_test_utils.hpp +++ b/packages/stk/stk_unit_tests/stk_topology/topology_test_utils.hpp @@ -32,9 +32,9 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include +#include "gtest/gtest.h" +#include "stk_ngp_test/ngp_test.hpp" +#include "stk_topology/topology.hpp" #include "stk_util/util/ReportHandler.hpp" #include #define INVALID UINT_MAX diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_beam.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_beam.cpp index 228a0164b68b..0ceb30cba0a2 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_beam.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_beam.cpp @@ -32,10 +32,12 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include -#include "topology_test_utils.hpp" +#include "Kokkos_Core.hpp" // for parallel_for, KOKKOS_LAMBDA +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_EQ +#include "stk_ngp_test/ngp_test.hpp" // for NGP_EXPECT_EQ, NGP_EXPECT_FALSE, NGP_EXPECT_TRUE +#include "stk_topology/topology.hpp" // for topology, topology::BEAM_2, topology::EDGE_RANK +#include "topology_test_utils.hpp" // for check_edge_node_ordinals, check_edge_node_ordinals_ngp +#include // for vector TEST( stk_topology, beam_2) { diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_hex.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_hex.cpp index 20d619874e6e..9b305593c3cf 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_hex.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_hex.cpp @@ -32,11 +32,15 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include -#include // for cpu_time -#include "topology_test_utils.hpp" +#include "Kokkos_Core.hpp" // for parallel_for, KOKKOS_LAMBDA +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult +#include "stk_ngp_test/ngp_test.hpp" // for NGP_EXPECT_EQ, NGP_EXPECT_FALSE, NGP_EXPECT_... +#include "stk_topology/topology.hpp" // for topology, topology::QUAD_4, topology::QUAD_8 +#include "stk_util/environment/CPUTime.hpp" // for cpu_time +#include "topology_test_utils.hpp" // for check_edge_node_ordinals, check_edge_node_or... +#include // for size_t +#include // for operator<<, basic_ostream, basic_ostream<>::... +#include // for vector TEST(stk_topology, DISABLED_hex_8_defined_on_spatial_dimension_perf) { diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_invalid.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_invalid.cpp index 3f5733822042..435b3a4ec16e 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_invalid.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_invalid.cpp @@ -32,10 +32,10 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include -#include "topology_test_utils.hpp" +#include "Kokkos_Core.hpp" // for parallel_for, KOKKOS_LAMBDA +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_EQ +#include "stk_ngp_test/ngp_test.hpp" // for NGP_EXPECT_EQ, NGP_EXPECT_FALSE, NGP_TEST +#include "stk_topology/topology.hpp" // for topology, topology::INVALID_TOPOLOGY, topology::INV... TEST( stk_topology, invalid_topology) { diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_is_topology.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_is_topology.cpp index 1e2b49a3a9f2..a502b7a20d55 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_is_topology.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_is_topology.cpp @@ -32,8 +32,8 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include +#include "gtest/gtest.h" // for AssertionResult, Test, Message, SuiteApiResolver +#include "stk_topology/topology.hpp" // for topology, topology::END_TOPOLOGY, topology::topology_t TEST(stk_topology, isTri) { diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_line.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_line.cpp index bf5cc7fd1f4c..675d8a6f0a8a 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_line.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_line.cpp @@ -32,11 +32,13 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include +#include "Kokkos_Core.hpp" // for parallel_for, KOKKOS_LAMBDA +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_EQ +#include "stk_ngp_test/ngp_test.hpp" // for NGP_EXPECT_EQ, NGP_EXPECT_FALSE, NGP_EXPECT_TRUE +#include "stk_topology/topology.hpp" // for topology, topology::INVALID_TOPOLOGY, topology::NOD... +#include "topology_test_utils.hpp" // for check_equivalent, check_equivalent_ngp, check_lexic... +#include // for vector -#include -#include -#include "topology_test_utils.hpp" TEST(stk_topology, line_2) { diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_node.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_node.cpp index 646d28d67b3c..816d2ed07d16 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_node.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_node.cpp @@ -32,10 +32,12 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include -#include "topology_test_utils.hpp" +#include "Kokkos_Core.hpp" // for parallel_for, KOKKOS_LAMBDA +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_EQ +#include "stk_ngp_test/ngp_test.hpp" // for NGP_EXPECT_EQ, NGP_EXPECT_TRUE, NGP_EXPECT_FALSE +#include "stk_topology/topology.hpp" // for topology, topology::NODE, topology::INVALID_RANK +#include "topology_test_utils.hpp" // for check_lexicographical_smallest_permutation, check_l... +#include // for vector TEST(stk_topology, node) { diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_particle.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_particle.cpp index 9a4943d36eac..18579d78f021 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_particle.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_particle.cpp @@ -32,10 +32,12 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include -#include "topology_test_utils.hpp" +#include "Kokkos_Core.hpp" // for parallel_for, KOKKOS_LAMBDA +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_EQ +#include "stk_ngp_test/ngp_test.hpp" // for NGP_EXPECT_EQ, NGP_EXPECT_TRUE, NGP_EXPECT_FALSE +#include "stk_topology/topology.hpp" // for topology, topology::PARTICLE, topology::ELEMENT_RANK +#include "topology_test_utils.hpp" // for check_equivalent, check_equivalent_ngp, check_lexic... +#include // for vector TEST(stk_topology, particle) { diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_pyramid.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_pyramid.cpp index 0c24c4a3ef27..9eefc1af89ee 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_pyramid.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_pyramid.cpp @@ -32,10 +32,12 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include -#include "topology_test_utils.hpp" +#include "Kokkos_Core.hpp" // for parallel_for, KOKKOS_LAMBDA +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_EQ +#include "stk_ngp_test/ngp_test.hpp" // for NGP_EXPECT_EQ, NGP_EXPECT_FALSE, NGP_EXPECT_TRUE +#include "stk_topology/topology.hpp" // for topology, topology::TRI_6, topology::PYRAMID_5, top... +#include "topology_test_utils.hpp" // for INVALID, check_edge_node_ordinals, check_edge_node_... +#include // for vector TEST(stk_topology, pyramid_5) { diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_quad.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_quad.cpp index ca430a659745..02ea93368d0a 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_quad.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_quad.cpp @@ -32,10 +32,14 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include -#include "topology_test_utils.hpp" +#include "Kokkos_Core.hpp" // for parallel_for, KOKKOS_LAMBDA +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_EQ +#include "stk_ngp_test/ngp_test.hpp" // for NGP_EXPECT_EQ, NGP_EXPECT_FALSE, NGP_EXPECT_TRUE +#include "stk_topology/topology.hpp" // for topology, topology::QUAD_4, topology::EDGE_RANK +#include "topology_test_utils.hpp" // for check_edge_node_ordinals, check_edge_node_ordinals_ngp +#include // for printf +#include // for operator<<, endl, basic_ostream, cout, ostream +#include // for vector TEST( stk_topology, quad_4 ) { diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_shell_line.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_shell_line.cpp index e82f1a152ee5..a9c1d02ee803 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_shell_line.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_shell_line.cpp @@ -32,10 +32,12 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include -#include "topology_test_utils.hpp" +#include "Kokkos_Core.hpp" // for parallel_for, KOKKOS_LAMBDA +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_EQ +#include "stk_ngp_test/ngp_test.hpp" // for NGP_EXPECT_EQ, NGP_EXPECT_FALSE, NGP_EXPECT_TRUE +#include "stk_topology/topology.hpp" // for topology, topology::SHELL_LINE_2, topology::EDGE_RANK +#include "topology_test_utils.hpp" // for check_edge_node_ordinals, check_edge_node_ordinals_ngp +#include // for vector TEST(stk_topology, shell_line_2) { diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_spring.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_spring.cpp index 3bfb02e913de..394f2d71fb7a 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_spring.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_spring.cpp @@ -32,11 +32,13 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include +#include "Kokkos_Core.hpp" // for parallel_for, KOKKOS_LAMBDA +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_EQ +#include "stk_ngp_test/ngp_test.hpp" // for NGP_EXPECT_EQ, NGP_EXPECT_TRUE, NGP_EXPECT_FALSE +#include "stk_topology/topology.hpp" // for topology, topology::SPRING_2, topology::ELEMENT_RANK +#include "topology_test_utils.hpp" // for check_equivalent, check_equivalent_ngp, check_lexic... +#include // for vector -#include -#include -#include "topology_test_utils.hpp" TEST(stk_topology, spring_2) { diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_topology.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_topology.cpp index 566b87f20a1c..12fabd640536 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_topology.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_topology.cpp @@ -32,14 +32,13 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include - -#include -#include -#include -#include -#include +#include "gtest/gtest.h" // for Message, TestPartResult, Test, EXPECT_EQ, Assertion... +#include "stk_topology/topology.hpp" // for topology, operator<<, create_superedge_topology +#include // for size_t +#include // for array +#include // for operator<<, ostringstream, ostream, basic_ostream +#include // for string, char_traits + TEST( stk_topology, lexicographical_smallest_permutation) diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_triangle.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_triangle.cpp index 3fe034f59982..c131e4e494dd 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_triangle.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_triangle.cpp @@ -32,10 +32,12 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include -#include "topology_test_utils.hpp" +#include "Kokkos_Core.hpp" // for parallel_for, KOKKOS_LAMBDA +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_EQ +#include "stk_ngp_test/ngp_test.hpp" // for NGP_EXPECT_EQ, NGP_EXPECT_FALSE, NGP_EXPECT_TRUE +#include "stk_topology/topology.hpp" // for topology, topology::EDGE_RANK, topology::FACE_RANK +#include "topology_test_utils.hpp" // for check_edge_node_ordinals, check_edge_node_ordinals_ngp +#include // for vector TEST( stk_topology, tri_3) { diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_validate_topology_data.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_validate_topology_data.cpp index 5ceb9dd017e3..a5711a6050b3 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_validate_topology_data.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_validate_topology_data.cpp @@ -32,12 +32,13 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_TRUE +#include "stk_topology/topology.hpp" // for topology, topology::topology_t, topology_data, oper... +#include // for map, _Rb_tree_const_iterator, map<>::const_iterator +#include // for enable_if +#include // for pair +#include // for vector, operator<, operator== -#include -#include -#include using namespace stk; using namespace stk::topology_detail; diff --git a/packages/stk/stk_unit_tests/stk_topology/unit_test_wedge.cpp b/packages/stk/stk_unit_tests/stk_topology/unit_test_wedge.cpp index 92081c3b7498..3d8b4aa67689 100644 --- a/packages/stk/stk_unit_tests/stk_topology/unit_test_wedge.cpp +++ b/packages/stk/stk_unit_tests/stk_topology/unit_test_wedge.cpp @@ -32,10 +32,14 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#include -#include -#include -#include "topology_test_utils.hpp" +#include "Kokkos_Core.hpp" // for parallel_for, KOKKOS_LAMBDA +#include "gtest/gtest.h" // for AssertionResult, Message, TestPartResult, EXPECT_EQ +#include "stk_ngp_test/ngp_test.hpp" // for NGP_EXPECT_EQ, NGP_EXPECT_FALSE, NGP_EXPECT_TRUE +#include "stk_topology/topology.hpp" // for topology, topology::TRI_6, topology::WEDGE_6, topol... +#include "topology_test_utils.hpp" // for INVALID, check_edge_node_ordinals, check_edge_node_... +#include // for printf +#include // for operator<<, endl, basic_ostream, cout, ostream +#include // for vector TEST(stk_topology, wedge_6) { diff --git a/packages/stk/stk_unit_tests/stk_util/util/UnitTestParameterList.cpp b/packages/stk/stk_unit_tests/stk_util/util/UnitTestParameterList.cpp new file mode 100644 index 000000000000..9537f819f569 --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_util/util/UnitTestParameterList.cpp @@ -0,0 +1,98 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include // for AssertHelper, ASSERT_TRUE, etc +#include +#include +#include +#include +#include + +namespace { + +TEST(UnitTestParameterList, set_and_get_int) +{ + stk::util::ParameterList params; + EXPECT_EQ(0, std::distance(params.begin(), params.end())); + + int intValue = 3; + + params.set_param("intValue", intValue, true, true); + + EXPECT_EQ(1u, std::distance(params.begin(), params.end())); + + stk::util::Parameter& param = params.get_param("intValue"); + EXPECT_TRUE(stk::util::ParameterType::INTEGER == param.type); + EXPECT_TRUE(param.toResultsFile); + EXPECT_TRUE(param.toRestartFile); + + EXPECT_EQ(intValue, params.get_value("intValue")); +} + +TEST(UnitTestParameterList, set_and_get_vectorint) +{ + stk::util::ParameterList params; + EXPECT_EQ(0, std::distance(params.begin(), params.end())); + + std::vector vectorIntValues = {1, 2, 3}; + + params.set_value("vectorIntValue", vectorIntValues); + + EXPECT_EQ(1u, std::distance(params.begin(), params.end())); + + stk::util::Parameter& param = params.get_param("vectorIntValue"); + EXPECT_TRUE(stk::util::ParameterType::INTEGERVECTOR == param.type); + EXPECT_FALSE(param.toResultsFile); + EXPECT_FALSE(param.toRestartFile); + + EXPECT_EQ(vectorIntValues, params.get_value>("vectorIntValue")); +} + +TEST(UnitTestParameterList, set_and_find) +{ + stk::util::ParameterList params; + + int intValue = 3; + std::vector vectorIntValues = {1, 2, 3}; + + params.set_param("intValue", intValue, true, true); + params.set_value("vectorIntValue", vectorIntValues); + + EXPECT_TRUE(params.find("intValue") != params.end()); + EXPECT_TRUE(params.find("vectorIntValue") != params.end()); + EXPECT_TRUE(params.find("badName") == params.end()); +} + +} + diff --git a/packages/stk/stk_unit_tests/stk_util/util/UnitTestThrowMacros.cpp b/packages/stk/stk_unit_tests/stk_util/util/UnitTestThrowMacros.cpp index 7390fe228400..b3fb68a5e966 100644 --- a/packages/stk/stk_unit_tests/stk_util/util/UnitTestThrowMacros.cpp +++ b/packages/stk/stk_unit_tests/stk_util/util/UnitTestThrowMacros.cpp @@ -259,7 +259,7 @@ void testNGPThrowRequireMsg() TEST(UnitTestingOfThrowMacros, NGP_ThrowRequireMsg) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) +#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize_all(). // Also, OpenMP seems to produce an abort (in adddition to a throw?). @@ -299,7 +299,7 @@ void testNGPThrowRequire() TEST(UnitTestingOfThrowMacros, NGP_ThrowRequire) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) +#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize_all(). // @@ -331,7 +331,7 @@ void testNGPThrowAssertMsg() TEST(UnitTestingOfThrowMacros, NGP_ThrowAssertMsg_debug) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) +#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize_all(). // @@ -365,7 +365,7 @@ void testNGPThrowAssertMsg() TEST(UnitTestingOfThrowMacros, NGP_ThrowAssertMsg_release) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) +#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize_all(). // @@ -388,7 +388,7 @@ void testNGPThrowErrorMsgIf() TEST(UnitTestingOfThrowMacros, NGP_ThrowErrorMsgIf) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) +#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize_all(). // @@ -420,7 +420,7 @@ void testNGPThrowErrorIf() TEST(UnitTestingOfThrowMacros, NGP_ThrowErrorIf) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) +#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize_all(). // @@ -449,7 +449,7 @@ void testNGPThrowErrorMsg() TEST(UnitTestingOfThrowMacros, NGP_ThrowErrorMsg) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) +#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize_all(). // diff --git a/packages/stk/stk_util/Jamfile b/packages/stk/stk_util/Jamfile index 1da0c71c5020..06e57e3c1ed6 100644 --- a/packages/stk/stk_util/Jamfile +++ b/packages/stk/stk_util/Jamfile @@ -1,4 +1,4 @@ -#-------------------------------------------------------------------- +#`-------------------------------------------------------------------- # Copyright 2002 - 2008, 2010, 2011 National Technology & # Engineering Solutions of Sandia, LLC (NTESS). Under the terms # of Contract DE-NA0003525 with NTESS, there is a @@ -45,6 +45,13 @@ if $(RTenv-arg) = "user" { stk_util-root-inc = $(stk_util-root) ; } +local stk_util_ngp-root-inc ; +if $(RTenv-arg) = "user" { + stk_util_ngp-root-inc = $(stk_util-root)/stk_util/ngp/include ; +} else { + stk_util_ngp-root-inc = $(stk_util-root)/stk_util/ngp ; +} + project votd : requirements $(sierra-warnings) @@ -137,6 +144,7 @@ install install-user-lib stk_util_diag stk_util_util stk_util_command_line + stk_util_ngp : $(install-root)/stk/stk_util/lib ; @@ -342,3 +350,11 @@ lib stk_util_command_line ] ; +# This is currently a header-only library, so it must not use the lib rule. +# # If sources are added to this library change this target to use the lib rule instead. +alias stk_util_ngp + : # No sources defined for header-only libraries. + : # No build requirements + : # No default build + : $(stk_util_ngp-root-inc) + ; diff --git a/packages/stk/stk_util/stk_util/CMakeLists.txt b/packages/stk/stk_util/stk_util/CMakeLists.txt index 08955bb8eda6..ae0f6b6753b8 100644 --- a/packages/stk/stk_util/stk_util/CMakeLists.txt +++ b/packages/stk/stk_util/stk_util/CMakeLists.txt @@ -5,6 +5,7 @@ ADD_SUBDIRECTORY(environment) ADD_SUBDIRECTORY(registry) ADD_SUBDIRECTORY(diag) ADD_SUBDIRECTORY(command_line) +ADD_SUBDIRECTORY(ngp) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) diff --git a/packages/stk/stk_util/stk_util/ngp/CMakeLists.txt b/packages/stk/stk_util/stk_util/ngp/CMakeLists.txt new file mode 100644 index 000000000000..bfa4aac4c352 --- /dev/null +++ b/packages/stk/stk_util/stk_util/ngp/CMakeLists.txt @@ -0,0 +1,57 @@ +# Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +# Solutions of Sandia, LLC (NTESS). Under the terms of Contract +# DE-NA0003525 with NTESS, the U.S. Government retains certain rights +# in this software. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * Neither the name of NTESS nor the names of its contributors +# may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + + +# +# A) Package-specific configuration options +# + +# +# B) Define the header and source files (and directories) +# + +# +# src +# + +SET(HEADERS "") +SET(SOURCES "") + +FILE(GLOB HEADERS *.hpp *.h) + +INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) + +INSTALL(FILES ${HEADERS} DESTINATION + ${CMAKE_INSTALL_PREFIX}/${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_util/ngp/) + diff --git a/packages/stk/stk_util/stk_util/ngp/NgpSpaces.hpp b/packages/stk/stk_util/stk_util/ngp/NgpSpaces.hpp new file mode 100644 index 000000000000..4a938431901e --- /dev/null +++ b/packages/stk/stk_util/stk_util/ngp/NgpSpaces.hpp @@ -0,0 +1,99 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef STK_NGP_NGPSPACES_HPP +#define STK_NGP_NGPSPACES_HPP + +#include + +#ifdef KOKKOS_ENABLE_OPENMP +#include +#endif + +namespace stk { +namespace ngp { + +#ifdef KOKKOS_ENABLE_CUDA +using ExecSpace = Kokkos::Cuda; +#elif defined(KOKKOS_ENABLE_HIP) +using ExecSpace = Kokkos::Experimental::HIP; +#elif defined(KOKKOS_ENABLE_OPENMP) +using ExecSpace = Kokkos::OpenMP; +#else +using ExecSpace = Kokkos::Serial; +#endif + +#ifdef KOKKOS_ENABLE_CUDA +using HostExecSpace = Kokkos::Serial; +#elif defined(KOKKOS_ENABLE_HIP) +using HostExecSpace = Kokkos::Serial; +#elif defined(KOKKOS_ENABLE_OPENMP) +using HostExecSpace = Kokkos::OpenMP; +#else +using HostExecSpace = Kokkos::Serial; +#endif + +#ifdef KOKKOS_ENABLE_CUDA +using MemSpace = Kokkos::CudaSpace; +#elif defined(KOKKOS_ENABLE_HIP) +using MemSpace = Kokkos::Experimental::HIPSpace; +#elif defined(KOKKOS_ENABLE_OPENMP) +using MemSpace = Kokkos::OpenMP; +#else +using MemSpace = Kokkos::HostSpace; +#endif + +#ifdef KOKKOS_ENABLE_CUDA +using UVMMemSpace = Kokkos::CudaUVMSpace; +#elif defined(KOKKOS_ENABLE_HIP) +using UVMMemSpace = Kokkos::Experimental::HIPHostPinnedSpace; +#elif defined(KOKKOS_ENABLE_OPENMP) +using UVMMemSpace = Kokkos::OpenMP; +#else +using UVMMemSpace = Kokkos::HostSpace; +#endif + +#ifdef KOKKOS_ENABLE_CUDA +using HostPinnedSpace = Kokkos::CudaHostPinnedSpace; +#elif defined(KOKKOS_ENABLE_HIP) +using HostPinnedSpace = Kokkos::Experimental::HIPHostPinnedSpace; +#else +using HostPinnedSpace = MemSpace; +#endif + +using ScheduleType = Kokkos::Schedule; + +} +} + +#endif diff --git a/packages/stk/stk_util/stk_util/parallel/ParallelComm.cpp b/packages/stk/stk_util/stk_util/parallel/ParallelComm.cpp index fad9db5388f5..6a9ac2b08a18 100644 --- a/packages/stk/stk_util/stk_util/parallel/ParallelComm.cpp +++ b/packages/stk/stk_util/stk_util/parallel/ParallelComm.cpp @@ -55,19 +55,6 @@ enum { STK_MPI_TAG_SIZING = 0 , STK_MPI_TAG_DATA = 1 }; //---------------------------------------------------------------------- -namespace { - -inline -size_t align_quad( size_t n ) -{ - enum { Size = 4 * sizeof(int) }; - return n + CommBufferAlign::align(n); -} - -} - -//---------------------------------------------------------------------- - void CommBuffer::pack_overflow() const { #ifndef NDEBUG @@ -99,61 +86,6 @@ void CommBuffer::set_buffer_ptrs(unsigned char* begin, unsigned char* ptr, unsig m_end = end; } -#ifndef STK_HIDE_DEPRECATED_CODE // Delete after June 2021 -STK_DEPRECATED void CommBuffer::deallocate( const unsigned number , CommBuffer * buffers ) -{ - if ( nullptr != buffers ) { - for ( unsigned i = 0 ; i < number ; ++i ) { - ( buffers + i )->~CommBuffer(); - } - free( buffers ); - } -} - -STK_DEPRECATED CommBuffer * CommBuffer::allocate( - const unsigned number , const unsigned * const size ) -{ - const size_t n_base = align_quad( number * sizeof(CommBuffer) ); - size_t n_size = n_base ; - - if ( nullptr != size ) { - for ( unsigned i = 0 ; i < number ; ++i ) { - n_size += align_quad( size[i] ); - } - } - - // Allocate space for buffers - - void * const p_malloc = malloc( n_size ); - - CommBuffer * const b_base = - p_malloc != nullptr ? reinterpret_cast(p_malloc) - : reinterpret_cast( NULL ); - - if ( p_malloc != nullptr ) { - - for ( unsigned i = 0 ; i < number ; ++i ) { - new( b_base + i ) CommBuffer(); - } - - if ( nullptr != size ) { - - ucharp ptr = reinterpret_cast( p_malloc ); - - ptr += n_base ; - - for ( unsigned i = 0 ; i < number ; ++i ) { - CommBuffer & b = b_base[i] ; - b.set_buffer_ptrs(ptr, ptr, ptr + size[i]); - ptr += align_quad( size[i] ); - } - } - } - - return b_base ; -} -#endif - //---------------------------------------------------------------------- //---------------------------------------------------------------------- diff --git a/packages/stk/stk_util/stk_util/parallel/ParallelComm.hpp b/packages/stk/stk_util/stk_util/parallel/ParallelComm.hpp index 6fb8f7f3b529..8d527135df3c 100644 --- a/packages/stk/stk_util/stk_util/parallel/ParallelComm.hpp +++ b/packages/stk/stk_util/stk_util/parallel/ParallelComm.hpp @@ -180,11 +180,6 @@ class CommBuffer { friend class CommNeighbors ; friend class CommBroadcast ; -#ifndef STK_HIDE_DEPRECATED_CODE // Delete after June 2021 - STK_DEPRECATED static CommBuffer * allocate( const unsigned, const unsigned * const ); - STK_DEPRECATED static void deallocate( const unsigned , CommBuffer * ); -#endif - void pack_overflow() const ; void unpack_overflow() const ; diff --git a/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp b/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp index f9a3d8a5c38b..4b700e4603f4 100644 --- a/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp +++ b/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp @@ -42,7 +42,7 @@ //In Sierra, STK_VERSION_STRING is provided on the compile line by bake. //For Trilinos stk snapshots, the following macro definition gets populated with //the real version string by the trilinos_snapshot.sh script. -#define STK_VERSION_STRING "5.1.6-299-ge2242fa1" +#define STK_VERSION_STRING "5.1.7-280-gbeeccf79" #endif namespace stk { diff --git a/packages/stk/stk_util/stk_util/stk_kokkos_macros.h b/packages/stk/stk_util/stk_util/stk_kokkos_macros.h index c134961ed8d8..8a45bb1af5a5 100644 --- a/packages/stk/stk_util/stk_util/stk_kokkos_macros.h +++ b/packages/stk/stk_util/stk_util/stk_kokkos_macros.h @@ -6,7 +6,7 @@ // This should eventually need to be supplemented with checks for ROCM and // other accelerator platforms // -#ifdef KOKKOS_ENABLE_CUDA +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) #ifndef STK_USE_DEVICE_MESH #define STK_USE_DEVICE_MESH #endif diff --git a/packages/stk/stk_util/stk_util/util/ParameterList.hpp b/packages/stk/stk_util/stk_util/util/ParameterList.hpp index 4357aa93fdae..102bdbd482c8 100644 --- a/packages/stk/stk_util/stk_util/util/ParameterList.hpp +++ b/packages/stk/stk_util/stk_util/util/ParameterList.hpp @@ -128,6 +128,18 @@ struct Parameter{ toResultsFile(false), toRestartFile(false) {} + + template + T get_value() const + { + if (type == ParameterType::get_type(T())) { + return STK_ANY_NAMESPACE::any_cast(value); + } else { + std::cerr << "ERROR: Parameter has an incorrect type specified for the get_value" + << " template type.\n"; + } + return T(); + } }; typedef std::map ParameterMapType; @@ -177,14 +189,7 @@ class ParameterList { template T get_value (const std::string name) { Parameter p = get_param(name); - if (p.type == ParameterType::get_type(T())) { - return STK_ANY_NAMESPACE::any_cast(p.value); - } else { - std::cerr << "ERROR: Parameter named '" << name - << "' has an incorrect type specified for the get_value" - << " template type.\n"; - } - return T(); + return p.get_value(); } void write_parameter_list(std::ostream & stream); diff --git a/packages/stk/stk_util/stk_util/util/ReportHandler.hpp b/packages/stk/stk_util/stk_util/util/ReportHandler.hpp index e7b24fc3c756..88d4d6f340c5 100644 --- a/packages/stk/stk_util/stk_util/util/ReportHandler.hpp +++ b/packages/stk/stk_util/stk_util/util/ReportHandler.hpp @@ -362,6 +362,8 @@ STK_INLINE_FUNCTION void ThrowErrorMsgDevice(const char * message) #define ThrowRequireMsg(expr,message) ThrowGenericCond(expr, message, handle_assert) #define ThrowRequire(expr) ThrowRequireMsg(expr, "") +#ifndef __HIP_DEVICE_COMPILE__ + #ifdef NDEBUG # define ThrowAssert(expr) (static_cast(0)) # define ThrowAssertMsg(expr,message) (static_cast(0)) @@ -377,6 +379,18 @@ STK_INLINE_FUNCTION void ThrowErrorMsgDevice(const char * message) #define ThrowInvalidArgMsgIf(expr, message) ThrowGenericCond( !(expr), message, handle_invalid_arg) #define ThrowInvalidArgIf(expr) ThrowInvalidArgMsgIf(expr, "") +#else +//FIXME: unsupported indirect call to function on HIP-Clang +#define ThrowAssert(expr) +#define ThrowAssertMsg(expr,message) + +#define ThrowErrorMsgIf(expr, message) +#define ThrowErrorIf(expr) +#define ThrowErrorMsg(message) + +#define ThrowInvalidArgMsgIf(expr, message) +#define ThrowInvalidArgIf(expr) +#endif #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ > 0)) #define NGP_ThrowRequireMsg(expr, message) \ @@ -385,6 +399,9 @@ do { \ ThrowMsgDevice(message ": " __FILE__ ":" LINE_STRING); \ } \ } while(false); +#elif defined(__HIP_DEVICE_COMPILE__) +//FIXME: unsupported indirect call to function on HIP-Clang +#define NGP_ThrowRequireMsg(expr, message) #else #define NGP_ThrowRequireMsg(expr, message) \ do { \ @@ -401,6 +418,9 @@ do { \ ThrowMsgDevice("(" #expr "): " __FILE__ ":" LINE_STRING); \ } \ } while(false); +#elif defined(__HIP_DEVICE_COMPILE__) +//FIXME: unsupported indirect call to function on HIP-Clang +#define NGP_ThrowRequire(expr) #else #define NGP_ThrowRequire(expr) \ do { \ @@ -420,6 +440,9 @@ do { \ #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ > 0)) #define NGP_ThrowErrorMsgIf(expr, message) NGP_ThrowRequireMsg(!(expr), message); +#elif defined(__HIP_DEVICE_COMPILE__) +//FIXME: unsupported indirect call to function on HIP-Clang +#define NGP_ThrowErrorMsgIf(expr, message) #else #define NGP_ThrowErrorMsgIf(expr, message) \ do { \ @@ -431,6 +454,9 @@ do { \ #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ > 0)) #define NGP_ThrowErrorIf(expr) NGP_ThrowRequireMsg(!(expr), "!(" #expr ")"); +#elif defined(__HIP_DEVICE_COMPILE__) +//FIXME: unsupported indirect call to function on HIP-Clang +#define NGP_ThrowErrorIf(expr) #else #define NGP_ThrowErrorIf(expr) \ do { \ @@ -442,6 +468,9 @@ do { \ #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ > 0)) #define NGP_ThrowErrorMsg(message) ThrowErrorMsgDevice(message ": " __FILE__ ":" LINE_STRING); +#elif defined(__HIP_DEVICE_COMPILE__) +//FIXME: unsupported indirect call to function on HIP-Clang +#define NGP_ThrowErrorMsg(message) #else #define NGP_ThrowErrorMsg(message) ThrowErrorMsgHost(message, STK_STR_TRACE); #endif diff --git a/packages/stk/stk_util/stk_util/util/StkNgpVector.hpp b/packages/stk/stk_util/stk_util/util/StkNgpVector.hpp index 699c2f460aeb..fb68c9d86b0e 100644 --- a/packages/stk/stk_util/stk_util/util/StkNgpVector.hpp +++ b/packages/stk/stk_util/stk_util/util/StkNgpVector.hpp @@ -106,6 +106,8 @@ class NgpVector protected: #ifdef KOKKOS_ENABLE_CUDA using DeviceSpace = Kokkos::CudaSpace; +#elif defined(KOKKOS_ENABLE_HIP) + using DeviceSpace = Kokkos::Experimental::HIPSpace; #else using DeviceSpace = Kokkos::HostSpace; #endif @@ -118,7 +120,7 @@ class NgpVector { return deviceVals(i); } -#ifdef KOKKOS_ENABLE_CUDA +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) template STK_FUNCTION Datatype & get( typename std::enable_if< diff --git a/packages/stk/stk_util/stk_util/util/StridedArray.hpp b/packages/stk/stk_util/stk_util/util/StridedArray.hpp index f0cd3dc4ca52..8caaf0441fd7 100644 --- a/packages/stk/stk_util/stk_util/util/StridedArray.hpp +++ b/packages/stk/stk_util/stk_util/util/StridedArray.hpp @@ -57,7 +57,7 @@ class StridedArray STK_FUNCTION T operator[](unsigned i) const { -#ifdef __CUDA_ARCH__ +#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__) return dataPointer[stride*i]; #else return dataPointer[i]; From 87901d94d26752fd3e442cc4520cbe4fef025576 Mon Sep 17 00:00:00 2001 From: Greg Sjaardema Date: Tue, 31 Aug 2021 07:22:37 -0600 Subject: [PATCH 45/70] Update for recent seacas changes --- .../addFileContentsToOutputDatabase.cpp | 20 +++++++++---------- .../stk_io/UnitTestTopologyMap.cpp | 1 + 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/stk/stk_doc_tests/stk_io/addFileContentsToOutputDatabase.cpp b/packages/stk/stk_doc_tests/stk_io/addFileContentsToOutputDatabase.cpp index 1a7547781a18..62b617402d2f 100644 --- a/packages/stk/stk_doc_tests/stk_io/addFileContentsToOutputDatabase.cpp +++ b/packages/stk/stk_doc_tests/stk_io/addFileContentsToOutputDatabase.cpp @@ -118,25 +118,25 @@ TEST(StkMeshIoBrokerHowTo, addFileContentsToOutputDatabase) stkIo.create_input_mesh(); const std::vector &info_records = stkIo.get_info_records(); - // First 2 lines of info records are host information (node name, + // First line of info records is host information (node name, // os version) (2) Next record is the file name of the input file // data that follows (1) File contains 4 records; 1 is longer than // 80 characters, so it wraps (4+1) Next line is the // "additional_info_record" added above (1) // Last records are the IOSS configuration summary (35). - size_t expected_min_info_record_count = 2 + (4+1) + 1 + 1; + size_t expected_min_info_record_count = 1 + (4+1) + 1 + 1; EXPECT_TRUE(expected_min_info_record_count <= info_records.size()); - EXPECT_STREQ(input_file.c_str(), info_records[2].c_str()); + EXPECT_STREQ(input_file.c_str(), info_records[1].c_str()); - EXPECT_STREQ(info1.c_str(), info_records[3].c_str()); - EXPECT_STREQ(info2.substr(0,79).c_str(), info_records[4].substr(0,79).c_str()); - EXPECT_STREQ(info2.substr(79).c_str(), info_records[5].c_str()); - EXPECT_STREQ(info3.c_str(), info_records[6].c_str()); - EXPECT_STREQ(info4.c_str(), info_records[7].c_str()); - EXPECT_STREQ(additional_info_record.c_str(), info_records[8].c_str()); + EXPECT_STREQ(info1.c_str(), info_records[2].c_str()); + EXPECT_STREQ(info2.substr(0,79).c_str(), info_records[3].substr(0,79).c_str()); + EXPECT_STREQ(info2.substr(79).c_str(), info_records[4].c_str()); + EXPECT_STREQ(info3.c_str(), info_records[5].c_str()); + EXPECT_STREQ(info4.c_str(), info_records[6].c_str()); + EXPECT_STREQ(additional_info_record.c_str(), info_records[7].c_str()); - unlink(filename.c_str()); + // unlink(filename.c_str()); unlink(input_file.c_str()); } } diff --git a/packages/stk/stk_unit_tests/stk_io/UnitTestTopologyMap.cpp b/packages/stk/stk_unit_tests/stk_io/UnitTestTopologyMap.cpp index 302cc0612464..84bfb78965c9 100644 --- a/packages/stk/stk_unit_tests/stk_io/UnitTestTopologyMap.cpp +++ b/packages/stk/stk_unit_tests/stk_io/UnitTestTopologyMap.cpp @@ -76,6 +76,7 @@ int testElement(const std::string &name, unsigned spatialDim) if (element->name() == "unknown" || element->name() == "bar4" || element->name() == "edge4" || + element->name() == "hex9" || element->name() == "hex16" || element->name() == "hex32" || element->name() == "hex64" || From b0a149eb1a017d895b504d5f0883c246c8966b9e Mon Sep 17 00:00:00 2001 From: gsjaardema Date: Tue, 31 Aug 2021 07:24:48 -0600 Subject: [PATCH 46/70] IOSS: fixes for build/test --- packages/seacas/libraries/ioss/src/exodus/Ioex_Internals.C | 6 +++--- packages/seacas/libraries/ioss/src/utest/CMakeLists.txt | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/seacas/libraries/ioss/src/exodus/Ioex_Internals.C b/packages/seacas/libraries/ioss/src/exodus/Ioex_Internals.C index 27cc62038388..babbb39051bc 100644 --- a/packages/seacas/libraries/ioss/src/exodus/Ioex_Internals.C +++ b/packages/seacas/libraries/ioss/src/exodus/Ioex_Internals.C @@ -2957,10 +2957,10 @@ int Internals::put_metadata(const std::vector &nodesets, bool count_onl } // define variable to store node set node list here instead of in expns - int dims[1] = {dimid}; + int dims1[1] = {dimid}; int varid; status = - nc_def_var(exodusFilePtr, VAR_NODE_NS(cur_num_node_sets + 1), bulk_type, 1, dims, &varid); + nc_def_var(exodusFilePtr, VAR_NODE_NS(cur_num_node_sets + 1), bulk_type, 1, dims1, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { @@ -2994,7 +2994,7 @@ int Internals::put_metadata(const std::vector &nodesets, bool count_onl } // create variable for distribution factors status = nc_def_var(exodusFilePtr, VAR_FACT_NS(cur_num_node_sets + 1), - nc_flt_code(exodusFilePtr), 1, dims, &varid); + nc_flt_code(exodusFilePtr), 1, dims1, &varid); if (status != NC_NOERR) { ex_opts(EX_VERBOSE); if (status == NC_ENAMEINUSE) { diff --git a/packages/seacas/libraries/ioss/src/utest/CMakeLists.txt b/packages/seacas/libraries/ioss/src/utest/CMakeLists.txt index 04a5a2db2436..a464f8784bb8 100644 --- a/packages/seacas/libraries/ioss/src/utest/CMakeLists.txt +++ b/packages/seacas/libraries/ioss/src/utest/CMakeLists.txt @@ -29,8 +29,7 @@ TRIBITS_ADD_EXECUTABLE( TRIBITS_ADD_TEST( Utst_blob_write_read NAME Utst_blob_write_read - NUM_MPI_PROCS ${NPROCS} - COMM mpi serial + NUM_MPI_PROCS 1 XHOSTTYPE Windows ) From 5002f8f369b005bb6183867788f93b2cb0f8b6c4 Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Tue, 31 Aug 2021 08:48:05 -0600 Subject: [PATCH 47/70] have more examples running --- .../Example_SimpleSource_impl.hpp | 21 +++++++------ .../Example_Solution_impl.hpp | 31 +++++++++++-------- .../src/evaluators/Panzer_DOFCurl_impl.hpp | 7 +++-- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/packages/panzer/adapters-stk/example/PoissonInterfaceExample/Example_SimpleSource_impl.hpp b/packages/panzer/adapters-stk/example/PoissonInterfaceExample/Example_SimpleSource_impl.hpp index 7846cee3e15f..127240d32743 100644 --- a/packages/panzer/adapters-stk/example/PoissonInterfaceExample/Example_SimpleSource_impl.hpp +++ b/packages/panzer/adapters-stk/example/PoissonInterfaceExample/Example_SimpleSource_impl.hpp @@ -82,19 +82,22 @@ template void SimpleSource::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < source.extent_int(1); ++point) { - const double & x = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,0); - const double & y = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,1); + auto ip_coordinates = workset.int_rules[ir_index]->ip_coordinates.get_static_view(); + auto source_v = source.get_static_view(); + Kokkos::parallel_for ("SimpleSource", workset.num_cells, KOKKOS_LAMBDA (const index_t cell) { + for (int point = 0; point < source_v.extent_int(1); ++point) { + const double & x = ip_coordinates(cell,point,0); + const double & y = ip_coordinates(cell,point,1); - if (this->wda(workset).int_rules[ir_index]->ip_coordinates.extent(2) == 2) { - source(cell,point) = 4*M_PI*M_PI*sin(2*M_PI*x)*cos(2*M_PI*y); + if (ip_coordinates.extent(2) == 2) { + source_v(cell,point) = 4*M_PI*M_PI*sin(2*M_PI*x)*cos(2*M_PI*y); } else { - const double & z = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,2); - source(cell,point) = 4*M_PI*M_PI*sin(2*M_PI*x)*cos(2*M_PI*y)*cos(2*M_PI*z); + const double & z = ip_coordinates(cell,point,2); + source_v(cell,point) = 4*M_PI*M_PI*sin(2*M_PI*x)*cos(2*M_PI*y)*cos(2*M_PI*z); } } - } + }); + Kokkos::fence(); } //********************************************************************** diff --git a/packages/panzer/adapters-stk/example/PoissonInterfaceExample/Example_Solution_impl.hpp b/packages/panzer/adapters-stk/example/PoissonInterfaceExample/Example_Solution_impl.hpp index f0a9ae978a56..2d75fe8e2cc2 100644 --- a/packages/panzer/adapters-stk/example/PoissonInterfaceExample/Example_Solution_impl.hpp +++ b/packages/panzer/adapters-stk/example/PoissonInterfaceExample/Example_Solution_impl.hpp @@ -76,26 +76,31 @@ template void Solution::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < solution.extent_int(1); ++point) { - const double& x = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,0); - const double& y = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,1); + auto ip_coordinates = workset.int_rules[ir_index]->ip_coordinates.get_static_view(); + auto solution_v = solution.get_static_view(); + auto l_linear_Robin = linear_Robin; + auto workset_block_0 = (workset.block_id[7] == '0'); + Kokkos::parallel_for ("SimpleSolutin", workset.num_cells, KOKKOS_LAMBDA (const index_t cell) { + for (int point = 0; point < solution_v.extent_int(1); ++point) { + const double& x = ip_coordinates(cell,point,0); + const double& y = ip_coordinates(cell,point,1); - if (linear_Robin) { - if (this->wda(workset).int_rules[ir_index]->ip_coordinates.extent(2) == 2) { - solution(cell,point) = 0.5 - 0.8*x + 0.5*sin(2*M_PI*x)*cos(2*M_PI*y); + if (l_linear_Robin) { + if (ip_coordinates.extent(2) == 2) { + solution_v(cell,point) = 0.5 - 0.8*x + 0.5*sin(2*M_PI*x)*cos(2*M_PI*y); } else { - const double & z = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,2); - solution(cell,point) = 0.5 - 0.8*x + sin(2*M_PI*x)*cos(2*M_PI*y)*cos(2*M_PI*z)/3.0; + const double & z = ip_coordinates(cell,point,2); + solution_v(cell,point) = 0.5 - 0.8*x + sin(2*M_PI*x)*cos(2*M_PI*y)*cos(2*M_PI*z)/3.0; } } else { - if (workset.block_id[7] == '0') - solution(cell,point) = 0.5 - 0.4*x; + if (workset_block_0) + solution_v(cell,point) = 0.5 - 0.4*x; else - solution(cell,point) = 0.1 - 0.4*x; + solution_v(cell,point) = 0.1 - 0.4*x; } } - } + }); + Kokkos::fence(); } } diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_DOFCurl_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_DOFCurl_impl.hpp index 56f8430db4b3..f44ea6a7dad4 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_DOFCurl_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_DOFCurl_impl.hpp @@ -447,10 +447,11 @@ DOFCurl(const Teuchos::ParameterList & p) : offsets = *p.get > >("Jacobian Offsets Vector"); // allocate and copy offsets vector to Kokkos array - PHX::View offsets_array_nc - = PHX::View("offsets",offsets.size()); + PHX::View offsets_array_nc("offsets",offsets.size()); + auto offsets_array_nc_h = Kokkos::create_mirror_view(offsets_array_nc); for(std::size_t i=0;i Date: Fri, 27 Aug 2021 16:53:51 -0600 Subject: [PATCH 48/70] Ifpack2 Chebyshev: Add "cg" option for "eigen-analysis: type" --- .../src/Ifpack2_Details_Chebyshev_decl.hpp | 30 ++++ .../src/Ifpack2_Details_Chebyshev_def.hpp | 161 +++++++++++++++++- 2 files changed, 184 insertions(+), 7 deletions(-) diff --git a/packages/ifpack2/src/Ifpack2_Details_Chebyshev_decl.hpp b/packages/ifpack2/src/Ifpack2_Details_Chebyshev_decl.hpp index dc4ccb447299..f515ea039ac9 100644 --- a/packages/ifpack2/src/Ifpack2_Details_Chebyshev_decl.hpp +++ b/packages/ifpack2/src/Ifpack2_Details_Chebyshev_decl.hpp @@ -482,6 +482,9 @@ class Chebyshev : public Teuchos::Describable { //! Whether the iteration vectors of the power method should be saved. bool eigKeepVectors_; + //! Type of eigen-analysis, i.e. power method or cg. + std::string eigenAnalysisType_; + //! Iteration vectors of the power method. Can be saved for the purpose of multiple setups. Teuchos::RCP eigVector_; Teuchos::RCP eigVector2_; @@ -693,6 +696,33 @@ class Chebyshev : public Teuchos::Describable { ST powerMethod (const op_type& A, const V& D_inv, const int numIters); + /// \brief Use the cg method to estimate the maximum eigenvalue + /// of A*D_inv, given an initial guess vector x. + /// + /// \param A [in] The Operator to use. + /// \param D_inv [in] Vector to use as implicit right scaling of A. + /// \param numIters [in] Maximum number of iterations of the power + /// method. + /// \param x [in/out] On input: Initial guess Vector for the cg + /// method. Its Map must be the same as that of the domain Map of + /// A. This method may use this Vector as scratch space. + /// + /// \return Estimate of the maximum eigenvalue of A*D_inv. + ST + cgMethodWithInitGuess (const op_type& A, const V& D_inv, const int numIters, V& x); + + /// \brief Use the cg method to estimate the maximum eigenvalue + /// of A*D_inv. + /// + /// \param A [in] The Operator to use. + /// \param D_inv [in] Vector to use as implicit right scaling of A. + /// \param numIters [in] Maximum number of iterations of the power + /// method. + /// + /// \return Estimate of the maximum eigenvalue of A*D_inv. + ST + cgMethod (const op_type& A, const V& D_inv, const int numIters); + //! The maximum infinity norm of all the columns of X. static MT maxNormInf (const MV& X); diff --git a/packages/ifpack2/src/Ifpack2_Details_Chebyshev_def.hpp b/packages/ifpack2/src/Ifpack2_Details_Chebyshev_def.hpp index 56d4b2a0a255..57809145f0fa 100644 --- a/packages/ifpack2/src/Ifpack2_Details_Chebyshev_def.hpp +++ b/packages/ifpack2/src/Ifpack2_Details_Chebyshev_def.hpp @@ -58,6 +58,8 @@ #include "Teuchos_FancyOStream.hpp" #include "Teuchos_oblackholestream.hpp" #include "Tpetra_Details_residual.hpp" +#include "Teuchos_LAPACK.hpp" +#include "Ifpack2_Details_LapackSupportsScalar.hpp" #include #include @@ -218,6 +220,51 @@ class PositivizeVector { } // namespace (anonymous) + +template::value> +struct LapackHelper{ + static + ScalarType + tri_diag_spectral_radius(Teuchos::ArrayRCP::magnitudeType> diag, + Teuchos::ArrayRCP::magnitudeType> offdiag) { + throw std::runtime_error("LAPACK does not support the scalar type."); + } +}; + + +template +struct LapackHelper { + static + ScalarType + tri_diag_spectral_radius(Teuchos::ArrayRCP::magnitudeType> diag, + Teuchos::ArrayRCP::magnitudeType> offdiag) { + using STS = Teuchos::ScalarTraits; + using MagnitudeType = typename STS::magnitudeType; + int info = 0; + const int N = diag.size (); + ScalarType scalar_dummy; + std::vector mag_dummy(4*N); + char char_N = 'N'; + + // lambdaMin = one; + ScalarType lambdaMax = STS::one(); + if( N > 2 ) { + Teuchos::LAPACK lapack; + lapack.PTEQR (char_N, N, diag.getRawPtr (), offdiag.getRawPtr (), + &scalar_dummy, 1, &mag_dummy[0], &info); + TEUCHOS_TEST_FOR_EXCEPTION + (info < 0, std::logic_error, "Ifpack2::Details::LapackHelper::tri_diag_spectral_radius:" + "LAPACK's _PTEQR failed with info = " + << info << " < 0. This suggests there might be a bug in the way Ifpack2 " + "is calling LAPACK. Please report this to the Ifpack2 developers."); + // lambdaMin = Teuchos::as (diag[N-1]); + lambdaMax = Teuchos::as (diag[0]); + } + return lambdaMax; + } +}; + + template void Chebyshev::checkInputMatrix () const { @@ -288,6 +335,7 @@ Chebyshev (Teuchos::RCP A) : eigMaxIters_ (10), eigRelTolerance_(Teuchos::ScalarTraits::zero ()), eigKeepVectors_(false), + eigenAnalysisType_("power method"), eigNormalizationFreq_(1), zeroStartingSolution_ (true), assumeMatrixUnchanged_ (false), @@ -318,6 +366,7 @@ Chebyshev (Teuchos::RCP A, eigMaxIters_ (10), eigRelTolerance_(Teuchos::ScalarTraits::zero ()), eigKeepVectors_(false), + eigenAnalysisType_("power method"), eigNormalizationFreq_(1), zeroStartingSolution_ (true), assumeMatrixUnchanged_ (false), @@ -649,13 +698,10 @@ setParameters (Teuchos::ParameterList& plist) eigenAnalysisType = plist.get ("eigen-analysis: type"); TEUCHOS_TEST_FOR_EXCEPTION( eigenAnalysisType != "power-method" && - eigenAnalysisType != "power method", + eigenAnalysisType != "power method" && + eigenAnalysisType != "cg", std::invalid_argument, - "Ifpack2::Chebyshev: This class supports the ML parameter \"eigen-" - "analysis: type\" for backwards compatibility. However, Ifpack2 " - "currently only supports the \"power-method\" option for this " - "parameter. This imitates Ifpack, which only implements the power " - "method for eigenanalysis."); + "Ifpack2::Chebyshev: Ifpack2 only supports \"power method\" and \"cg\" for \"eigen-analysis: type\"."); } // We've validated all the parameters, so it's safe now to "commit" them. @@ -670,6 +716,7 @@ setParameters (Teuchos::ParameterList& plist) eigRelTolerance_ = eigRelTolerance; eigKeepVectors_ = eigKeepVectors; eigNormalizationFreq_ = eigNormalizationFreq; + eigenAnalysisType_ = eigenAnalysisType; zeroStartingSolution_ = zeroStartingSolution; assumeMatrixUnchanged_ = assumeMatrixUnchanged; textbookAlgorithm_ = textbookAlgorithm; @@ -852,7 +899,11 @@ Chebyshev::compute () // most important one if using Chebyshev as a smoother. if (! assumeMatrixUnchanged_ || (! computedEigenvalueEstimates && STS::isnaninf (userLambdaMax_))) { - const ST computedLambdaMax = powerMethod (*A_, *D_, eigMaxIters_); + ST computedLambdaMax; + if ((eigenAnalysisType_ == "power method") || (eigenAnalysisType_ == "power-method")) + computedLambdaMax = powerMethod (*A_, *D_, eigMaxIters_); + else + computedLambdaMax = cgMethod (*A_, *D_, eigMaxIters_); TEUCHOS_TEST_FOR_EXCEPTION( STS::isnaninf (computedLambdaMax), std::runtime_error, @@ -1548,6 +1599,102 @@ powerMethod (const op_type& A, const V& D_inv, const int numIters) return lambdaMax; } + +template +typename Chebyshev::ST +Chebyshev:: +cgMethodWithInitGuess (const op_type& A, + const V& D_inv, + const int numIters, + V& r) +{ + using std::endl; + using STS = Teuchos::ScalarTraits; + using MagnitudeType = typename STS::magnitudeType; + if (debug_) { + *out_ << " cgMethodWithInitGuess:" << endl; + } + + const ST one = STS::one(); + ST beta, betaOld = one, pAp, pApOld = one, alpha, rHz, rHzOld, rHzOld2 = one, lambdaMax; + // ST lambdaMin; + Teuchos::ArrayRCP diag, offdiag; + Teuchos::RCP p, z, Ap; + diag.resize(numIters); + offdiag.resize(numIters-1); + + p = rcp(new V(A.getRangeMap ())); + z = rcp(new V(A.getRangeMap ())); + Ap = rcp(new V(A.getRangeMap ())); + + // Tpetra::Details::residual (A, x, *b, *r); + solve (*p, D_inv, r); + rHz = r.dot (*p); + + for (int iter = 0; iter < numIters; ++iter) { + if (debug_) { + *out_ << " Iteration " << iter << endl; + } + A.apply (*p, *Ap); + pAp = p->dot (*Ap); + alpha = rHz/pAp; + r.update (-alpha, *Ap, one); + rHzOld = rHz; + solve (*z, D_inv, r); + rHz = r.dot (*z); + beta = rHz / rHzOld; + p->update (one, *z, beta); + if (iter>0) { + diag[iter] = STS::real((betaOld*betaOld * pApOld + pAp) / rHzOld); + offdiag[iter-1] = -STS::real((betaOld * pApOld / (sqrt(rHzOld * rHzOld2)))); + if (debug_) { + *out_ << " diag[" << iter << "] = " << diag[iter] << endl; + *out_ << " offdiag["<< iter-1 << "] = " << offdiag[iter-1] << endl; + } + } else { + diag[iter] = STS::real(pAp/rHzOld); + if (debug_) { + *out_ << " diag[" << iter << "] = " << diag[iter] << endl; + } + } + rHzOld2 = rHzOld; + betaOld = beta; + pApOld = pAp; + } + + lambdaMax = LapackHelper::tri_diag_spectral_radius(diag, offdiag); + + return lambdaMax; +} + + +template +typename Chebyshev::ST +Chebyshev:: +cgMethod (const op_type& A, const V& D_inv, const int numIters) +{ + using std::endl; + if (debug_) { + *out_ << "cgMethod:" << endl; + } + + Teuchos::RCP r; + if (eigVector_.is_null()) { + r = rcp(new V(A.getDomainMap ())); + if (eigKeepVectors_) + eigVector_ = r; + // For the first pass, just let the pseudorandom number generator + // fill x with whatever values it wants; don't try to make its + // entries nonnegative. + computeInitialGuessForPowerMethod (*r, false); + } else + r = eigVector_; + + ST lambdaMax = cgMethodWithInitGuess (A, D_inv, numIters, *r); + + return lambdaMax; +} + template Teuchos::RCP::row_matrix_type> Chebyshev::getMatrix () const { From 02ee4aa9e23ea7abaf49c0e57c80b39bf390e268 Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Mon, 30 Aug 2021 18:36:06 -0600 Subject: [PATCH 49/70] Ifpack2 Chebyshev: unit test computation of spectral radius --- .../unit_tests/Ifpack2_UnitTestChebyshev.cpp | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestChebyshev.cpp b/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestChebyshev.cpp index 8b37c1417e90..54d9fe14c838 100644 --- a/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestChebyshev.cpp +++ b/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestChebyshev.cpp @@ -151,6 +151,30 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(Ifpack2Chebyshev, Test0, Scalar, LocalOrdinal, Teuchos::ArrayRCP yview = y.get1dView(); TEST_COMPARE_FLOATING_ARRAYS(yview, halfs(), tol); } + + crsmatrix = tif_utest::create_test_matrix(rowmap, -one); + Scalar n = Teuchos::as(rowmap->getGlobalNumElements()); + Scalar expectedLambdaMax = one-std::cos(Teuchos::ScalarTraits::pi()*n/(n+1)); + + prec.setMatrix(crsmatrix); + + params.set("debug", true); + params.remove("chebyshev: max eigenvalue"); + params.remove("chebyshev: min eigenvalue"); + + params.set("eigen-analysis: type", "power method"); + params.set("chebyshev: eigenvalue max iterations",30); + prec.setParameters(params); + prec.compute(); + + TEST_FLOATING_EQUALITY(prec.getLambdaMaxForApply(),expectedLambdaMax,2e-2); + + params.set("eigen-analysis: type", "cg"); + params.set("chebyshev: eigenvalue max iterations",10); + prec.setParameters(params); + prec.compute(); + + TEST_FLOATING_EQUALITY(prec.getLambdaMaxForApply(),expectedLambdaMax,2e-2); } #define UNIT_TEST_GROUP_SC_LO_GO(Scalar,LocalOrdinal,GlobalOrdinal) \ From 0b30ff3db9ed6f69d6b37a635f624fa0e4ca2720 Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Mon, 30 Aug 2021 18:40:24 -0600 Subject: [PATCH 50/70] MueLu ML2MueLuTranslator: Convert "eigen-analysis: type" for Chebyshev --- .../muelu/src/Interface/MueLu_ML2MueLuParameterTranslator.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/muelu/src/Interface/MueLu_ML2MueLuParameterTranslator.cpp b/packages/muelu/src/Interface/MueLu_ML2MueLuParameterTranslator.cpp index 3a9e875e807c..5c9e0c3c2e22 100644 --- a/packages/muelu/src/Interface/MueLu_ML2MueLuParameterTranslator.cpp +++ b/packages/muelu/src/Interface/MueLu_ML2MueLuParameterTranslator.cpp @@ -164,6 +164,8 @@ namespace MueLu { else { mueluss << "" << std::endl; } if ( paramList.isParameter("smoother: Chebyshev alpha") ) { mueluss << "("smoother: Chebyshev alpha") << "\"/>" << std::endl; adaptingParamList.remove("smoother: Chebyshev alpha",false); } else { mueluss << "" << std::endl; adaptingParamList.remove("smoother: Chebyshev alpha",false); } + if ( paramList.isParameter("eigen-analysis: type") ) { mueluss << "("eigen-analysis: type") << "\"/>" << std::endl; adaptingParamList.remove("eigen-analysis: type",false); } + else { mueluss << "" << std::endl; } } // MLS @@ -174,6 +176,8 @@ namespace MueLu { if ( paramList.isParameter("smoother: MLS alpha") ) { mueluss << "("smoother: MLS alpha") << "\"/>" << std::endl; adaptingParamList.remove("smoother: MLS alpha",false); } else if ( paramList.isParameter("smoother: Chebyshev alpha") ) { mueluss << "("smoother: Chebyshev alpha") << "\"/>" << std::endl; adaptingParamList.remove("smoother: Chebyshev alpha",false); } else { mueluss << "" << std::endl; } + if ( paramList.isParameter("eigen-analysis: type") ) { mueluss << "("eigen-analysis: type") << "\"/>" << std::endl; adaptingParamList.remove("eigen-analysis: type",false); } + else { mueluss << "" << std::endl; } } // parameters for ILU based preconditioners From c9148be492077d101ffa5b650741c306ef864e55 Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Tue, 31 Aug 2021 17:19:50 -0600 Subject: [PATCH 51/70] MueLu: Rebase gold files --- .../test/interface/default/Output/MLsmoother4b_epetra.gold | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/muelu/test/interface/default/Output/MLsmoother4b_epetra.gold b/packages/muelu/test/interface/default/Output/MLsmoother4b_epetra.gold index 9f844e7025d5..d45be8763b50 100644 --- a/packages/muelu/test/interface/default/Output/MLsmoother4b_epetra.gold +++ b/packages/muelu/test/interface/default/Output/MLsmoother4b_epetra.gold @@ -4,6 +4,7 @@ Setup Smoother (MueLu::IfpackSmoother{type = Chebyshev}) smoother -> chebyshev: degree = 2 chebyshev: ratio eigenvalue = 20 + eigen-analysis: type = cg Level 1 Prolongator smoothing (MueLu::SaPFactory) Matrix filtering (MueLu::FilteredAFactory) @@ -39,6 +40,7 @@ Setup Smoother (MueLu::IfpackSmoother{type = Chebyshev}) smoother -> chebyshev: degree = 2 chebyshev: ratio eigenvalue = 20 [unused] + eigen-analysis: type = cg Level 2 Prolongator smoothing (MueLu::SaPFactory) Matrix filtering (MueLu::FilteredAFactory) @@ -74,6 +76,7 @@ Setup Smoother (MueLu::IfpackSmoother{type = Chebyshev}) smoother -> chebyshev: degree = 2 chebyshev: ratio eigenvalue = 20 [unused] + eigen-analysis: type = cg Level 3 Prolongator smoothing (MueLu::SaPFactory) Matrix filtering (MueLu::FilteredAFactory) @@ -109,6 +112,7 @@ Setup Smoother (MueLu::IfpackSmoother{type = Chebyshev}) smoother -> chebyshev: degree = 2 chebyshev: ratio eigenvalue = 20 [unused] + eigen-analysis: type = cg Level 4 Prolongator smoothing (MueLu::SaPFactory) Matrix filtering (MueLu::FilteredAFactory) From 65fc34e08b2e08d6347ae3784711d2b1c2e21db5 Mon Sep 17 00:00:00 2001 From: Jacob Domagala Date: Thu, 2 Sep 2021 15:00:44 +0200 Subject: [PATCH 52/70] Fix PanzerAdaptersSTK_tScatterResidual_MPI_2 and PanzerAdaptersSTK_tScatterDirichletResidual_MPI_2 with UVM off (#9644) --- ...nzer_GatherSolution_BlockedEpetra_impl.hpp | 206 +++++++----------- ...erDirichletResidual_BlockedEpetra_decl.hpp | 49 +++-- ...erDirichletResidual_BlockedEpetra_impl.hpp | 196 ++++++++++------- ...zer_ScatterResidual_BlockedEpetra_impl.hpp | 120 +++++----- 4 files changed, 294 insertions(+), 277 deletions(-) diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedEpetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedEpetra_impl.hpp index a6c6a6295652..e01fa7b36ad8 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedEpetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedEpetra_impl.hpp @@ -202,7 +202,7 @@ preEvaluate( xBvRoGed_ = rcp_dynamic_cast(ged, true); return; } // end of the refactored ReadOnly way - + // Now try the old path. { ged = d.gedc->getDataObject(globalDataKey_); @@ -255,70 +255,53 @@ evaluateFields( const vector& localCellIds = this->wda(workset).cell_local_ids; int numFields(gatherFields_.size()), numCells(localCellIds.size()); - if (x_.is_null()) + // Loop over the fields to be gathered. + for (int fieldInd(0); fieldInd < numFields; ++fieldInd) { - // Loop over the fields to be gathered. - for (int fieldInd(0); fieldInd < numFields; ++fieldInd) - { - MDField& field = gatherFields_[fieldInd]; - int indexerId(indexerIds_[fieldInd]), - subFieldNum(subFieldIds_[fieldInd]); + MDField& field = gatherFields_[fieldInd]; + auto field_h = Kokkos::create_mirror_view(field.get_static_view()); - // Grab the local data for inputing. - auto xEvRoGed = xBvRoGed_->getGEDBlock(indexerId); - auto subRowIndexer = indexers_[indexerId]; - const vector& elmtOffset = - subRowIndexer->getGIDFieldOffsets(blockId, subFieldNum); - int numBases(elmtOffset.size()); + int indexerId(indexerIds_[fieldInd]), + subFieldNum(subFieldIds_[fieldInd]); - // Gather operation for each cell in the workset. - for (int cell(0); cell < numCells; ++cell) - { - LO cellLocalId = localCellIds[cell]; - auto LIDs = subRowIndexer->getElementLIDs(cellLocalId); - - // Loop over the basis functions and fill the fields. - for (int basis(0); basis < numBases; ++basis) - { - int offset(elmtOffset[basis]), lid(LIDs[offset]); - field(cell, basis) = (*xEvRoGed)[lid]; - } // end loop over the basis functions - } // end loop over localCellIds - } // end loop over the fields to be gathered - } - else // if (not x_.is_null()) - { - // Loop over the fields to be gathered. - for (int fieldInd(0); fieldInd < numFields; ++fieldInd) - { - MDField& field = gatherFields_[fieldInd]; - int indexerId(indexerIds_[fieldInd]), - subFieldNum(subFieldIds_[fieldInd]); + // Grab the local data for inputing. + ArrayRCP x; + Teuchos::RCP xEvRoGed; - // Grab the local data for inputing. - ArrayRCP x; + if(not x_.is_null()) { rcp_dynamic_cast>(x_-> getNonconstVectorBlock(indexerId))->getLocalData(ptrFromRef(x)); - auto subRowIndexer = indexers_[indexerId]; - const vector& elmtOffset = - subRowIndexer->getGIDFieldOffsets(blockId, subFieldNum); - int numBases(elmtOffset.size()); + } + else { + xEvRoGed = xBvRoGed_->getGEDBlock(indexerId); + } + + auto subRowIndexer = indexers_[indexerId]; + const vector& elmtOffset = + subRowIndexer->getGIDFieldOffsets(blockId, subFieldNum); + int numBases(elmtOffset.size()); + + auto LIDs = subRowIndexer->getLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); + + // Gather operation for each cell in the workset. + for (int cell(0); cell < numCells; ++cell) + { + LO cellLocalId = localCellIds[cell]; - // Gather operation for each cell in the workset. - for (int cell(0); cell < numCells; ++cell) + // Loop over the basis functions and fill the fields. + for (int basis(0); basis < numBases; ++basis) { - LO cellLocalId = localCellIds[cell]; - auto LIDs = subRowIndexer->getElementLIDs(cellLocalId); - - // Loop over the basis functions and fill the fields. - for (int basis(0); basis < numBases; ++basis) - { - int offset(elmtOffset[basis]), lid(LIDs[offset]); - field(cell, basis) = x[lid]; - } // end loop over the basis functions - } // end loop over localCellIds - } // end loop over the fields to be gathered - } // end if (x_.is_null()) or not + int offset(elmtOffset[basis]), lid(LIDs_h(cellLocalId, offset)); + if(x_.is_null()) + field_h(cell, basis) = (*xEvRoGed)[lid]; + else + field_h(cell, basis) = x[lid]; + } // end loop over the basis functions + } // end loop over localCellIds + Kokkos::deep_copy(field.get_static_view(), field_h); + } // end loop over the fields to be gathered } // end of evaluateFields() (Residual Specialization) /////////////////////////////////////////////////////////////////////////////// @@ -451,7 +434,7 @@ preEvaluate( xBvRoGed_ = rcp_dynamic_cast(ged, true); return; } // end of the refactored ReadOnly way - + // Now try the old path. { ged = d.gedc->getDataObject(globalDataKey_); @@ -721,7 +704,7 @@ preEvaluate( xBvRoGed_ = rcp_dynamic_cast(ged, true); return; } // end of the refactored ReadOnly way - + // Now try the old path. { ged = d.gedc->getDataObject(globalDataKey_); @@ -799,74 +782,53 @@ evaluateFields( // NOTE: A reordering of these loops will likely improve performance. The // "getGIDFieldOffsets may be expensive. However the "getElementGIDs" // can be cheaper. However the lookup for LIDs may be more expensive! - if (x_.is_null()) - { - // Loop over the fields to be gathered. - for (int fieldInd(0); fieldInd < numFields; ++fieldInd) - { - MDField& field = gatherFields_[fieldInd]; - int indexerId(indexerIds_[fieldInd]), - subFieldNum(subFieldIds_[fieldInd]); - // Grab the local data for inputing. - auto xEvRoGed = xBvRoGed_->getGEDBlock(indexerId); - auto subRowIndexer = indexers_[indexerId]; - const vector& elmtOffset = - subRowIndexer->getGIDFieldOffsets(blockId, subFieldNum); - int startBlkOffset(blockOffsets[indexerId]), numBases(elmtOffset.size()); - - // Gather operation for each cell in the workset. - for (int cell(0); cell < numCells; ++cell) - { - LO cellLocalId = localCellIds[cell]; - auto LIDs = subRowIndexer->getElementLIDs(cellLocalId); - - // Loop over the basis functions and fill the fields. - for (int basis(0); basis < numBases; ++basis) - { - // Set the value and seed the FAD object. - int offset(elmtOffset[basis]), lid(LIDs[offset]); - field(cell, basis) = ScalarT(numDerivs, (*xEvRoGed)[lid]); - field(cell, basis).fastAccessDx(startBlkOffset + offset) = seedValue; - } // end loop over the basis functions - } // end loop over localCellIds - } // end loop over the fields to be gathered - } - else // if (not x_.is_null()) + // Loop over the fields to be gathered. + for (int fieldInd(0); fieldInd < numFields; ++fieldInd) { - // Loop over the fields to be gathered. - for (int fieldInd(0); fieldInd < numFields; ++fieldInd) + MDField& field = gatherFields_[fieldInd]; + auto field_h = Kokkos::create_mirror_view(field.get_view()); + + int indexerId(indexerIds_[fieldInd]), subFieldNum(subFieldIds_[fieldInd]); + + // Grab the local data for inputing. + ArrayRCP x; + Teuchos::RCP xEvRoGed; + if(not x_.is_null()) { + rcp_dynamic_cast>(x_->getNonconstVectorBlock(indexerId))->getLocalData(ptrFromRef(x)); + }else { + xEvRoGed = xBvRoGed_->getGEDBlock(indexerId); + } + + auto subRowIndexer = indexers_[indexerId]; + const vector& elmtOffset = + subRowIndexer->getGIDFieldOffsets(blockId, subFieldNum); + int startBlkOffset(blockOffsets[indexerId]), numBases(elmtOffset.size()); + + auto LIDs = subRowIndexer->getLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); + + // Gather operation for each cell in the workset. + for (int cell(0); cell < numCells; ++cell) { - MDField& field = gatherFields_[fieldInd]; - int indexerId(indexerIds_[fieldInd]), - subFieldNum(subFieldIds_[fieldInd]); - - // Grab the local data for inputing. - ArrayRCP x; - rcp_dynamic_cast>(x_-> - getNonconstVectorBlock(indexerId))->getLocalData(ptrFromRef(x)); - auto subRowIndexer = indexers_[indexerId]; - const vector& elmtOffset = - subRowIndexer->getGIDFieldOffsets(blockId, subFieldNum); - int startBlkOffset(blockOffsets[indexerId]), numBases(elmtOffset.size()); + LO cellLocalId = localCellIds[cell]; - // Gather operation for each cell in the workset. - for (int cell(0); cell < numCells; ++cell) + // Loop over the basis functions and fill the fields. + for (int basis(0); basis < numBases; ++basis) { - LO cellLocalId = localCellIds[cell]; - auto LIDs = subRowIndexer->getElementLIDs(cellLocalId); - - // Loop over the basis functions and fill the fields. - for (int basis(0); basis < numBases; ++basis) - { - // Set the value and seed the FAD object. - int offset(elmtOffset[basis]), lid(LIDs[offset]); - field(cell, basis) = ScalarT(numDerivs, x[lid]); - field(cell, basis).fastAccessDx(startBlkOffset + offset) = seedValue; - } // end loop over the basis functions - } // end loop over localCellIds - } // end loop over the fields to be gathered - } // end if (x_.is_null()) or not + // Set the value and seed the FAD object. + int offset(elmtOffset[basis]), lid(LIDs_h(cellLocalId, offset)); + if(x_.is_null()) + field_h(cell, basis) = ScalarT(numDerivs, (*xEvRoGed)[lid]); + else + field_h(cell, basis) = ScalarT(numDerivs, x[lid]); + + field_h(cell, basis).fastAccessDx(startBlkOffset + offset) = seedValue; + } // end loop over the basis functions + } // end loop over localCellIds + Kokkos::deep_copy(field.get_static_view(), field_h); + } // end loop over the fields to be gathered } // end of evaluateFields() (Jacobian Specialization) #endif // __Panzer_GatherSolution_BlockedEpetra_impl_hpp__ diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedEpetra_decl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedEpetra_decl.hpp index a648cdfe0b6c..acbc4abda97c 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedEpetra_decl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedEpetra_decl.hpp @@ -66,7 +66,7 @@ namespace panzer { class GlobalIndexer; -/** \brief Pushes residual values into the residual vector for a +/** \brief Pushes residual values into the residual vector for a Newton-based solve Currently makes an assumption that the stride is constant for dofs @@ -86,9 +86,9 @@ template class ScatterD virtual Teuchos::RCP clone(const Teuchos::ParameterList & pl) const { return Teuchos::rcp(new ScatterDirichletResidual_BlockedEpetra(pl)); } - void postRegistrationSetup(typename TRAITS::SetupData d, PHX::FieldManager& vm) + void postRegistrationSetup(typename TRAITS::SetupData d, PHX::FieldManager& vm) { } - void evaluateFields(typename TRAITS::EvalData d) + void evaluateFields(typename TRAITS::EvalData d) { std::cout << "unspecialized version of \"ScatterDirichletResidual_BlockedEpetra::evaluateFields\" on \""+PHX::print()+"\" should not be used!" << std::endl; TEUCHOS_ASSERT(false); } }; @@ -101,37 +101,38 @@ template class ScatterD // ************************************************************** -// Residual +// Residual // ************************************************************** template class ScatterDirichletResidual_BlockedEpetra : public panzer::EvaluatorWithBaseImpl, public PHX::EvaluatorDerived, public panzer::CloneableEvaluator { - + public: ScatterDirichletResidual_BlockedEpetra(const std::vector > & rIndexers, const std::vector > & /* cIndexers */) : rowIndexers_(rIndexers) {} - + ScatterDirichletResidual_BlockedEpetra(const std::vector > & rIndexers, const std::vector > & cIndexers, const Teuchos::ParameterList& p, bool useDiscreteAdjoint=false); - + void postRegistrationSetup(typename TRAITS::SetupData d, PHX::FieldManager& vm); void preEvaluate(typename TRAITS::PreEvalData d); - + void evaluateFields(typename TRAITS::EvalData workset); - + virtual Teuchos::RCP clone(const Teuchos::ParameterList & pl) const { return Teuchos::rcp(new ScatterDirichletResidual_BlockedEpetra(rowIndexers_,colIndexers_,pl)); } private: typedef typename panzer::Traits::Residual::ScalarT ScalarT; + using BCFieldType = PHX::MDField; // dummy field so that the evaluator will have something to do Teuchos::RCP scatterHolder_; @@ -167,43 +168,44 @@ class ScatterDirichletResidual_BlockedEpetra > applyBC_; + std::vector< BCFieldType > applyBC_; ScatterDirichletResidual_BlockedEpetra() {} }; // ************************************************************** -// Tangent +// Tangent // ************************************************************** template class ScatterDirichletResidual_BlockedEpetra : public panzer::EvaluatorWithBaseImpl, public PHX::EvaluatorDerived, public panzer::CloneableEvaluator { - + public: ScatterDirichletResidual_BlockedEpetra(const std::vector > & rIndexers, const std::vector > & /* cIndexers */) : rowIndexers_(rIndexers) {} - + ScatterDirichletResidual_BlockedEpetra(const std::vector > & rIndexers, const std::vector > & cIndexers, const Teuchos::ParameterList& p, bool useDiscreteAdjoint=false); - + void postRegistrationSetup(typename TRAITS::SetupData d, PHX::FieldManager& vm); void preEvaluate(typename TRAITS::PreEvalData d); - + void evaluateFields(typename TRAITS::EvalData workset); - + virtual Teuchos::RCP clone(const Teuchos::ParameterList & pl) const { return Teuchos::rcp(new ScatterDirichletResidual_BlockedEpetra(rowIndexers_,colIndexers_,pl)); } private: typedef typename panzer::Traits::Tangent::ScalarT ScalarT; + using BCFieldType = PHX::MDField; // dummy field so that the evaluator will have something to do Teuchos::RCP scatterHolder_; @@ -239,7 +241,7 @@ class ScatterDirichletResidual_BlockedEpetra > applyBC_; + std::vector< BCFieldType > applyBC_; ScatterDirichletResidual_BlockedEpetra() {} }; @@ -252,31 +254,32 @@ class ScatterDirichletResidual_BlockedEpetra, public PHX::EvaluatorDerived, public panzer::CloneableEvaluator { - + public: ScatterDirichletResidual_BlockedEpetra(const std::vector > & rIndexers, const std::vector > & cIndexers) : rowIndexers_(rIndexers), colIndexers_(cIndexers) {} - + ScatterDirichletResidual_BlockedEpetra(const std::vector > & rIndexers, const std::vector > & cIndexers, const Teuchos::ParameterList& p, bool useDiscreteAdjoint=false); void preEvaluate(typename TRAITS::PreEvalData d); - + void postRegistrationSetup(typename TRAITS::SetupData d, PHX::FieldManager& vm); - + void evaluateFields(typename TRAITS::EvalData workset); virtual Teuchos::RCP clone(const Teuchos::ParameterList & pl) const { return Teuchos::rcp(new ScatterDirichletResidual_BlockedEpetra(rowIndexers_,colIndexers_,pl)); } - + private: typedef typename panzer::Traits::Jacobian::ScalarT ScalarT; + using BCFieldType = PHX::MDField; // dummy field so that the evaluator will have something to do Teuchos::RCP scatterHolder_; @@ -312,7 +315,7 @@ class ScatterDirichletResidual_BlockedEpetra > applyBC_; + std::vector< BCFieldType > applyBC_; ScatterDirichletResidual_BlockedEpetra(); }; diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedEpetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedEpetra_impl.hpp index 8f88b533ea8f..6f56ef3eec2a 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedEpetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_BlockedEpetra_impl.hpp @@ -85,13 +85,13 @@ ScatterDirichletResidual_BlockedEpetra(const std::vector("Scatter Name"); - scatterHolder_ = + scatterHolder_ = Teuchos::rcp(new PHX::Tag(scatterName,Teuchos::rcp(new PHX::MDALayout(0)))); // get names to be evaluated - const std::vector& names = + const std::vector& names = *(p.get< Teuchos::RCP< std::vector > >("Dependent Names")); // grab map from evaluated names to field names @@ -136,9 +136,9 @@ ScatterDirichletResidual_BlockedEpetra(const std::vector +template void panzer::ScatterDirichletResidual_BlockedEpetra:: -postRegistrationSetup(typename TRAITS::SetupData /* d */, +postRegistrationSetup(typename TRAITS::SetupData /* d */, PHX::FieldManager& /* fm */) { indexerIds_.resize(scatterFields_.size()); @@ -169,7 +169,7 @@ preEvaluate(typename TRAITS::PreEvalData d) using Thyra::ProductVectorBase; // extract dirichlet counter from container - Teuchos::RCP blockContainer + Teuchos::RCP blockContainer = Teuchos::rcp_dynamic_cast(d.gedc->getDataObject("Dirichlet Counter"),true); dirichletCounter_ = Teuchos::rcp_dynamic_cast >(blockContainer->get_f(),true); @@ -181,11 +181,11 @@ preEvaluate(typename TRAITS::PreEvalData d) // if its blocked do this if(blockedContainer!=Teuchos::null) - r_ = (!scatterIC_) ? + r_ = (!scatterIC_) ? rcp_dynamic_cast >(blockedContainer->get_f(),true) : rcp_dynamic_cast >(blockedContainer->get_x(),true); else if(epetraContainer!=Teuchos::null) // if its straight up epetra do this - r_ = (!scatterIC_) ? + r_ = (!scatterIC_) ? Thyra::castOrCreateNonconstProductVectorBase(epetraContainer->get_f_th()) : Thyra::castOrCreateNonconstProductVectorBase(epetraContainer->get_x_th()); @@ -196,7 +196,7 @@ preEvaluate(typename TRAITS::PreEvalData d) template void panzer::ScatterDirichletResidual_BlockedEpetra:: evaluateFields(typename TRAITS::EvalData workset) -{ +{ using Teuchos::RCP; using Teuchos::ArrayRCP; using Teuchos::ptrFromRef; @@ -214,7 +214,7 @@ evaluateFields(typename TRAITS::EvalData workset) // The "getGIDFieldOffsets may be expensive. However the // "getElementGIDs" can be cheaper. However the lookup for LIDs // may be more expensive! - + // loop over each field to be scattered Teuchos::ArrayRCP local_r; Teuchos::ArrayRCP local_dc; @@ -230,35 +230,46 @@ evaluateFields(typename TRAITS::EvalData workset) ->getNonconstLocalData(ptrFromRef(local_r)); auto subRowIndexer = rowIndexers_[rowIndexer]; + auto LIDs = subRowIndexer->getLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); + + auto field = scatterFields_[fieldIndex].get_view(); + auto field_h = Kokkos::create_mirror_view(field); + Kokkos::deep_copy(field_h, field); + + BCFieldType::array_type::HostMirror applyBC_h; + if(checkApplyBC_){ + auto applyBC = applyBC_[fieldIndex].get_static_view(); + applyBC_h = Kokkos::create_mirror_view(applyBC); + Kokkos::deep_copy(applyBC_h, applyBC); + } // scatter operation for each cell in workset for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementLIDs(cellLocalId); - if (!scatterIC_) { // this call "should" get the right ordering according to the Intrepid2 basis - const std::pair,std::vector > & indicePair + const std::pair,std::vector > & indicePair = subRowIndexer->getGIDFieldOffsets_closure(blockId,subFieldNum, side_subcell_dim_, local_side_id_); const std::vector & elmtOffset = indicePair.first; const std::vector & basisIdMap = indicePair.second; - + // loop over basis functions for(std::size_t basis=0;basis("Scatter Name"); - scatterHolder_ = + scatterHolder_ = Teuchos::rcp(new PHX::Tag(scatterName,Teuchos::rcp(new PHX::MDALayout(0)))); // get names to be evaluated - const std::vector& names = + const std::vector& names = *(p.get< Teuchos::RCP< std::vector > >("Dependent Names")); // grab map from evaluated names to field names @@ -348,9 +359,9 @@ ScatterDirichletResidual_BlockedEpetra(const std::vector +template void panzer::ScatterDirichletResidual_BlockedEpetra:: -postRegistrationSetup(typename TRAITS::SetupData /* d */, +postRegistrationSetup(typename TRAITS::SetupData /* d */, PHX::FieldManager& /* fm */) { indexerIds_.resize(scatterFields_.size()); @@ -381,7 +392,7 @@ preEvaluate(typename TRAITS::PreEvalData d) using Thyra::ProductVectorBase; // extract dirichlet counter from container - Teuchos::RCP blockContainer + Teuchos::RCP blockContainer = Teuchos::rcp_dynamic_cast(d.gedc->getDataObject("Dirichlet Counter"),true); dirichletCounter_ = Teuchos::rcp_dynamic_cast >(blockContainer->get_f(),true); @@ -393,11 +404,11 @@ preEvaluate(typename TRAITS::PreEvalData d) // if its blocked do this if(blockedContainer!=Teuchos::null) - r_ = (!scatterIC_) ? + r_ = (!scatterIC_) ? rcp_dynamic_cast >(blockedContainer->get_f(),true) : rcp_dynamic_cast >(blockedContainer->get_x(),true); else if(epetraContainer!=Teuchos::null) // if its straight up epetra do this - r_ = (!scatterIC_) ? + r_ = (!scatterIC_) ? Thyra::castOrCreateNonconstProductVectorBase(epetraContainer->get_f_th()) : Thyra::castOrCreateNonconstProductVectorBase(epetraContainer->get_x_th()); @@ -408,7 +419,7 @@ preEvaluate(typename TRAITS::PreEvalData d) template void panzer::ScatterDirichletResidual_BlockedEpetra:: evaluateFields(typename TRAITS::EvalData workset) -{ +{ TEUCHOS_ASSERT(false); using Teuchos::RCP; @@ -445,33 +456,45 @@ evaluateFields(typename TRAITS::EvalData workset) auto subRowIndexer = rowIndexers_[rowIndexer]; + auto LIDs = subRowIndexer->getLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); + + auto field = scatterFields_[fieldIndex].get_view(); + auto field_h = Kokkos::create_mirror_view(field); + Kokkos::deep_copy(field_h, field); + + BCFieldType::array_type::HostMirror applyBC_h; + if(checkApplyBC_){ + auto applyBC = applyBC_[fieldIndex].get_static_view(); + applyBC_h = Kokkos::create_mirror_view(applyBC); + Kokkos::deep_copy(applyBC_h, applyBC); + } + // scatter operation for each cell in workset for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementLIDs(cellLocalId); - if (!scatterIC_) { // this call "should" get the right ordering according to the Intrepid2 basis - const std::pair,std::vector > & indicePair + const std::pair,std::vector > & indicePair = subRowIndexer->getGIDFieldOffsets_closure(blockId,subFieldNum, side_subcell_dim_, local_side_id_); const std::vector & elmtOffset = indicePair.first; const std::vector & basisIdMap = indicePair.second; - + // loop over basis functions for(std::size_t basis=0;basis("Scatter Name"); - scatterHolder_ = + scatterHolder_ = Teuchos::rcp(new PHX::Tag(scatterName,Teuchos::rcp(new PHX::MDALayout(0)))); // get names to be evaluated - const std::vector& names = + const std::vector& names = *(p.get< Teuchos::RCP< std::vector > >("Dependent Names")); // grab map from evaluated names to field names fieldMap_ = p.get< Teuchos::RCP< std::map > >("Dependent Map"); - Teuchos::RCP dl = + Teuchos::RCP dl = p.get< Teuchos::RCP >("Basis")->functional; side_subcell_dim_ = p.get("Side Subcell Dimension"); local_side_id_ = p.get("Local Side ID"); - + // build the vector of fields that this is dependent on scatterFields_.resize(names.size()); for (std::size_t eq = 0; eq < names.size(); ++eq) { @@ -559,7 +582,7 @@ ScatterDirichletResidual_BlockedEpetra(const std::vector +template void panzer::ScatterDirichletResidual_BlockedEpetra:: postRegistrationSetup(typename TRAITS::SetupData /* d */, PHX::FieldManager& /* fm */) @@ -591,7 +614,7 @@ preEvaluate(typename TRAITS::PreEvalData d) using Teuchos::rcp_dynamic_cast; // extract dirichlet counter from container - Teuchos::RCP blockContainer + Teuchos::RCP blockContainer = rcp_dynamic_cast(d.gedc->getDataObject("Dirichlet Counter"),true); dirichletCounter_ = rcp_dynamic_cast >(blockContainer->get_f(),true); @@ -609,7 +632,7 @@ preEvaluate(typename TRAITS::PreEvalData d) template void panzer::ScatterDirichletResidual_BlockedEpetra:: evaluateFields(typename TRAITS::EvalData workset) -{ +{ using Teuchos::RCP; using Teuchos::ArrayRCP; using Teuchos::ptrFromRef; @@ -632,7 +655,7 @@ evaluateFields(typename TRAITS::EvalData workset) // The "getGIDFieldOffsets may be expensive. However the // "getElementGIDs" can be cheaper. However the lookup for LIDs // may be more expensive! - + for(std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) { int rowIndexer = indexerIds_[fieldIndex]; int subFieldNum = subFieldIds_[fieldIndex]; @@ -653,39 +676,51 @@ evaluateFields(typename TRAITS::EvalData workset) const std::vector & subElmtOffset = subIndicePair.first; const std::vector & subBasisIdMap = subIndicePair.second; + auto rLIDs = subRowIndexer->getLIDs(); + auto rLIDs_h = Kokkos::create_mirror_view(rLIDs); + Kokkos::deep_copy(rLIDs_h, rLIDs); + + auto field = scatterFields_[fieldIndex].get_view(); + auto field_h = Kokkos::create_mirror_view(field); + Kokkos::deep_copy(field_h, field); + + BCFieldType::array_type::HostMirror applyBC_h; + if(checkApplyBC_){ + auto applyBC = applyBC_[fieldIndex].get_static_view(); + applyBC_h = Kokkos::create_mirror_view(applyBC); + Kokkos::deep_copy(applyBC_h, applyBC); + } + // scatter operation for each cell in workset for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementLIDs(cellLocalId); - // loop over basis functions for(std::size_t basis=0;basis blockIndex = std::make_pair(rowIndexer,colIndexer); Teuchos::RCP subJac = jacEpetraBlocks[blockIndex]; -// if you didn't find one before, add it to the hash table + // if you didn't find one before, add it to the hash table if(subJac==Teuchos::null) { - Teuchos::RCP > tOp = Jac_->getNonconstBlock(blockIndex.first,blockIndex.second); + Teuchos::RCP > tOp = Jac_->getNonconstBlock(blockIndex.first,blockIndex.second); // block operator is null, don't do anything (it is excluded) if(Teuchos::is_null(tOp)) @@ -705,28 +740,31 @@ evaluateFields(typename TRAITS::EvalData workset) for(int i=0;i jacRow(scatterField.size(),0.0); - + for(int sensIndex=0;sensIndexgetElementLIDs(cellLocalId); + + auto cLIDs = subColIndexer->getElementLIDs(cellLocalId); + auto cLIDs_h = Kokkos::create_mirror_view(cLIDs); + Kokkos::deep_copy(cLIDs_h, cLIDs); TEUCHOS_ASSERT(end-start==Teuchos::as(cLIDs.size())); @@ -736,7 +774,7 @@ evaluateFields(typename TRAITS::EvalData workset) // if you didn't find one before, add it to the hash table if(subJac==Teuchos::null) { - Teuchos::RCP > tOp = Jac_->getNonconstBlock(blockIndex.first,blockIndex.second); + Teuchos::RCP > tOp = Jac_->getNonconstBlock(blockIndex.first,blockIndex.second); // block operator is null, don't do anything (it is excluded) if(Teuchos::is_null(tOp)) @@ -748,19 +786,19 @@ evaluateFields(typename TRAITS::EvalData workset) } // Sum Jacobian - int err = subJac->ReplaceMyValues(lid, end-start, &jacRow[start],&cLIDs[0]); + int err = subJac->ReplaceMyValues(lid, end-start, &jacRow[start],&cLIDs_h[0]); if(err!=0) { std::stringstream ss; ss << "Failed inserting row: " << " (" << lid << "): "; for(int i=0;i("Scatter Name"); - scatterHolder_ = + scatterHolder_ = Teuchos::rcp(new PHX::Tag(scatterName,Teuchos::rcp(new PHX::MDALayout(0)))); // get names to be evaluated - const std::vector& names = + const std::vector& names = *(p.get< Teuchos::RCP< std::vector > >("Dependent Names")); // grab map from evaluated names to field names fieldMap_ = p.get< Teuchos::RCP< std::map > >("Dependent Map"); - Teuchos::RCP dl = + Teuchos::RCP dl = p.get< Teuchos::RCP >("Basis")->functional; - + // build the vector of fields that this is dependent on scatterFields_.resize(names.size()); for (std::size_t eq = 0; eq < names.size(); ++eq) { @@ -118,9 +118,9 @@ ScatterResidual_BlockedEpetra(const std::vector +template void panzer::ScatterResidual_BlockedEpetra:: -postRegistrationSetup(typename TRAITS::SetupData /* d */, +postRegistrationSetup(typename TRAITS::SetupData /* d */, PHX::FieldManager& /* fm */) { indexerIds_.resize(scatterFields_.size()); @@ -161,7 +161,7 @@ preEvaluate(typename TRAITS::PreEvalData d) template void panzer::ScatterResidual_BlockedEpetra:: evaluateFields(typename TRAITS::EvalData workset) -{ +{ using Teuchos::RCP; using Teuchos::ptrFromRef; using Teuchos::rcp_dynamic_cast; @@ -178,7 +178,7 @@ evaluateFields(typename TRAITS::EvalData workset) // The "getGIDFieldOffsets may be expensive. However the // "getElementGIDs" can be cheaper. However the lookup for LIDs // may be more expensive! - + // loop over each field to be scattered Teuchos::ArrayRCP local_r; for (std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) { @@ -191,17 +191,23 @@ evaluateFields(typename TRAITS::EvalData workset) auto subRowIndexer = rowIndexers_[indexerId]; const std::vector & elmtOffset = subRowIndexer->getGIDFieldOffsets(blockId,subFieldNum); + auto field = PHX::as_view(scatterFields_[fieldIndex]); + auto field_h = Kokkos::create_mirror_view(field); + Kokkos::deep_copy(field_h, field); + // scatter operation for each cell in workset for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementLIDs(cellLocalId); + auto LIDs = subRowIndexer->getElementLIDs(cellLocalId); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); // loop over basis functions for(std::size_t basis=0;basis > & cIndexers, const Teuchos::ParameterList& p, bool /* useDiscreteAdjoint */) - : rowIndexers_(rIndexers) - , colIndexers_(cIndexers) + : rowIndexers_(rIndexers) + , colIndexers_(cIndexers) , globalDataKey_("Residual Scatter Container") -{ +{ std::string scatterName = p.get("Scatter Name"); - scatterHolder_ = + scatterHolder_ = Teuchos::rcp(new PHX::Tag(scatterName,Teuchos::rcp(new PHX::MDALayout(0)))); // get names to be evaluated - const std::vector& names = + const std::vector& names = *(p.get< Teuchos::RCP< std::vector > >("Dependent Names")); // grab map from evaluated names to field names fieldMap_ = p.get< Teuchos::RCP< std::map > >("Dependent Map"); - Teuchos::RCP dl = + Teuchos::RCP dl = p.get< Teuchos::RCP >("Basis")->functional; - + // build the vector of fields that this is dependent on scatterFields_.resize(names.size()); for (std::size_t eq = 0; eq < names.size(); ++eq) { @@ -254,9 +260,9 @@ ScatterResidual_BlockedEpetra(const std::vector +template void panzer::ScatterResidual_BlockedEpetra:: -postRegistrationSetup(typename TRAITS::SetupData /* d */, +postRegistrationSetup(typename TRAITS::SetupData /* d */, PHX::FieldManager& /* fm */) { indexerIds_.resize(scatterFields_.size()); @@ -297,7 +303,7 @@ preEvaluate(typename TRAITS::PreEvalData d) template void panzer::ScatterResidual_BlockedEpetra:: evaluateFields(typename TRAITS::EvalData workset) -{ +{ TEUCHOS_ASSERT(false); using Teuchos::RCP; @@ -333,8 +339,8 @@ evaluateFields(typename TRAITS::EvalData workset) for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementLIDs(cellLocalId); - + auto LIDs = subRowIndexer->getElementLIDs(cellLocalId); + // loop over basis functions for(std::size_t basis=0;basis > & cIndexers, const Teuchos::ParameterList& p, bool useDiscreteAdjoint) - : rowIndexers_(rIndexers) - , colIndexers_(cIndexers) + : rowIndexers_(rIndexers) + , colIndexers_(cIndexers) , globalDataKey_("Residual Scatter Container") , useDiscreteAdjoint_(useDiscreteAdjoint) -{ +{ std::string scatterName = p.get("Scatter Name"); - scatterHolder_ = + scatterHolder_ = Teuchos::rcp(new PHX::Tag(scatterName,Teuchos::rcp(new PHX::MDALayout(0)))); // get names to be evaluated - const std::vector& names = + const std::vector& names = *(p.get< Teuchos::RCP< std::vector > >("Dependent Names")); // grab map from evaluated names to field names fieldMap_ = p.get< Teuchos::RCP< std::map > >("Dependent Map"); - Teuchos::RCP dl = + Teuchos::RCP dl = p.get< Teuchos::RCP >("Basis")->functional; - + // build the vector of fields that this is dependent on scatterFields_.resize(names.size()); for (std::size_t eq = 0; eq < names.size(); ++eq) { @@ -402,7 +408,7 @@ ScatterResidual_BlockedEpetra(const std::vector +template void panzer::ScatterResidual_BlockedEpetra:: postRegistrationSetup(typename TRAITS::SetupData /* d */, PHX::FieldManager& /* fm */) @@ -457,7 +463,7 @@ preEvaluate(typename TRAITS::PreEvalData d) template void panzer::ScatterResidual_BlockedEpetra:: evaluateFields(typename TRAITS::EvalData workset) -{ +{ using Teuchos::RCP; using Teuchos::ArrayRCP; using Teuchos::ptrFromRef; @@ -488,48 +494,56 @@ evaluateFields(typename TRAITS::EvalData workset) int subFieldNum = subFieldIds_[fieldIndex]; // grab local data for inputing - if(r_!=Teuchos::null) + if(r_!=Teuchos::null) rcp_dynamic_cast >(r_->getNonconstVectorBlock(rowIndexer))->getNonconstLocalData(ptrFromRef(local_r)); auto subRowIndexer = rowIndexers_[rowIndexer]; const std::vector & elmtOffset = subRowIndexer->getGIDFieldOffsets(blockId,subFieldNum); + auto field = scatterFields_[fieldIndex].get_view(); + auto field_h = Kokkos::create_mirror_view(field); + Kokkos::deep_copy(field_h, field); + + auto rLIDs = subRowIndexer->getLIDs(); + auto rLIDs_h = Kokkos::create_mirror_view(rLIDs); + Kokkos::deep_copy(rLIDs_h, rLIDs); + // scatter operation for each cell in workset for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementLIDs(cellLocalId); - // loop over the basis functions (currently they are nodes) for(std::size_t rowBasisNum = 0; rowBasisNum < elmtOffset.size(); rowBasisNum++) { - const ScalarT scatterField = (scatterFields_[fieldIndex])(worksetCellIndex,rowBasisNum); + const ScalarT scatterField = field_h(worksetCellIndex,rowBasisNum); int rowOffset = elmtOffset[rowBasisNum]; - int r_lid = rLIDs[rowOffset]; - + int r_lid = rLIDs_h(cellLocalId, rowOffset); + // Sum residual if(local_r!=Teuchos::null) local_r[r_lid] += (scatterField.val()); // loop over the sensitivity indices: all DOFs on a cell jacRow.resize(scatterField.size()); - + // For Neumann conditions with no dependence on degrees of freedom, there should be no Jacobian contribution if(scatterField.size() == 0) continue; - + for(int sensIndex=0;sensIndexgetElementLIDs(cellLocalId); + auto cLIDs = subColIndexer->getElementLIDs(cellLocalId); + auto cLIDs_h = Kokkos::create_mirror_view(cLIDs); + Kokkos::deep_copy(cLIDs_h, cLIDs); TEUCHOS_ASSERT(end-start==Teuchos::as(cLIDs.size())); @@ -540,7 +554,7 @@ evaluateFields(typename TRAITS::EvalData workset) // if you didn't find one before, add it to the hash table if(subJac==Teuchos::null) { - Teuchos::RCP > tOp = Jac_->getNonconstBlock(blockIndex.first,blockIndex.second); + Teuchos::RCP > tOp = Jac_->getNonconstBlock(blockIndex.first,blockIndex.second); // block operator is null, don't do anything (it is excluded) if(Teuchos::is_null(tOp)) @@ -553,26 +567,26 @@ evaluateFields(typename TRAITS::EvalData workset) // Sum Jacobian if(!useDiscreteAdjoint_) { - int err = subJac->SumIntoMyValues(r_lid, end-start, &jacRow[start],&cLIDs[0]); + int err = subJac->SumIntoMyValues(r_lid, end-start, &jacRow[start],&cLIDs_h[0]); if(err!=0) { - + std::stringstream ss; ss << "Failed inserting row: " << "LID = " << r_lid << "): "; for(int i=start;i >::const_iterator itr = meshCoordFields_.find(eBlock); if(itr==meshCoordFields_.end()) { // no coordinate field set for this element block @@ -1856,10 +1856,11 @@ void STK_Interface::getElementVertices_FromField(const std::vector diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_CoordinatesEvaluator_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_CoordinatesEvaluator_impl.hpp index 071eade9bf46..c40c4c1d5aa1 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_CoordinatesEvaluator_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_CoordinatesEvaluator_impl.hpp @@ -78,13 +78,17 @@ CoordinatesEvaluator:: evaluateFields( typename Traits::EvalData d) { - PHX::MDField coords = this->wda(d).cell_vertex_coordinates; + auto coords = this->wda(d).cell_vertex_coordinates.get_static_view(); + auto coordinate_v = coordinate.get_static_view(); + auto l_dimension = dimension; // const Kokkos::DynRankView & coords = this->wda(d).cell_vertex_coordinates; // copy coordinates directly into the field - for(index_t i=0;i Date: Thu, 2 Sep 2021 17:38:26 +0200 Subject: [PATCH 54/70] Fix PanzerAdaptersSTK_STK_ResponseLibraryTest2_MPI_2 with UVM off --- ...er_ResponseScatterEvaluator_Functional.hpp | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional.hpp b/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional.hpp index a99f1a8dad19..4cdfaa240a44 100644 --- a/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional.hpp +++ b/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional.hpp @@ -65,23 +65,23 @@ class FunctionalScatterBase { virtual ~FunctionalScatterBase() {} virtual void scatterDerivative(const PHX::MDField & cellIntegral, - panzer::Traits::EvalData workset, + panzer::Traits::EvalData workset, WorksetDetailsAccessor& wda, const std::vector > & dgdx) const = 0; #ifdef Panzer_BUILD_HESSIAN_SUPPORT virtual void scatterHessian(const PHX::MDField & cellIntegral, - panzer::Traits::EvalData workset, + panzer::Traits::EvalData workset, WorksetDetailsAccessor& wda, const std::vector > & d2gdx2) const = 0; #endif }; - + template class FunctionalScatter : public FunctionalScatterBase { public: FunctionalScatter(const Teuchos::RCP & globalIndexer) - { + { if(globalIndexer!=Teuchos::null) ugis_.push_back(globalIndexer); } @@ -90,19 +90,19 @@ class FunctionalScatter : public FunctionalScatterBase { : ugis_(ugis) {} void scatterDerivative(const PHX::MDField & cellIntegral, - panzer::Traits::EvalData workset, + panzer::Traits::EvalData workset, WorksetDetailsAccessor& wda, const std::vector > & dgdx) const; #ifdef Panzer_BUILD_HESSIAN_SUPPORT void scatterHessian(const PHX::MDField & cellIntegral, - panzer::Traits::EvalData workset, + panzer::Traits::EvalData workset, WorksetDetailsAccessor& wda, const std::vector > & d2gdx2) const; #endif private: - + std::vector > ugis_; }; @@ -111,7 +111,7 @@ class FunctionalScatter : public FunctionalScatterBase { */ template class ResponseScatterEvaluator_Functional : public panzer::EvaluatorWithBaseImpl, - public PHX::EvaluatorDerived { + public PHX::EvaluatorDerived { public: //! A constructor with concrete arguments instead of a parameter list. @@ -137,12 +137,12 @@ class ResponseScatterEvaluator_Functional : public panzer::EvaluatorWithBaseImpl template void FunctionalScatter::scatterDerivative(const PHX::MDField & cellIntegral, - panzer::Traits::EvalData workset, + panzer::Traits::EvalData workset, WorksetDetailsAccessor& wda, - const std::vector > & dgdx) const + const std::vector > & dgdx) const { PHX::View LIDs; - + // for convenience pull out some objects from workset std::string blockId = wda(workset).block_id; @@ -151,6 +151,9 @@ void FunctionalScatter::scatterDerivative(const PHX::MDField & localCellIds = wda(workset).cell_local_ids; for(std::size_t worksetCellIndex=0;worksetCellIndex::scatterDerivative(const PHX::MDFieldgetElementLIDs(cellLocalId); + LIDs = ugis_[b]->getElementLIDs(cellLocalId); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); Teuchos::ArrayRCP dgdx_b = dgdx[b]; // loop over basis functions for(std::size_t i=0;i::scatterDerivative(const PHX::MDField void FunctionalScatter::scatterHessian(const PHX::MDField & cellIntegral, - panzer::Traits::EvalData workset, + panzer::Traits::EvalData workset, WorksetDetailsAccessor& wda, - const std::vector > & d2gdx2) const + const std::vector > & d2gdx2) const { PHX::View LIDs; - + // for convenience pull out some objects from workset std::string blockId = wda(workset).block_id; @@ -196,7 +201,7 @@ void FunctionalScatter::scatterHessian(const PHX::MDFieldgetElementLIDs(cellLocalId); + LIDs = ugis_[b]->getElementLIDs(cellLocalId); Teuchos::ArrayRCP d2gdx2_b = d2gdx2[b]; From 07cd229e3fb3c9ba11029fb83370e2e17bc1ffd3 Mon Sep 17 00:00:00 2001 From: Sean Miller Date: Thu, 2 Sep 2021 14:00:27 -0600 Subject: [PATCH 55/70] Adding fix to stop basis values from applying orientations to virtual cells --- .../disc-fe/src/Panzer_BasisValues2.hpp | 7 ++++++- .../disc-fe/src/Panzer_BasisValues2_impl.hpp | 20 ++++++++++++------- .../panzer/disc-fe/src/Panzer_Workset.cpp | 14 ++++++------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/packages/panzer/disc-fe/src/Panzer_BasisValues2.hpp b/packages/panzer/disc-fe/src/Panzer_BasisValues2.hpp index 8be15b5bf69e..20bbdc3e1fec 100644 --- a/packages/panzer/disc-fe/src/Panzer_BasisValues2.hpp +++ b/packages/panzer/disc-fe/src/Panzer_BasisValues2.hpp @@ -284,6 +284,10 @@ namespace panzer { PHX::MDField cell_vertex_coordinates_; + // Number of cells to apply orientations to (required in situations where virtual cells exist) + int num_orientations_cells_; + + // Orientations object Teuchos::RCP orientations_; /// Used to check if arrays have been cached @@ -354,7 +358,8 @@ namespace panzer { /// Set the orientations object for applying orientations using the lazy evaluation path - required for certain bases void - setOrientations(const Teuchos::RCP & orientations); + setOrientations(const Teuchos::RCP & orientations, + const int num_orientations_cells = -1); /// Set the cubature weights (weighted measure) for the basis values object - required to get weighted basis objects void diff --git a/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp b/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp index e45c38db0bd1..fcfd580eb29b 100644 --- a/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp +++ b/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp @@ -94,6 +94,7 @@ applyOrientationsImpl(const int num_cells, const std::vector & orientations, const typename BasisValues2::IntrepidBasis & basis) { + // Move orientations vector to device Kokkos::DynRankView device_orientations("drv_orts", num_cells); auto host_orientations = Kokkos::create_mirror_view(device_orientations); @@ -651,8 +652,13 @@ setupUniform(const Teuchos::RCP & basis, template void BasisValues2:: -setOrientations(const Teuchos::RCP & orientations) +setOrientations(const Teuchos::RCP & orientations, + const int num_orientations_cells) { + if(num_orientations_cells < 0) + num_orientations_cells_ = num_evaluate_cells_; + else + num_orientations_cells_ = num_orientations_cells; if(orientations == Teuchos::null){ orientations_applied_ = false; orientations_ = Teuchos::null; @@ -1160,7 +1166,7 @@ getBasisValues(const bool weighted, // fix the logic. if(orientations_ != Teuchos::null) - applyOrientationsImpl(num_evaluate_cells_, tmp_basis_scalar.get_view(), *orientations_->getOrientations(), *intrepid_basis); + applyOrientationsImpl(num_orientations_cells_, tmp_basis_scalar.get_view(), *orientations_->getOrientations(), *intrepid_basis); // Store for later if cache is enabled PANZER_CACHE_DATA(basis_scalar); @@ -1341,7 +1347,7 @@ getVectorBasisValues(const bool weighted, } if(orientations_ != Teuchos::null) - applyOrientationsImpl(num_evaluate_cells_, tmp_basis_vector.get_view(), *orientations_->getOrientations(), *intrepid_basis); + applyOrientationsImpl(num_orientations_cells_, tmp_basis_vector.get_view(), *orientations_->getOrientations(), *intrepid_basis); // Store for later if cache is enabled PANZER_CACHE_DATA(basis_vector); @@ -1486,7 +1492,7 @@ getGradBasisValues(const bool weighted, } if(orientations_ != Teuchos::null) - applyOrientationsImpl(num_evaluate_cells_, tmp_grad_basis.get_view(), *orientations_->getOrientations(), *intrepid_basis); + applyOrientationsImpl(num_orientations_cells_, tmp_grad_basis.get_view(), *orientations_->getOrientations(), *intrepid_basis); // Store for later if cache is enabled PANZER_CACHE_DATA(grad_basis); @@ -1629,7 +1635,7 @@ getCurl2DVectorBasis(const bool weighted, } if(orientations_ != Teuchos::null) - applyOrientationsImpl(num_evaluate_cells_, tmp_curl_basis_scalar.get_view(), *orientations_->getOrientations(), *intrepid_basis); + applyOrientationsImpl(num_orientations_cells_, tmp_curl_basis_scalar.get_view(), *orientations_->getOrientations(), *intrepid_basis); // Store for later if cache is enabled PANZER_CACHE_DATA(curl_basis_scalar); @@ -1775,7 +1781,7 @@ getCurlVectorBasis(const bool weighted, } if(orientations_ != Teuchos::null) - applyOrientationsImpl(num_evaluate_cells_, tmp_curl_basis_vector.get_view(), *orientations_->getOrientations(), *intrepid_basis); + applyOrientationsImpl(num_orientations_cells_, tmp_curl_basis_vector.get_view(), *orientations_->getOrientations(), *intrepid_basis); // Store for later if cache is enabled PANZER_CACHE_DATA(curl_basis_vector); @@ -1911,7 +1917,7 @@ getDivVectorBasis(const bool weighted, } if(orientations_ != Teuchos::null) - applyOrientationsImpl(num_evaluate_cells_, tmp_div_basis.get_view(), *orientations_->getOrientations(), *intrepid_basis); + applyOrientationsImpl(num_orientations_cells_, tmp_div_basis.get_view(), *orientations_->getOrientations(), *intrepid_basis); // Store for later if cache is enabled PANZER_CACHE_DATA(div_basis); diff --git a/packages/panzer/disc-fe/src/Panzer_Workset.cpp b/packages/panzer/disc-fe/src/Panzer_Workset.cpp index 9e3ad49f2d33..c31f7a7bebc2 100644 --- a/packages/panzer/disc-fe/src/Panzer_Workset.cpp +++ b/packages/panzer/disc-fe/src/Panzer_Workset.cpp @@ -428,15 +428,15 @@ getBasisValues(const panzer::BasisDescriptor & basis_description, biv = Teuchos::rcp(new BasisValues2()); - biv->setOrientations(options_.orientations_); - biv->setWeightedMeasure(iv.weighted_measure); - biv->setCellVertexCoordinates(cell_vertex_coordinates); - if(integration_description.getType() == IntegrationDescriptor::VOLUME) biv->setupUniform(bir, iv.cub_points, iv.jac, iv.jac_det, iv.jac_inv); else biv->setup(bir, iv.ref_ip_coordinates, iv.jac, iv.jac_det, iv.jac_inv); + biv->setOrientations(options_.orientations_, numOwnedCells()+numGhostCells()); + biv->setWeightedMeasure(iv.weighted_measure); + biv->setCellVertexCoordinates(cell_vertex_coordinates); + } else { // Standard, fully allocated version of BasisValues2 @@ -557,11 +557,11 @@ getBasisValues(const panzer::BasisDescriptor & basis_description, bpv = Teuchos::rcp(new BasisValues2()); - bpv->setOrientations(options_.orientations_); - bpv->setCellVertexCoordinates(cell_vertex_coordinates); - bpv->setupUniform(bir, pv.coords_ref, pv.jac, pv.jac_det, pv.jac_inv); + bpv->setOrientations(options_.orientations_, numOwnedCells()+numGhostCells()); + bpv->setCellVertexCoordinates(cell_vertex_coordinates); + } else { // Standard fully allocated version From fce95807c02f1902812e121945f53f2b62fea452 Mon Sep 17 00:00:00 2001 From: Jacob Domagala Date: Fri, 3 Sep 2021 12:05:25 +0200 Subject: [PATCH 56/70] Simplify the scatterDerivative function, to reduce the number of deep copies --- ...er_ResponseScatterEvaluator_Functional.hpp | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional.hpp b/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional.hpp index 4cdfaa240a44..fbc1cb9381c6 100644 --- a/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional.hpp +++ b/packages/panzer/disc-fe/src/responses/Panzer_ResponseScatterEvaluator_Functional.hpp @@ -141,8 +141,6 @@ void FunctionalScatter::scatterDerivative(const PHX::MDField > & dgdx) const { - PHX::View LIDs; - // for convenience pull out some objects from workset std::string blockId = wda(workset).block_id; @@ -154,23 +152,24 @@ void FunctionalScatter::scatterDerivative(const PHX::MDField & localCellIds = wda(workset).cell_local_ids; - for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementLIDs(cellLocalId); - auto LIDs_h = Kokkos::create_mirror_view(LIDs); - Kokkos::deep_copy(LIDs_h, LIDs); + auto LIDs = ugis_[b]->getLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); - Teuchos::ArrayRCP dgdx_b = dgdx[b]; + Teuchos::ArrayRCP dgdx_b = dgdx[b]; + + // scatter operation for each cell in workset + for(std::size_t worksetCellIndex=0;worksetCellIndex Date: Fri, 3 Sep 2021 06:41:30 -0600 Subject: [PATCH 57/70] have epetra stack going partially through --- .../src/solvers/MiniEM_DiscreteCurl.hpp | 38 +++++++++++++------ .../src/solvers/MiniEM_DiscreteGradient.hpp | 8 +++- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/packages/panzer/mini-em/src/solvers/MiniEM_DiscreteCurl.hpp b/packages/panzer/mini-em/src/solvers/MiniEM_DiscreteCurl.hpp index 15754e240b35..3713995d4541 100644 --- a/packages/panzer/mini-em/src/solvers/MiniEM_DiscreteCurl.hpp +++ b/packages/panzer/mini-em/src/solvers/MiniEM_DiscreteCurl.hpp @@ -157,7 +157,6 @@ void addDiscreteCurlToRequestHandler( face_basis->getDofCoeffs(refDofCoeffs); auto dofCoeffs_h = Kokkos::create_mirror_view(dofCoeffs); auto refDofCoeffs_h = Kokkos::create_mirror_view(refDofCoeffs); - Kokkos::deep_copy(dofCoeffs_h, dofCoeffs); Kokkos::deep_copy(refDofCoeffs_h, refDofCoeffs); // set up the topology of each face in an element for computing DOF coefficients // in 2D coefficients are same as reference coefficients @@ -208,6 +207,7 @@ void addDiscreteCurlToRequestHandler( } } + Kokkos::deep_copy(dofCoeffs, dofCoeffs_h); //orient basis Kokkos::DynRankView elemOrts_d("elemOrts_d", 1); Kokkos::deep_copy(elemOrts_d, elemOrts); @@ -221,6 +221,7 @@ void addDiscreteCurlToRequestHandler( li::getBasisCoeffs(Kokkos::subview(basisCoeffsLI,Kokkos::ALL(),curlIter,Kokkos::ALL()), Kokkos::subview(curlAtDofCoords,Kokkos::ALL(),curlIter,Kokkos::ALL(),Kokkos::ALL()), dofCoeffs); auto basisCoeffsLI_h = Kokkos::create_mirror_view(basisCoeffsLI); Kokkos::deep_copy(basisCoeffsLI_h, basisCoeffsLI); + // get IDs for edges and faces std::vector fGIDs; face_ugi->getElementGIDs(elementIds[elemIter],fGIDs); @@ -348,10 +349,10 @@ void addDiscreteCurlToRequestHandler( // allocate some view Kokkos::DynRankView dofCoords("dofCoords", 1, hdivCardinality, dim); Kokkos::DynRankView basisCoeffsLI("basisCoeffsLI", 1, hcurlCardinality, hdivCardinality); - Kokkos::DynRankView elemOrts("elemOrts", 1); + typename Kokkos::DynRankView::HostMirror elemOrts("elemOrts", 1); typename Kokkos::DynRankView::HostMirror elemNodes("elemNodes", 1, numElemVertices); - Kokkos::DynRankView fOrt("fOrt", hdivCardinality); - Kokkos::DynRankView ortJacobian("ortJacobian", 2, 2); + typename Kokkos::DynRankView::HostMirror fOrt("fOrt", hdivCardinality); + typename Kokkos::DynRankView::HostMirror ortJacobian("ortJacobian", 2, 2); // the ranks of these depend on dimension Kokkos::DynRankView dofCoeffs; @@ -371,12 +372,16 @@ void addDiscreteCurlToRequestHandler( } face_basis->getDofCoeffs(refDofCoeffs); + auto dofCoeffs_h = Kokkos::create_mirror_view(dofCoeffs); + auto refDofCoeffs_h = Kokkos::create_mirror_view(refDofCoeffs); + Kokkos::deep_copy(refDofCoeffs_h, refDofCoeffs); // set up the topology of each face in an element for computing DOF coefficients // in 2D coefficients are same as reference coefficients - Kokkos::DynRankView sub_topologies(Kokkos::ViewAllocateWithoutInitializing("sub_topologies"), hdivCardinality); + typename Kokkos::DynRankView::HostMirror + sub_topologies(Kokkos::ViewAllocateWithoutInitializing("sub_topologies"), hdivCardinality); if(dim < 3) for(int i = 0; i < hdivCardinality; i++) - dofCoeffs(0,i) = refDofCoeffs(i); + dofCoeffs_h(0,i) = refDofCoeffs_h(i); else { for(int iface = 0; iface < hdivCardinality; iface++){ shards::CellTopology sub_topology(topology.getCellTopologyData(dim-1,iface)); @@ -415,27 +420,36 @@ void addDiscreteCurlToRequestHandler( Intrepid2::Impl::OrientationTools::getJacobianOfOrientationMap(ortJacobian, sub_topologies(iface), fOrt(iface)); auto ortJacobianDet = ortJacobian(0,0)*ortJacobian(1,1)-ortJacobian(1,0)*ortJacobian(0,1); for(int idim = 0; idim < dim; idim++) - dofCoeffs(0,iface,idim) = refDofCoeffs(iface,idim)*ortJacobianDet; + dofCoeffs_h(0,iface,idim) = refDofCoeffs_h(iface,idim)*ortJacobianDet; } } + Kokkos::deep_copy(dofCoeffs, dofCoeffs_h); //orient basis + Kokkos::DynRankView elemOrts_d("elemOrts_d", 1); + Kokkos::deep_copy(elemOrts_d, elemOrts); ots::modifyBasisByOrientation(curlAtDofCoords, curlAtDofCoordsNonOriented, - elemOrts, + elemOrts_d, edge_basis.get()); //get basis coefficients (dofs) for(int curlIter=0; curlIter fGIDs; face_ugi->getElementGIDs(elementIds[elemIter],fGIDs); std::vector eGIDs; edge_ugi->getElementGIDs(elementIds[elemIter],eGIDs); - auto fLIDs = face_ugi->getElementLIDs(elementIds[elemIter]); - auto eLIDs = edge_ugi->getElementLIDs(elementIds[elemIter]); + auto eLIDs_k = edge_ugi->getElementLIDs(elementIds[elemIter]); + auto fLIDs_k = face_ugi->getElementLIDs(elementIds[elemIter]); + auto eLIDs = Kokkos::create_mirror_view(eLIDs_k); + auto fLIDs = Kokkos::create_mirror_view(fLIDs_k); + Kokkos::deep_copy(eLIDs, eLIDs_k); + Kokkos::deep_copy(fLIDs, fLIDs_k); // need to know which faces are owned by this proc std::vector isOwned(fGIDs.size()); @@ -463,8 +477,8 @@ void addDiscreteCurlToRequestHandler( auto it = std::find(edges_on_face.begin(), edges_on_face.end(), curlIter); if(it!=edges_on_face.end()){ // normalize the values - if(std::abs(basisCoeffsLI(0,curlIter,fIter)) > 1.0e-10) - values[curlIter] = basisCoeffsLI(0,curlIter,fIter)*area_scaling; + if(std::abs(basisCoeffsLI_h(0,curlIter,fIter)) > 1.0e-10) + values[curlIter] = basisCoeffsLI_h(0,curlIter,fIter)*area_scaling; } } diff --git a/packages/panzer/mini-em/src/solvers/MiniEM_DiscreteGradient.hpp b/packages/panzer/mini-em/src/solvers/MiniEM_DiscreteGradient.hpp index c8f4b22b8934..c2055bcad0c0 100644 --- a/packages/panzer/mini-em/src/solvers/MiniEM_DiscreteGradient.hpp +++ b/packages/panzer/mini-em/src/solvers/MiniEM_DiscreteGradient.hpp @@ -212,8 +212,12 @@ void addDiscreteGradientToRequestHandler( eUgi->getElementGIDs(elementIds[elemIter],eGIDs); std::vector nGIDs; nUgi->getElementGIDs(elementIds[elemIter],nGIDs); - auto eLIDs = eUgi->getElementLIDs(elementIds[elemIter]); - auto nLIDs = nUgi->getElementLIDs(elementIds[elemIter]); + auto eLIDs_k = eUgi->getElementLIDs(elementIds[elemIter]); + auto nLIDs_k = nUgi->getElementLIDs(elementIds[elemIter]); + auto eLIDs = Kokkos::create_mirror_view(eLIDs_k); + auto nLIDs = Kokkos::create_mirror_view(nLIDs_k); + Kokkos::deep_copy(eLIDs, eLIDs_k); + Kokkos::deep_copy(nLIDs, nLIDs_k); std::vector isOwned; eUgi->ownedIndices(eGIDs,isOwned); From fda2a74710df83947402ebf1a314a2b5a2bd380d Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Fri, 3 Sep 2021 06:52:41 -0600 Subject: [PATCH 58/70] Fixed inv perm func --- .../MiniEM_InversePermeability_impl.hpp | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/panzer/mini-em/src/closures/MiniEM_InversePermeability_impl.hpp b/packages/panzer/mini-em/src/closures/MiniEM_InversePermeability_impl.hpp index 66b71d8731df..4c1ba0a6e60d 100644 --- a/packages/panzer/mini-em/src/closures/MiniEM_InversePermeability_impl.hpp +++ b/packages/panzer/mini-em/src/closures/MiniEM_InversePermeability_impl.hpp @@ -42,15 +42,17 @@ template void InversePermeability::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; - - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < permeability.extent_int(1); ++point) { - // const ScalarT& x = coords(cell,point,0); - // const ScalarT& y = coords(cell,point,1); - // const ScalarT& z = coords(cell,point,2); - permeability(cell,point) = 1.0/mu; - } - } + auto perm = permeability.get_static_view(); + auto inv_mu = 1./mu; + Kokkos::parallel_for (workset.num_cells, KOKKOS_LAMBDA (int cell) { + for (int point = 0; point < perm.extent_int(1); ++point) { + // const ScalarT& x = coords(cell,point,0); + // const ScalarT& y = coords(cell,point,1); + // const ScalarT& z = coords(cell,point,2); + perm(cell,point) = inv_mu; + } + }); + Kokkos::fence(); } //********************************************************************** From be9c70cb0a2f383afadf3d8f04bd6b4a31d31f14 Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Fri, 3 Sep 2021 07:20:22 -0600 Subject: [PATCH 59/70] fixed centroid code --- ...TK_ResponseEvaluatorFactory_SolutionWriter.hpp | 3 ++- ...sponseEvaluatorFactory_SolutionWriter_impl.hpp | 15 +++++++-------- .../Panzer_ReorderADValues_Evaluator_impl.hpp | 10 +++++++--- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/panzer/adapters-stk/src/responses/Panzer_STK_ResponseEvaluatorFactory_SolutionWriter.hpp b/packages/panzer/adapters-stk/src/responses/Panzer_STK_ResponseEvaluatorFactory_SolutionWriter.hpp index 8b43d6622e4e..4b7da90caea6 100644 --- a/packages/panzer/adapters-stk/src/responses/Panzer_STK_ResponseEvaluatorFactory_SolutionWriter.hpp +++ b/packages/panzer/adapters-stk/src/responses/Panzer_STK_ResponseEvaluatorFactory_SolutionWriter.hpp @@ -108,11 +108,12 @@ class ResponseEvaluatorFactory_SolutionWriter : public panzer::ResponseEvaluator void removeField(const std::string & fieldName) { removedFields_.push_back(fieldName); } -private: + // should be private but needs a lambda void computeReferenceCentroid(const std::map > & bases, int baseDimension, Kokkos::DynRankView & centroid) const; +private: //! Delete from the argument all the fields that are in the removedFields array void deleteRemovedFields(const std::vector & removedFields, std::vector > > & fields) const; diff --git a/packages/panzer/adapters-stk/src/responses/Panzer_STK_ResponseEvaluatorFactory_SolutionWriter_impl.hpp b/packages/panzer/adapters-stk/src/responses/Panzer_STK_ResponseEvaluatorFactory_SolutionWriter_impl.hpp index c57fffe5fc48..e72862c99816 100644 --- a/packages/panzer/adapters-stk/src/responses/Panzer_STK_ResponseEvaluatorFactory_SolutionWriter_impl.hpp +++ b/packages/panzer/adapters-stk/src/responses/Panzer_STK_ResponseEvaluatorFactory_SolutionWriter_impl.hpp @@ -280,6 +280,7 @@ computeReferenceCentroid(const std::map("centroid",1,baseDimension); + auto l_centroid = centroid; // loop over each possible basis for(std::map >::const_iterator itr=bases.begin(); @@ -293,14 +294,12 @@ computeReferenceCentroid(const std::mapgetDofCoords(coords); TEUCHOS_ASSERT(coords.rank()==2); TEUCHOS_ASSERT(coords.extent_int(1)==baseDimension); - - for(int i=0;i& inField = inFields_[fieldIndex]; - const PHX::MDField& outField = outFields_[fieldIndex]; + + const auto & inField_v = inFields_[fieldIndex].get_view(); + const auto & outField_v = outFields_[fieldIndex].get_view(); + auto inField = Kokkos::create_mirror_view(inField_v); + auto outField = Kokkos::create_mirror_view(outField_v); + Kokkos::deep_copy(inField, inField_v); if(inField.size()>0) { @@ -275,7 +279,7 @@ evaluateFields(typename TRAITS::EvalData /* workset */) } } - + Kokkos::deep_copy(outField_v, outField); } //Irina TOFIX From 7f12fa8329e2061a95f6b91bc056263cf7196303 Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Fri, 3 Sep 2021 07:47:34 -0600 Subject: [PATCH 60/70] more progress --- .../panzer/mini-em/example/BlockPrec/main.cpp | 1 + .../src/closures/MiniEM_Permittivity_impl.hpp | 9 +-------- .../src/closures/MiniEM_RandomForcing_impl.hpp | 17 +++++++++-------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/packages/panzer/mini-em/example/BlockPrec/main.cpp b/packages/panzer/mini-em/example/BlockPrec/main.cpp index 0eb3e88f0087..08ab2ada6e5e 100644 --- a/packages/panzer/mini-em/example/BlockPrec/main.cpp +++ b/packages/panzer/mini-em/example/BlockPrec/main.cpp @@ -13,6 +13,7 @@ #include "Teuchos_StackedTimer.hpp" #include "Teuchos_ScalarTraits.hpp" +#include "Kokkos_View_Fad.hpp" #include "KokkosCompat_ClassicNodeAPI_Wrapper.hpp" #include "Panzer_NodeType.hpp" diff --git a/packages/panzer/mini-em/src/closures/MiniEM_Permittivity_impl.hpp b/packages/panzer/mini-em/src/closures/MiniEM_Permittivity_impl.hpp index f06556abba53..cdbdaf0467a2 100644 --- a/packages/panzer/mini-em/src/closures/MiniEM_Permittivity_impl.hpp +++ b/packages/panzer/mini-em/src/closures/MiniEM_Permittivity_impl.hpp @@ -42,15 +42,8 @@ template void Permittivity::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; + Kokkos::deep_copy(permittivity.get_static_view(), epsilon); - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < permittivity.extent_int(1); ++point) { - // const ScalarT& x = coords(cell,point,0); - // const ScalarT& y = coords(cell,point,1); - // const ScalarT& z = coords(cell,point,2); - permittivity(cell,point) = epsilon; - } - } } //********************************************************************** diff --git a/packages/panzer/mini-em/src/closures/MiniEM_RandomForcing_impl.hpp b/packages/panzer/mini-em/src/closures/MiniEM_RandomForcing_impl.hpp index ecbaca3a6b34..1a7db6e8d735 100644 --- a/packages/panzer/mini-em/src/closures/MiniEM_RandomForcing_impl.hpp +++ b/packages/panzer/mini-em/src/closures/MiniEM_RandomForcing_impl.hpp @@ -47,29 +47,30 @@ void RandomForcing::evaluateFields(typename Traits::EvalData works using panzer::index_t; // double time = workset.time; - + auto current_h = Kokkos::create_mirror_view(current.get_static_view()); if (ir_dim == 3) { for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < current.extent_int(1); ++point) { + for (int point = 0; point < current_h.extent_int(1); ++point) { // const ScalarT& x = coords(cell,point,0); // const ScalarT& y = coords(cell,point,1); // const ScalarT& z = coords(cell,point,2); - current(cell,point,0) = rangeMult_ * double(std::rand())/double(RAND_MAX) + rangeShift_; - current(cell,point,1) = rangeMult_ * double(std::rand())/double(RAND_MAX) + rangeShift_; - current(cell,point,2) = rangeMult_ * double(std::rand())/double(RAND_MAX) + rangeShift_; + current_h(cell,point,0) = rangeMult_ * double(std::rand())/double(RAND_MAX) + rangeShift_; + current_h(cell,point,1) = rangeMult_ * double(std::rand())/double(RAND_MAX) + rangeShift_; + current_h(cell,point,2) = rangeMult_ * double(std::rand())/double(RAND_MAX) + rangeShift_; } } } else { for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < current.extent_int(1); ++point) { + for (int point = 0; point < current_h.extent_int(1); ++point) { // const ScalarT& x = coords(cell,point,0); // const ScalarT& y = coords(cell,point,1); - current(cell,point,0) = rangeMult_ * double(std::rand())/double(RAND_MAX) + rangeShift_; - current(cell,point,1) = rangeMult_ * double(std::rand())/double(RAND_MAX) + rangeShift_; + current_h(cell,point,0) = rangeMult_ * double(std::rand())/double(RAND_MAX) + rangeShift_; + current_h(cell,point,1) = rangeMult_ * double(std::rand())/double(RAND_MAX) + rangeShift_; } } } + Kokkos::deep_copy(current.get_static_view(), current_h); } //********************************************************************** From 6b92b5b28b69c5696e1985fa2aeac8d6b3ad64a9 Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Fri, 3 Sep 2021 08:06:03 -0600 Subject: [PATCH 61/70] more progress --- .../MiniEM_TensorConductivity_impl.hpp | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/packages/panzer/mini-em/src/closures/MiniEM_TensorConductivity_impl.hpp b/packages/panzer/mini-em/src/closures/MiniEM_TensorConductivity_impl.hpp index b63392f960e7..3d0d7741f542 100644 --- a/packages/panzer/mini-em/src/closures/MiniEM_TensorConductivity_impl.hpp +++ b/packages/panzer/mini-em/src/closures/MiniEM_TensorConductivity_impl.hpp @@ -49,39 +49,40 @@ template void TensorConductivity::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; - + auto conductivity_h = Kokkos::create_mirror_view(conductivity.get_static_view()); if (ir_dim == 3) { for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < conductivity.extent_int(1); ++point) { + for (int point = 0; point < conductivity_h.extent_int(1); ++point) { // const ScalarT& x = coords(cell,point,0); // const ScalarT& y = coords(cell,point,1); // const ScalarT& z = coords(cell,point,2); - conductivity(cell,point,0,0) = sigma * (1.0 + betax*betax); - conductivity(cell,point,0,1) = sigma * ( betax*betay - betaz); - conductivity(cell,point,0,2) = sigma * ( betax*betaz + betay); + conductivity_h(cell,point,0,0) = sigma * (1.0 + betax*betax); + conductivity_h(cell,point,0,1) = sigma * ( betax*betay - betaz); + conductivity_h(cell,point,0,2) = sigma * ( betax*betaz + betay); - conductivity(cell,point,1,0) = sigma * ( betay*betax + betaz); - conductivity(cell,point,1,1) = sigma * (1.0 + betay*betay); - conductivity(cell,point,1,2) = sigma * ( betay*betaz - betax); + conductivity_h(cell,point,1,0) = sigma * ( betay*betax + betaz); + conductivity_h(cell,point,1,1) = sigma * (1.0 + betay*betay); + conductivity_h(cell,point,1,2) = sigma * ( betay*betaz - betax); - conductivity(cell,point,2,0) = sigma * ( betaz*betax - betay); - conductivity(cell,point,2,1) = sigma * ( betaz*betay + betax); - conductivity(cell,point,2,2) = sigma * (1.0 + betaz*betaz); + conductivity_h(cell,point,2,0) = sigma * ( betaz*betax - betay); + conductivity_h(cell,point,2,1) = sigma * ( betaz*betay + betax); + conductivity_h(cell,point,2,2) = sigma * (1.0 + betaz*betaz); } } } else { for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < conductivity.extent_int(1); ++point) { + for (int point = 0; point < conductivity_h.extent_int(1); ++point) { // const ScalarT& x = coords(cell,point,0); // const ScalarT& y = coords(cell,point,1); - conductivity(cell,point,0,0) = sigma; - conductivity(cell,point,0,1) = 0.; + conductivity_h(cell,point,0,0) = sigma; + conductivity_h(cell,point,0,1) = 0.; - conductivity(cell,point,1,0) = 0.; - conductivity(cell,point,1,1) = sigma; + conductivity_h(cell,point,1,0) = 0.; + conductivity_h(cell,point,1,1) = sigma; } } } + Kokkos::deep_copy(conductivity.get_static_view(), conductivity_h); } //********************************************************************** From 01e243060d46c2ec9776d929da7567ebc3355af1 Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Mon, 6 Sep 2021 08:25:27 -0600 Subject: [PATCH 62/70] have some memory access errors resolved --- .../Example_SimpleSource_impl.hpp | 21 +++++++++++-------- .../Panzer_GatherSolution_Tpetra_impl.hpp | 6 ++---- .../Panzer_ScatterResidual_Tpetra_impl.hpp | 6 ++---- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/panzer/adapters-stk/example/PoissonInterfaceTpetra/Example_SimpleSource_impl.hpp b/packages/panzer/adapters-stk/example/PoissonInterfaceTpetra/Example_SimpleSource_impl.hpp index 7846cee3e15f..3a794ecc7956 100644 --- a/packages/panzer/adapters-stk/example/PoissonInterfaceTpetra/Example_SimpleSource_impl.hpp +++ b/packages/panzer/adapters-stk/example/PoissonInterfaceTpetra/Example_SimpleSource_impl.hpp @@ -82,19 +82,22 @@ template void SimpleSource::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < source.extent_int(1); ++point) { - const double & x = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,0); - const double & y = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,1); + auto ip_coordinates = workset.int_rules[ir_index]->ip_coordinates.get_static_view(); + auto source_v = source.get_static_view(); + + Kokkos::parallel_for ("SimpleSource", workset.num_cells, KOKKOS_LAMBDA (const index_t cell) { + for (int point = 0; point < source_v.extent_int(1); ++point) { + const double & x = ip_coordinates(cell,point,0); + const double & y = ip_coordinates(cell,point,1); - if (this->wda(workset).int_rules[ir_index]->ip_coordinates.extent(2) == 2) { - source(cell,point) = 4*M_PI*M_PI*sin(2*M_PI*x)*cos(2*M_PI*y); + if (ip_coordinates.extent(2) == 2) { + source_v(cell,point) = 4*M_PI*M_PI*sin(2*M_PI*x)*cos(2*M_PI*y); } else { - const double & z = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,2); - source(cell,point) = 4*M_PI*M_PI*sin(2*M_PI*x)*cos(2*M_PI*y)*cos(2*M_PI*z); + const double & z = ip_coordinates(cell,point,2); + source_v(cell,point) = 4*M_PI*M_PI*sin(2*M_PI*x)*cos(2*M_PI*y)*cos(2*M_PI*z); } } - } + }); } //********************************************************************** diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Tpetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Tpetra_impl.hpp index 01a15dcd17c0..ff4c56c0f5a4 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Tpetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Tpetra_impl.hpp @@ -138,8 +138,7 @@ postRegistrationSetup(typename TRAITS::SetupData d, int fieldNum = fieldIds_[fd]; const std::vector & offsets = globalIndexer_->getGIDFieldOffsets(blockId,fieldNum); scratch_offsets_[fd] = PHX::View("offsets",offsets.size()); - for(std::size_t i=0;i(offsets.data(), offsets.size())); } scratch_lids_ = PHX::View("lids",gatherFields_[0].extent(0), @@ -461,8 +460,7 @@ postRegistrationSetup(typename TRAITS::SetupData d, int fieldNum = fieldIds_[fd]; const std::vector & offsets = globalIndexer_->getGIDFieldOffsets(blockId,fieldNum); scratch_offsets_[fd] = PHX::View("offsets",offsets.size()); - for(std::size_t i=0;i(offsets.data(), offsets.size())); } scratch_lids_ = PHX::View("lids",gatherFields_[0].extent(0), diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_Tpetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_Tpetra_impl.hpp index cf26b6f34f42..b648c5514867 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_Tpetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterResidual_Tpetra_impl.hpp @@ -126,8 +126,7 @@ postRegistrationSetup(typename TRAITS::SetupData d, const std::vector & offsets = globalIndexer_->getGIDFieldOffsets(blockId,fieldIds_[fd]); scratch_offsets_[fd] = PHX::View("offsets",offsets.size()); - for(std::size_t i=0;i(offsets.data(), offsets.size())); } scratch_lids_ = PHX::View("lids",scatterFields_[0].extent(0), globalIndexer_->getElementBlockGIDCount(blockId)); @@ -336,8 +335,7 @@ postRegistrationSetup(typename TRAITS::SetupData d, int fieldNum = fieldIds_[fd]; const std::vector & offsets = globalIndexer_->getGIDFieldOffsets(blockId,fieldNum); scratch_offsets_[fd] = PHX::View("offsets",offsets.size()); - for(std::size_t i=0;i(offsets.data(), offsets.size())); } my_derivative_size_ = globalIndexer_->getElementBlockGIDCount(blockId); From e2b0065a34cde3ad6bbfaaeb9a8df5b97bd59c98 Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Mon, 6 Sep 2021 08:47:49 -0600 Subject: [PATCH 63/70] Have simple tpetra example working --- .../Example_Solution_impl.hpp | 31 +++++---- ...Panzer_ScatterDirichletResidual_Tpetra.cpp | 2 + ...r_ScatterDirichletResidual_Tpetra_impl.hpp | 69 ++++++++++--------- 3 files changed, 56 insertions(+), 46 deletions(-) diff --git a/packages/panzer/adapters-stk/example/PoissonInterfaceTpetra/Example_Solution_impl.hpp b/packages/panzer/adapters-stk/example/PoissonInterfaceTpetra/Example_Solution_impl.hpp index f0a9ae978a56..9aa975333b97 100644 --- a/packages/panzer/adapters-stk/example/PoissonInterfaceTpetra/Example_Solution_impl.hpp +++ b/packages/panzer/adapters-stk/example/PoissonInterfaceTpetra/Example_Solution_impl.hpp @@ -76,26 +76,31 @@ template void Solution::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < solution.extent_int(1); ++point) { - const double& x = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,0); - const double& y = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,1); + auto ip_coordinates = workset.int_rules[ir_index]->ip_coordinates.get_static_view(); + auto solution_v = solution.get_static_view(); + bool ws7 = (workset.block_id[7] == '0'); + auto l_linear_Robin = linear_Robin; - if (linear_Robin) { - if (this->wda(workset).int_rules[ir_index]->ip_coordinates.extent(2) == 2) { - solution(cell,point) = 0.5 - 0.8*x + 0.5*sin(2*M_PI*x)*cos(2*M_PI*y); + Kokkos::parallel_for ("SimpleSource", workset.num_cells, KOKKOS_LAMBDA (const index_t cell) { + for (int point = 0; point < solution_v.extent_int(1); ++point) { + const double& x = ip_coordinates(cell,point,0); + const double& y = ip_coordinates(cell,point,1); + + if (l_linear_Robin) { + if (ip_coordinates.extent(2) == 2) { + solution_v(cell,point) = 0.5 - 0.8*x + 0.5*sin(2*M_PI*x)*cos(2*M_PI*y); } else { - const double & z = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,2); - solution(cell,point) = 0.5 - 0.8*x + sin(2*M_PI*x)*cos(2*M_PI*y)*cos(2*M_PI*z)/3.0; + const double & z = ip_coordinates(cell,point,2); + solution_v(cell,point) = 0.5 - 0.8*x + sin(2*M_PI*x)*cos(2*M_PI*y)*cos(2*M_PI*z)/3.0; } } else { - if (workset.block_id[7] == '0') - solution(cell,point) = 0.5 - 0.4*x; + if (ws7) + solution_v(cell,point) = 0.5 - 0.4*x; else - solution(cell,point) = 0.1 - 0.4*x; + solution_v(cell,point) = 0.1 - 0.4*x; } } - } + }); } } diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Tpetra.cpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Tpetra.cpp index 66ee4c22d89e..125f49b54187 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Tpetra.cpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Tpetra.cpp @@ -40,6 +40,8 @@ // *********************************************************************** // @HEADER +#include "Kokkos_View_Fad.hpp" + #include "PanzerDiscFE_config.hpp" #include "Panzer_ExplicitTemplateInstantiation.hpp" diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Tpetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Tpetra_impl.hpp index d4befecc6ebc..6f6f07c9a1fb 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Tpetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Tpetra_impl.hpp @@ -600,37 +600,40 @@ evaluateFields(typename TRAITS::EvalData workset) // may be more expensive! // scatter operation for each cell in workset - for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementGIDs(cellLocalId,GIDs); - auto LIDs = globalIndexer_->getElementLIDs(cellLocalId); - - // loop over each field to be scattered - for(std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) { - int fieldNum = fieldIds_[fieldIndex]; + auto LIDs = globalIndexer_->getLIDs(); + auto LIDs_h = Kokkos::create_mirror_view(LIDs); + Kokkos::deep_copy(LIDs_h, LIDs); + // loop over each field to be scattered + for(std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) { + int fieldNum = fieldIds_[fieldIndex]; + auto scatterFields_h = Kokkos::create_mirror_view(scatterFields_[fieldIndex].get_static_view()); + Kokkos::deep_copy(scatterFields_h, scatterFields_[fieldIndex].get_static_view()); + for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementGIDs(cellLocalId,GIDs); - // this call "should" get the right ordering according to the Intrepid2 basis - const std::pair,std::vector > & indicePair - = globalIndexer_->getGIDFieldOffsets_closure(blockId,fieldNum, side_subcell_dim_, local_side_id_); - const std::vector & elmtOffset = indicePair.first; - const std::vector & basisIdMap = indicePair.second; - - // loop over basis functions - for(std::size_t basis=0;basis,std::vector > & indicePair + = globalIndexer_->getGIDFieldOffsets_closure(blockId,fieldNum, side_subcell_dim_, local_side_id_); + const std::vector & elmtOffset = indicePair.first; + const std::vector & basisIdMap = indicePair.second; + + // loop over basis functions + for(std::size_t basis=0;basisgetNumEntriesInLocalRow(lid); std::size_t numEntries = 0; typename LOC::CrsMatrixType::nonconst_local_inds_host_view_type rowIndices("indices", sz); @@ -646,7 +649,7 @@ evaluateFields(typename TRAITS::EvalData workset) } GO gid = GIDs[offset]; - const ScalarT scatterField = (scatterFields_[fieldIndex])(worksetCellIndex,basisId); + const ScalarT scatterField = scatterFields_h(worksetCellIndex,basisId); r_array[lid] = scatterField.val(); dc_array[lid] = 1.0; // mark row as dirichlet @@ -659,8 +662,8 @@ evaluateFields(typename TRAITS::EvalData workset) TEUCHOS_ASSERT(jacRow.size()==GIDs.size()); Jac->replaceGlobalValues(gid, GIDs, jacRow); - } - } + } + } } } From a77cde430909ab9f4be643d0e6691758c6dcf9fa Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Mon, 6 Sep 2021 09:57:53 -0600 Subject: [PATCH 64/70] another device access --- ...r_ScatterDirichletResidual_Tpetra_impl.hpp | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Tpetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Tpetra_impl.hpp index 6f6f07c9a1fb..e59373ef899b 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Tpetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_ScatterDirichletResidual_Tpetra_impl.hpp @@ -189,22 +189,25 @@ evaluateFields(typename TRAITS::EvalData workset) // may be more expensive! - // scatter operation for each cell in workset - for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementGIDs(cellLocalId,GIDs); + // loop over each field to be scattered + for(std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) { + int fieldNum = fieldIds_[fieldIndex]; + auto scatterFields_h = Kokkos::create_mirror_view(scatterFields_[fieldIndex].get_static_view()); + Kokkos::deep_copy(scatterFields_h, scatterFields_[fieldIndex].get_static_view()); - // caculate the local IDs for this element - LIDs.resize(GIDs.size()); - for(std::size_t i=0;igetMap()->getLocalElement(GIDs[i]); + // scatter operation for each cell in workset + for(std::size_t worksetCellIndex=0;worksetCellIndexgetElementGIDs(cellLocalId,GIDs); - // loop over each field to be scattered - for(std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) { - int fieldNum = fieldIds_[fieldIndex]; + // caculate the local IDs for this element + LIDs.resize(GIDs.size()); + for(std::size_t i=0;igetMap()->getLocalElement(GIDs[i]); - if (!scatterIC_) { + if (!scatterIC_) { // this call "should" get the right ordering according to the Intrepid2 basis const std::pair,std::vector > & indicePair = globalIndexer_->getGIDFieldOffsets_closure(blockId,fieldNum, side_subcell_dim_, local_side_id_); @@ -224,7 +227,7 @@ evaluateFields(typename TRAITS::EvalData workset) if (!applyBC_[fieldIndex](worksetCellIndex,basisId)) continue; - r_array[lid] = (scatterFields_[fieldIndex])(worksetCellIndex,basisId); + r_array[lid] = scatterFields_h(worksetCellIndex,basisId); // record that you set a dirichlet condition dc_array[lid] = 1.0; @@ -240,7 +243,7 @@ evaluateFields(typename TRAITS::EvalData workset) if(lid<0) // not on this processor! continue; - r_array[lid] = (scatterFields_[fieldIndex])(worksetCellIndex,basis); + r_array[lid] = scatterFields_h(worksetCellIndex,basis); // record that you set a dirichlet condition dc_array[lid] = 1.0; From 87efd899606af22ca4343c8fa5cf70cd525d9122 Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Mon, 6 Sep 2021 10:30:31 -0600 Subject: [PATCH 65/70] reset view ref to equalize count --- .../disc-fe/src/evaluators/Panzer_GatherSolution_Tpetra_impl.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Tpetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Tpetra_impl.hpp index ff4c56c0f5a4..0d61454e435f 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Tpetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_Tpetra_impl.hpp @@ -604,6 +604,7 @@ evaluateFields(typename TRAITS::EvalData workset) else Kokkos::parallel_for(Kokkos::RangePolicy(0,workset.num_cells),*this); } + functor_data.x_data = Kokkos::View(); } // ********************************************************************** From 1838653a8916bb4f68f45e96ea53b53f712ff533 Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Mon, 6 Sep 2021 11:48:27 -0600 Subject: [PATCH 66/70] Minor fixes in some of the source terms for examples --- .../Example_SimpleSource_impl.hpp | 21 +++++++++++-------- .../Example_SimpleSource_impl.hpp | 21 +++++++++++-------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/packages/panzer/adapters-stk/example/CurlLaplacianExample/Example_SimpleSource_impl.hpp b/packages/panzer/adapters-stk/example/CurlLaplacianExample/Example_SimpleSource_impl.hpp index d9766a64a2f4..457211359759 100644 --- a/packages/panzer/adapters-stk/example/CurlLaplacianExample/Example_SimpleSource_impl.hpp +++ b/packages/panzer/adapters-stk/example/CurlLaplacianExample/Example_SimpleSource_impl.hpp @@ -82,20 +82,23 @@ template void SimpleSource::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < source.extent_int(1); ++point) { + auto ip_coordinates = workset.int_rules[ir_index]->ip_coordinates.get_static_view(); + auto source_v = source.get_static_view(); - const double & x = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,0); - const double & y = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,1); + Kokkos::parallel_for ("SimpleSource", workset.num_cells, KOKKOS_LAMBDA (const index_t cell) { + for (int point = 0; point < source_v.extent_int(1); ++point) { - source(cell,point,0) = 2.0+y-y*y + cos(2.0*M_PI*x)*sin(2.0*M_PI*y); - source(cell,point,1) = 2.0+x-x*x + sin(2.0*M_PI*x)*cos(2.0*M_PI*y); + const double & x = ip_coordinates(cell,point,0); + const double & y = ip_coordinates(cell,point,1); + + source_v(cell,point,0) = 2.0+y-y*y + cos(2.0*M_PI*x)*sin(2.0*M_PI*y); + source_v(cell,point,1) = 2.0+x-x*x + sin(2.0*M_PI*x)*cos(2.0*M_PI*y); // if three d - if(source.extent(2)==3) - source(cell,point,2) = 0.0; + if(source_v.extent(2)==3) + source_v(cell,point,2) = 0.0; } - } + }); } //********************************************************************** diff --git a/packages/panzer/adapters-stk/example/MixedCurlLaplacianExample/Example_SimpleSource_impl.hpp b/packages/panzer/adapters-stk/example/MixedCurlLaplacianExample/Example_SimpleSource_impl.hpp index d9766a64a2f4..4328bcaa74cc 100644 --- a/packages/panzer/adapters-stk/example/MixedCurlLaplacianExample/Example_SimpleSource_impl.hpp +++ b/packages/panzer/adapters-stk/example/MixedCurlLaplacianExample/Example_SimpleSource_impl.hpp @@ -82,20 +82,23 @@ template void SimpleSource::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < source.extent_int(1); ++point) { + auto ip_coordinates = workset.int_rules[ir_index]->ip_coordinates.get_static_view(); + auto source_v = source.get_static_view(); + + Kokkos::parallel_for ("SimpleSource", workset.num_cells, KOKKOS_LAMBDA (const index_t cell) { + for (int point = 0; point < source_v.extent_int(1); ++point) { - const double & x = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,0); - const double & y = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,1); + const double & x = ip_coordinates(cell,point,0); + const double & y = ip_coordinates(cell,point,1); - source(cell,point,0) = 2.0+y-y*y + cos(2.0*M_PI*x)*sin(2.0*M_PI*y); - source(cell,point,1) = 2.0+x-x*x + sin(2.0*M_PI*x)*cos(2.0*M_PI*y); + source_v(cell,point,0) = 2.0+y-y*y + cos(2.0*M_PI*x)*sin(2.0*M_PI*y); + source_v(cell,point,1) = 2.0+x-x*x + sin(2.0*M_PI*x)*cos(2.0*M_PI*y); // if three d - if(source.extent(2)==3) - source(cell,point,2) = 0.0; + if(source_v.extent(2)==3) + source_v(cell,point,2) = 0.0; } - } + }); } //********************************************************************** From 5d2e32a47d14690843a3c53e38c16df5436a39a3 Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Tue, 7 Sep 2021 07:44:54 -0600 Subject: [PATCH 67/70] Resolved UVM issues --- .../Example_SineSolution_impl.hpp | 17 ++++++++++------- .../Panzer_STK_ScatterVectorFields_impl.hpp | 8 ++++++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/panzer/adapters-stk/example/MixedPoissonExample/Example_SineSolution_impl.hpp b/packages/panzer/adapters-stk/example/MixedPoissonExample/Example_SineSolution_impl.hpp index 683304edcd3e..5e362d86bd87 100644 --- a/packages/panzer/adapters-stk/example/MixedPoissonExample/Example_SineSolution_impl.hpp +++ b/packages/panzer/adapters-stk/example/MixedPoissonExample/Example_SineSolution_impl.hpp @@ -82,16 +82,19 @@ template void SineSolution::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < solution.extent_int(1); ++point) { + auto ip_coordinates = this->wda(workset).int_rules[ir_index]->ip_coordinates.get_static_view(); + auto solution_v = solution.get_static_view(); - const double & x = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,0); - const double & y = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,1); - const double & z = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,2); + Kokkos::parallel_for (workset.num_cells, KOKKOS_LAMBDA (const index_t cell) { + for (int point = 0; point < solution_v.extent_int(1); ++point) { - solution(cell,point) = std::sin(2.0*M_PI*x)*std::sin(2*M_PI*y)*std::sin(2.0*M_PI*z); + const double & x = ip_coordinates(cell,point,0); + const double & y = ip_coordinates(cell,point,1); + const double & z = ip_coordinates(cell,point,2); + + solution_v(cell,point) = std::sin(2.0*M_PI*x)*std::sin(2*M_PI*y)*std::sin(2.0*M_PI*z); } - } + }); } //********************************************************************** diff --git a/packages/panzer/adapters-stk/src/evaluators/Panzer_STK_ScatterVectorFields_impl.hpp b/packages/panzer/adapters-stk/src/evaluators/Panzer_STK_ScatterVectorFields_impl.hpp index 1f3622ac8a8b..be6b88adc1a3 100644 --- a/packages/panzer/adapters-stk/src/evaluators/Panzer_STK_ScatterVectorFields_impl.hpp +++ b/packages/panzer/adapters-stk/src/evaluators/Panzer_STK_ScatterVectorFields_impl.hpp @@ -137,8 +137,12 @@ evaluateFields(panzer::Traits::EvalData workset) // scaline field value only if the scaling parameter is specified, otherwise use 1.0 double scaling = (scaling_.size()>0) ? scaling_[fieldIndex] : 1.0; - for(unsigned i=0; isetCellFieldData(fieldName,blockId,localCellIds,cellValue.get_view(),scaling); From 040a3a47ac55f2323976367f6ae5e3005e0e8237 Mon Sep 17 00:00:00 2001 From: Matt Bettencourt Date: Tue, 7 Sep 2021 09:03:58 -0600 Subject: [PATCH 68/70] Fixed Dirichlet resid for edge basis and UVM --- .../Example_CurlSolution_impl.hpp | 20 ++++++++------ .../Example_CurlSolution_impl.hpp | 20 ++++++++------ .../Panzer_Dirichlet_Residual_EdgeBasis.cpp | 2 ++ .../Panzer_Dirichlet_Residual_EdgeBasis.hpp | 4 +-- ...nzer_Dirichlet_Residual_EdgeBasis_impl.hpp | 27 ++++++++++++------- 5 files changed, 45 insertions(+), 28 deletions(-) diff --git a/packages/panzer/adapters-stk/example/CurlLaplacianExample/Example_CurlSolution_impl.hpp b/packages/panzer/adapters-stk/example/CurlLaplacianExample/Example_CurlSolution_impl.hpp index 6bbea230436d..170528cacdd5 100644 --- a/packages/panzer/adapters-stk/example/CurlLaplacianExample/Example_CurlSolution_impl.hpp +++ b/packages/panzer/adapters-stk/example/CurlLaplacianExample/Example_CurlSolution_impl.hpp @@ -88,18 +88,22 @@ template void CurlSolution::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < solution.extent_int(1); ++point) { + auto ip_coordinates = this->wda(workset).int_rules[ir_index]->ip_coordinates.get_static_view(); + auto solution_v = solution.get_static_view(); + auto solution_curl_v = solution_curl.get_static_view(); - const double & x = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,0); - const double & y = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,1); + Kokkos::parallel_for (workset.num_cells, KOKKOS_LAMBDA (const index_t cell) { + for (int point = 0; point < solution_v.extent_int(1); ++point) { - solution(cell,point,0) = -(y-1.0)*y + cos(2.0*M_PI*x)*sin(2.0*M_PI*y); - solution(cell,point,1) = -(x-1.0)*x + sin(2.0*M_PI*x)*cos(2.0*M_PI*y); + const double & x = ip_coordinates(cell,point,0); + const double & y = ip_coordinates(cell,point,1); - solution_curl(cell,point) = -2.0*x+2.0*y; + solution_v(cell,point,0) = -(y-1.0)*y + cos(2.0*M_PI*x)*sin(2.0*M_PI*y); + solution_v(cell,point,1) = -(x-1.0)*x + sin(2.0*M_PI*x)*cos(2.0*M_PI*y); + + solution_curl_v(cell,point) = -2.0*x+2.0*y; } - } + }); } //********************************************************************** diff --git a/packages/panzer/adapters-stk/example/MixedCurlLaplacianExample/Example_CurlSolution_impl.hpp b/packages/panzer/adapters-stk/example/MixedCurlLaplacianExample/Example_CurlSolution_impl.hpp index 6bbea230436d..170528cacdd5 100644 --- a/packages/panzer/adapters-stk/example/MixedCurlLaplacianExample/Example_CurlSolution_impl.hpp +++ b/packages/panzer/adapters-stk/example/MixedCurlLaplacianExample/Example_CurlSolution_impl.hpp @@ -88,18 +88,22 @@ template void CurlSolution::evaluateFields(typename Traits::EvalData workset) { using panzer::index_t; - for (index_t cell = 0; cell < workset.num_cells; ++cell) { - for (int point = 0; point < solution.extent_int(1); ++point) { + auto ip_coordinates = this->wda(workset).int_rules[ir_index]->ip_coordinates.get_static_view(); + auto solution_v = solution.get_static_view(); + auto solution_curl_v = solution_curl.get_static_view(); - const double & x = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,0); - const double & y = this->wda(workset).int_rules[ir_index]->ip_coordinates(cell,point,1); + Kokkos::parallel_for (workset.num_cells, KOKKOS_LAMBDA (const index_t cell) { + for (int point = 0; point < solution_v.extent_int(1); ++point) { - solution(cell,point,0) = -(y-1.0)*y + cos(2.0*M_PI*x)*sin(2.0*M_PI*y); - solution(cell,point,1) = -(x-1.0)*x + sin(2.0*M_PI*x)*cos(2.0*M_PI*y); + const double & x = ip_coordinates(cell,point,0); + const double & y = ip_coordinates(cell,point,1); - solution_curl(cell,point) = -2.0*x+2.0*y; + solution_v(cell,point,0) = -(y-1.0)*y + cos(2.0*M_PI*x)*sin(2.0*M_PI*y); + solution_v(cell,point,1) = -(x-1.0)*x + sin(2.0*M_PI*x)*cos(2.0*M_PI*y); + + solution_curl_v(cell,point) = -2.0*x+2.0*y; } - } + }); } //********************************************************************** diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_Dirichlet_Residual_EdgeBasis.cpp b/packages/panzer/disc-fe/src/evaluators/Panzer_Dirichlet_Residual_EdgeBasis.cpp index be8ce12a4fb0..c0d66e884c16 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_Dirichlet_Residual_EdgeBasis.cpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_Dirichlet_Residual_EdgeBasis.cpp @@ -40,6 +40,8 @@ // *********************************************************************** // @HEADER +#include "Kokkos_View_Fad.hpp" + #include "PanzerDiscFE_config.hpp" #include "Panzer_ExplicitTemplateInstantiation.hpp" diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_Dirichlet_Residual_EdgeBasis.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_Dirichlet_Residual_EdgeBasis.hpp index 546be53a038b..1933f4023661 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_Dirichlet_Residual_EdgeBasis.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_Dirichlet_Residual_EdgeBasis.hpp @@ -98,8 +98,8 @@ class DirichletResidual_EdgeBasis PointValues2 pointValues; Teuchos::RCP > orientations; - Intrepid2::RefSubcellParametrization::ConstViewType edgeParam; //edge parametrization - Intrepid2::RefSubcellParametrization::ConstViewType faceParam; //face parametrization + Intrepid2::RefSubcellParametrization::ConstViewType edgeParam; //edge parametrization + Intrepid2::RefSubcellParametrization::ConstViewType faceParam; //face parametrization }; // end of class DirichletResidual_EdgeBasis diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_Dirichlet_Residual_EdgeBasis_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_Dirichlet_Residual_EdgeBasis_impl.hpp index a43bc534ce3e..d08826f0aea3 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_Dirichlet_Residual_EdgeBasis_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_Dirichlet_Residual_EdgeBasis_impl.hpp @@ -122,10 +122,10 @@ postRegistrationSetup( const int edgeDim = 1; const int faceDim = 2; if(cellTopo.getDimension() > edgeDim) - edgeParam = Intrepid2::RefSubcellParametrization::get(edgeDim, cellTopo.getKey()); + edgeParam = Intrepid2::RefSubcellParametrization::get(edgeDim, cellTopo.getKey()); if(cellTopo.getDimension() > faceDim) - faceParam = Intrepid2::RefSubcellParametrization::get(faceDim, cellTopo.getKey()); + faceParam = Intrepid2::RefSubcellParametrization::get(faceDim, cellTopo.getKey()); } //********************************************************************** @@ -157,9 +157,14 @@ evaluateFields( const WorksetDetails & details = workset; //const bool is_normalize = true; - auto work = Kokkos::createDynRankView(residual.get_static_view(),"work", 4, cellDim); + auto work = Kokkos::create_mirror_view(Kokkos::createDynRankView(residual.get_static_view(),"work", 4, cellDim)); // compute residual + auto residual_h = Kokkos::create_mirror_view(residual.get_static_view()); + auto dof_h = Kokkos::create_mirror_view(dof.get_static_view()); + auto value_h = Kokkos::create_mirror_view(value.get_static_view()); + Kokkos::deep_copy(dof_h, dof.get_static_view()); + Kokkos::deep_copy(value_h, value.get_static_view()); switch (subcellDim) { case 1: { // 2D element Tri and Quad if (intrepid_basis->getDofCount(1, subcellOrd)) { @@ -169,6 +174,7 @@ evaluateFields( const int ndofsEdge = intrepid_basis->getDofCount(1, subcellOrd); const int numEdges = cellTopo.getEdgeCount(); /* */ int edgeOrts[4] = {}; + for(index_t c=0;cat(details.cell_local_ids[c]).getEdgeOrientation(edgeOrts, numEdges); @@ -180,11 +186,11 @@ evaluateFields( for (int i=0;igetDofOrdinal(1, subcellOrd, i); - auto J = Kokkos::subview(worksetJacobians, c, b, Kokkos::ALL(), Kokkos::ALL()); + auto J = Kokkos::create_mirror_view(Kokkos::subview(worksetJacobians, c, b, Kokkos::ALL(), Kokkos::ALL())); Intrepid2::Kernels::Serial::matvec_product(phyEdgeTan, J, ortEdgeTan); for(int d=0;d Date: Tue, 7 Sep 2021 10:08:08 -0600 Subject: [PATCH 69/70] Fixed uvm error with building the sideset coords --- .../disc-fe/src/Panzer_Workset_Builder_impl.hpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/panzer/disc-fe/src/Panzer_Workset_Builder_impl.hpp b/packages/panzer/disc-fe/src/Panzer_Workset_Builder_impl.hpp index 01f789486bc8..b4a27e610bdf 100644 --- a/packages/panzer/disc-fe/src/Panzer_Workset_Builder_impl.hpp +++ b/packages/panzer/disc-fe/src/Panzer_Workset_Builder_impl.hpp @@ -419,14 +419,22 @@ panzer::buildBCWorkset(const WorksetNeeds & needs_a, impl::subset(local_side_ids_b, idxs, lsi_b); auto vc_a = mdArrayFactory.buildStaticArray("vc_a", idxs.size(), d1, d2); auto vc_b = mdArrayFactory.buildStaticArray("vc_b", idxs.size(), d1, d2); + auto vc_a_h = Kokkos::create_mirror_view(vc_a.get_static_view()); + auto vc_b_h = Kokkos::create_mirror_view(vc_b.get_static_view()); + auto vertex_coordinates_a_h = Kokkos::create_mirror_view(PHX::as_view(vertex_coordinates_a)); + auto vertex_coordinates_b_h = Kokkos::create_mirror_view(PHX::as_view(vertex_coordinates_b)); + Kokkos::deep_copy(vertex_coordinates_a_h, PHX::as_view(vertex_coordinates_a)); + Kokkos::deep_copy(vertex_coordinates_b_h, PHX::as_view(vertex_coordinates_b)); for (std::size_t i = 0; i < idxs.size(); ++i) { const auto ii = idxs[i]; for (int j = 0; j < d1; ++j) for (int k = 0; k < d2; ++k) { - vc_a(i, j, k) = vertex_coordinates_a(ii, j, k); - vc_b(i, j, k) = vertex_coordinates_b(ii, j, k); + vc_a_h(i, j, k) = vertex_coordinates_a_h(ii, j, k); + vc_b_h(i, j, k) = vertex_coordinates_b_h(ii, j, k); } } + Kokkos::deep_copy(vc_a.get_static_view(), vc_a_h); + Kokkos::deep_copy(vc_b.get_static_view(), vc_b_h); auto mwa_it = impl::buildBCWorksetForUniqueSideId(needs_a,blockid_a, lci_a, lsi_a, vc_a, needs_b,blockid_b, lci_b, lsi_b, vc_b, needs_b); From cece41b4e803dcafc8da09c094b9199ad872af0f Mon Sep 17 00:00:00 2001 From: K Devine Date: Tue, 7 Sep 2021 14:49:23 -0600 Subject: [PATCH 70/70] Tpetra: added methods to replace Domain and Range maps in CrsMatrix and CrsGraph (#9626) * tpetra: added method replaceDomainMap for #9451; fixed doxygen comments for replaceDomainMapAndImport to match the error checks in code * tpetra: put re-used code into a macro * tpetra: new functions and test: replaceRangeMap, replaceRangeMapAndExporter * tpetra: Minor * tpetra: revisions to address @tasmit's review of #9451 --- .../tpetra/core/src/Tpetra_CrsGraph_decl.hpp | 36 ++ .../tpetra/core/src/Tpetra_CrsGraph_def.hpp | 85 ++++- .../tpetra/core/src/Tpetra_CrsMatrix_decl.hpp | 47 ++- .../tpetra/core/src/Tpetra_CrsMatrix_def.hpp | 46 +++ .../core/src/Tpetra_MultiVector_def.hpp | 3 +- .../tpetra/core/test/CrsMatrix/CMakeLists.txt | 11 + .../CrsMatrix_ReplaceDomainMapAndImporter.cpp | 223 +++++++++--- .../CrsMatrix_ReplaceRangeMapAndExporter.cpp | 321 ++++++++++++++++++ 8 files changed, 713 insertions(+), 59 deletions(-) create mode 100644 packages/tpetra/core/test/CrsMatrix/CrsMatrix_ReplaceRangeMapAndExporter.cpp diff --git a/packages/tpetra/core/src/Tpetra_CrsGraph_decl.hpp b/packages/tpetra/core/src/Tpetra_CrsGraph_decl.hpp index ed4f882a85ab..989e4a55c041 100644 --- a/packages/tpetra/core/src/Tpetra_CrsGraph_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_CrsGraph_decl.hpp @@ -647,6 +647,7 @@ namespace Tpetra { /// parameters from \c params sublist "CrsGraph". The domain map /// and range maps passed to fillComplete() are those of the map /// being cloned, if they exist. Otherwise, the row map is used. + /// \brief True if and only if \c CrsGraph is identical to this CrsGraph /// /// \warning THIS METHOD IS FOR TPETRA DEVELOPERS ONLY. DO NOT @@ -1513,6 +1514,15 @@ namespace Tpetra { const Teuchos::RCP& newImport = Teuchos::null, const bool sortIndicesInEachRow = true); + /// \brief Replace the current domain Map with the given objects. + /// + /// The Graph's Import object will be recomputed if needed. + /// + /// \pre isFillComplete() == true + /// \pre isFillActive() == false + void + replaceDomainMap (const Teuchos::RCP& newDomainMap); + /// \brief Replace the current domain Map and Import with the /// given parameters. /// @@ -1530,6 +1540,32 @@ namespace Tpetra { replaceDomainMapAndImporter (const Teuchos::RCP& newDomainMap, const Teuchos::RCP& newImporter); + /// \brief Replace the current Range Map with the given objects. + /// + /// The Graph's Export object will be recomputed if needed. + /// + /// \pre isFillComplete() == true + /// \pre isFillActive() == false + void + replaceRangeMap (const Teuchos::RCP& newRangeMap); + + /// \brief Replace the current Range Map and Export with the + /// given parameters. + /// + /// \warning This method is ONLY for use by experts. + /// \warning We make NO promises of backwards compatibility. + /// This method may change or disappear at any time. + /// + /// \pre isFillComplete() == true + /// \pre isFillActive() == false + /// \pre Either the given Export object is null, or the target Map + /// of the given Export is the same as this graph's Range Map. + /// \pre Either the given Export object is null, or the source Map + /// of the given Export is the same as this graph's Row Map. + void + replaceRangeMapAndExporter (const Teuchos::RCP& newRangeMap, + const Teuchos::RCP& newExporter); + /// \brief Remove processes owning zero rows from the Maps and /// their communicator. /// diff --git a/packages/tpetra/core/src/Tpetra_CrsGraph_def.hpp b/packages/tpetra/core/src/Tpetra_CrsGraph_def.hpp index 8e609554dde0..ee33e552a3e4 100644 --- a/packages/tpetra/core/src/Tpetra_CrsGraph_def.hpp +++ b/packages/tpetra/core/src/Tpetra_CrsGraph_def.hpp @@ -4326,6 +4326,26 @@ namespace Tpetra { } } + template + void + CrsGraph:: + replaceDomainMap (const Teuchos::RCP& newDomainMap) + { + const char prefix[] = "Tpetra::CrsGraph::replaceDomainMap: "; + TEUCHOS_TEST_FOR_EXCEPTION( + colMap_.is_null (), std::invalid_argument, prefix << "You may not call " + "this method unless the graph already has a column Map."); + TEUCHOS_TEST_FOR_EXCEPTION( + newDomainMap.is_null (), std::invalid_argument, + prefix << "The new domain Map must be nonnull."); + + // Create a new importer, if needed + Teuchos::RCP newImporter = Teuchos::null; + if (newDomainMap != colMap_ && (! newDomainMap->isSameAs (*colMap_))) { + newImporter = rcp(new import_type(newDomainMap, colMap_)); + } + this->replaceDomainMapAndImporter(newDomainMap, newImporter); + } template void @@ -4349,7 +4369,7 @@ namespace Tpetra { // than once. It's polite for them to do so, but not required. const bool colSameAsDom = colMap_->isSameAs (*newDomainMap); TEUCHOS_TEST_FOR_EXCEPTION - (colSameAsDom, std::invalid_argument, "If the new Import is null, " + (!colSameAsDom, std::invalid_argument, "If the new Import is null, " "then the new domain Map must be the same as the current column Map."); } else { @@ -4369,6 +4389,69 @@ namespace Tpetra { importer_ = Teuchos::rcp_const_cast (newImporter); } + template + void + CrsGraph:: + replaceRangeMap (const Teuchos::RCP& newRangeMap) + { + const char prefix[] = "Tpetra::CrsGraph::replaceRangeMap: "; + TEUCHOS_TEST_FOR_EXCEPTION( + rowMap_.is_null (), std::invalid_argument, prefix << "You may not call " + "this method unless the graph already has a row Map."); + TEUCHOS_TEST_FOR_EXCEPTION( + newRangeMap.is_null (), std::invalid_argument, + prefix << "The new range Map must be nonnull."); + + // Create a new exporter, if needed + Teuchos::RCP newExporter = Teuchos::null; + if (newRangeMap != rowMap_ && (! newRangeMap->isSameAs (*rowMap_))) { + newExporter = rcp(new export_type(rowMap_, newRangeMap)); + } + this->replaceRangeMapAndExporter(newRangeMap, newExporter); + } + + template + void + CrsGraph:: + replaceRangeMapAndExporter (const Teuchos::RCP& newRangeMap, + const Teuchos::RCP& newExporter) + { + const char prefix[] = "Tpetra::CrsGraph::replaceRangeMapAndExporter: "; + TEUCHOS_TEST_FOR_EXCEPTION( + rowMap_.is_null (), std::invalid_argument, prefix << "You may not call " + "this method unless the graph already has a column Map."); + TEUCHOS_TEST_FOR_EXCEPTION( + newRangeMap.is_null (), std::invalid_argument, + prefix << "The new domain Map must be nonnull."); + + if (debug_) { + if (newExporter.is_null ()) { + // It's not a good idea to put expensive operations in a macro + // clause, even if they are side effect - free, because macros + // don't promise that they won't evaluate their arguments more + // than once. It's polite for them to do so, but not required. + const bool rowSameAsRange = rowMap_->isSameAs (*newRangeMap); + TEUCHOS_TEST_FOR_EXCEPTION + (!rowSameAsRange, std::invalid_argument, "If the new Export is null, " + "then the new range Map must be the same as the current row Map."); + } + else { + const bool newRangeSameAsTgt = + newRangeMap->isSameAs (* (newExporter->getTargetMap ())); + const bool rowSameAsSrc = + rowMap_->isSameAs (* (newExporter->getSourceMap ())); + TEUCHOS_TEST_FOR_EXCEPTION + (! rowSameAsSrc || ! newRangeSameAsTgt, std::invalid_argument, "If the " + "new Export is nonnull, then the current row Map must be the same " + "as the new Export's source Map, and the new range Map must be the " + "same as the new Export's target Map."); + } + } + + rangeMap_ = newRangeMap; + exporter_ = Teuchos::rcp_const_cast (newExporter); + } + #ifdef TPETRA_ENABLE_DEPRECATED_CODE template typename CrsGraph::local_graph_device_type diff --git a/packages/tpetra/core/src/Tpetra_CrsMatrix_decl.hpp b/packages/tpetra/core/src/Tpetra_CrsMatrix_decl.hpp index 555e10b451d9..399055b2e47f 100644 --- a/packages/tpetra/core/src/Tpetra_CrsMatrix_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_CrsMatrix_decl.hpp @@ -52,6 +52,7 @@ #include "Tpetra_Vector.hpp" #include "Tpetra_Details_PackTraits.hpp" // unused here, could delete #include "KokkosSparse_CrsMatrix.hpp" +#include "Teuchos_DataAccess.hpp" #include // std::shared_ptr @@ -2150,11 +2151,22 @@ namespace Tpetra { const Teuchos::RCP& newImport = Teuchos::null, const bool sortEachRow = true); + /// \brief Replace the current domain Map with the given objects. + /// + /// The matrix's Import object will be recomputed if needed. + /// + /// \param newDomainMap [in] New domain Map. Must be nonnull. + /// + /// \pre The matrix must be fill complete: + /// isFillComplete() == true. + /// + void + replaceDomainMap (const Teuchos::RCP& newDomainMap); + /// \brief Replace the current domain Map and Import with the given objects. /// /// \param newDomainMap [in] New domain Map. Must be nonnull. - /// \param newImporter [in] Optional Import object. If null, we - /// will compute it. + /// \param newImporter [in] Optional Import object. If null, the new Domain Map must equal the matrix's Column Map /// /// \pre The matrix must be fill complete: /// isFillComplete() == true. @@ -2162,10 +2174,41 @@ namespace Tpetra { /// same as the column Map of the matrix. /// \pre If the Import is provided, its source Map must be the /// same as the provided new domain Map. + /// \pre If the Import is not provided, the new Domain Map must be the + /// same as the matrix's Column Map. void replaceDomainMapAndImporter (const Teuchos::RCP& newDomainMap, Teuchos::RCP& newImporter); + /// \brief Replace the current range Map with the given objects. + /// + /// The matrix's Export object will be recomputed if needed. + /// + /// \param newRangeMap [in] New Range Map. Must be nonnull. + /// + /// \pre The matrix must be fill complete: + /// isFillComplete() == true. + /// + void + replaceRangeMap (const Teuchos::RCP& newRangeMap); + + /// \brief Replace the current Range Map and Export with the given objects. + /// + /// \param newRangeMap [in] New domain Map. Must be nonnull. + /// \param newExporter [in] Optional Export object. If null, the new Range Map must equal the matrix's Row Map + /// + /// \pre The matrix must be fill complete: + /// isFillComplete() == true. + /// \pre If the Export is provided, its target Map must be the + /// same as the new Range Map of the matrix. + /// \pre If the Export is provided, its source Map must be the + /// same as the Row Map of the matrix + /// \pre If the Export is not provided, the new Range Map must be the + /// same as the matrix's Row Map. + void + replaceRangeMapAndExporter (const Teuchos::RCP& newRangeMap, + Teuchos::RCP& newExporter); + /// \brief Remove processes owning zero rows from the Maps and their communicator. /// /// \warning This method is ONLY for use by experts. We highly diff --git a/packages/tpetra/core/src/Tpetra_CrsMatrix_def.hpp b/packages/tpetra/core/src/Tpetra_CrsMatrix_def.hpp index 0147280f6659..36f2864980a7 100644 --- a/packages/tpetra/core/src/Tpetra_CrsMatrix_def.hpp +++ b/packages/tpetra/core/src/Tpetra_CrsMatrix_def.hpp @@ -4389,6 +4389,21 @@ CrsMatrix:: } } + template + void + CrsMatrix:: + replaceDomainMap (const Teuchos::RCP& newDomainMap) + { + const char tfecfFuncName[] = "replaceDomainMap: "; + TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC( + myGraph_.is_null (), std::runtime_error, + "This method does not work if the matrix has a const graph. The whole " + "idea of a const graph is that you are not allowed to change it, but this" + " method necessarily must modify the graph, since the graph owns the " + "matrix's domain Map and Import objects."); + myGraph_->replaceDomainMap (newDomainMap); + } + template void CrsMatrix:: @@ -4405,6 +4420,37 @@ CrsMatrix:: myGraph_->replaceDomainMapAndImporter (newDomainMap, newImporter); } + template + void + CrsMatrix:: + replaceRangeMap (const Teuchos::RCP& newRangeMap) + { + const char tfecfFuncName[] = "replaceRangeMap: "; + TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC( + myGraph_.is_null (), std::runtime_error, + "This method does not work if the matrix has a const graph. The whole " + "idea of a const graph is that you are not allowed to change it, but this" + " method necessarily must modify the graph, since the graph owns the " + "matrix's domain Map and Import objects."); + myGraph_->replaceRangeMap (newRangeMap); + } + + template + void + CrsMatrix:: + replaceRangeMapAndExporter (const Teuchos::RCP& newRangeMap, + Teuchos::RCP& newExporter) + { + const char tfecfFuncName[] = "replaceRangeMapAndExporter: "; + TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC( + myGraph_.is_null (), std::runtime_error, + "This method does not work if the matrix has a const graph. The whole " + "idea of a const graph is that you are not allowed to change it, but this" + " method necessarily must modify the graph, since the graph owns the " + "matrix's domain Map and Import objects."); + myGraph_->replaceRangeMapAndExporter (newRangeMap, newExporter); + } + template void CrsMatrix:: diff --git a/packages/tpetra/core/src/Tpetra_MultiVector_def.hpp b/packages/tpetra/core/src/Tpetra_MultiVector_def.hpp index aaed39769b78..2c8e946b8db6 100644 --- a/packages/tpetra/core/src/Tpetra_MultiVector_def.hpp +++ b/packages/tpetra/core/src/Tpetra_MultiVector_def.hpp @@ -4160,6 +4160,7 @@ namespace Tpetra { const LO C_lclNumRows = C_tmp->getLocalLength (); const LO C_numVecs = C_tmp->getNumVectors (); + auto C_lcl = C_tmp->getLocalViewDevice(Access::ReadWrite); auto C_sub = Kokkos::subview (C_lcl, std::make_pair (LO (0), C_lclNumRows), @@ -4172,8 +4173,6 @@ namespace Tpetra { ProfilingRegion regionGemm ("Tpetra::MV::multiply-call-gemm"); - //this->modify_device (); - KokkosBlas::gemm (&ctransA, &ctransB, alpha_IST, A_sub, B_sub, beta_local, C_sub); } diff --git a/packages/tpetra/core/test/CrsMatrix/CMakeLists.txt b/packages/tpetra/core/test/CrsMatrix/CMakeLists.txt index e26a1d7b22ec..343be34dd3ca 100644 --- a/packages/tpetra/core/test/CrsMatrix/CMakeLists.txt +++ b/packages/tpetra/core/test/CrsMatrix/CMakeLists.txt @@ -176,6 +176,17 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST( STANDARD_PASS_OUTPUT ) +TRIBITS_ADD_EXECUTABLE_AND_TEST( + CrsMatrix_ReplaceRangeMapAndExporter + SOURCES + CrsMatrix_ReplaceRangeMapAndExporter + ${TEUCHOS_STD_UNIT_TEST_MAIN} + ARGS "" + COMM serial mpi + STANDARD_PASS_OUTPUT + ) + + TRIBITS_COPY_FILES_TO_BINARY_DIR(CrsMatrixCopyFiles1 SOURCE_FILES west0067.rua mhd1280b.cua EXEDEPS CrsMatrix_UnitTests diff --git a/packages/tpetra/core/test/CrsMatrix/CrsMatrix_ReplaceDomainMapAndImporter.cpp b/packages/tpetra/core/test/CrsMatrix/CrsMatrix_ReplaceDomainMapAndImporter.cpp index a7079b50fe6b..735e41e50c08 100644 --- a/packages/tpetra/core/test/CrsMatrix/CrsMatrix_ReplaceDomainMapAndImporter.cpp +++ b/packages/tpetra/core/test/CrsMatrix/CrsMatrix_ReplaceDomainMapAndImporter.cpp @@ -51,7 +51,6 @@ namespace { using std::string; using Teuchos::as; - using Teuchos::FancyOStream; using Teuchos::RCP; using Teuchos::ArrayRCP; using Teuchos::rcp; @@ -99,15 +98,53 @@ namespace { // UNIT TESTS // - //// - TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL( CrsMatrix, ReplaceDomainMapAndImporter, LO, GO, Scalar, Node ) + template + void applyAndCheckResult( + Tpetra::CrsMatrix &A, + Tpetra::CrsMatrix &B, + Teuchos::RCP > &newMap, + Teuchos::FancyOStream &out, + bool &success + ) + { + typedef Teuchos::ScalarTraits STS; + typedef typename STS::magnitudeType MT; + MT norm = ScalarTraits::zero (); + + /* Fill a random vector on the original map */ + Vector AVecX(A.getDomainMap()); + AVecX.randomize(); + + /* Import this vector to the new domainmap */ + Vector BVecX(B.getDomainMap()); + Tpetra::Import TempImport(A.getDomainMap(), (newMap)); + BVecX.doImport(AVecX,TempImport,Tpetra::INSERT); + + /* Now do some multiplies */ + Vector AVecY(A.getRangeMap()); + Vector BVecY(B.getRangeMap()); + A.apply(AVecX,AVecY); + B.apply(BVecX,BVecY); + + BVecY.update (-STS::one (), AVecY, STS::one ()); + norm = BVecY.norm2(); + + std::cout << "Residual 2-norm: " << norm << std::endl + << "Residual 1-norm: " << BVecY.norm1 () << std::endl + << "Residual Inf-norm: " << BVecY.normInf () << std::endl; + + const bool normSmallEnough = (norm <= as (1e-10)); + TEST_EQUALITY ( normSmallEnough, true ); + } + + ////////////////////////////////////////////////////////////////////////// + + TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL( CrsMatrix, ReplaceDomainMap, LO, GO, Scalar, Node ) { // Based on the FullTriDiag tests... typedef CrsMatrix MAT; typedef ScalarTraits STS; - typedef typename STS::magnitudeType MT; - typedef ScalarTraits STM; const size_t ONE = OrdinalTraits::one(); const size_t ZERO = OrdinalTraits::zero(); @@ -118,9 +155,8 @@ namespace { const size_t myImageID = comm->getRank(); if (numImages < 3) return; // create a Map - RCP > map = createContigMapWithNode(INVALID,ONE,comm); - - // RCP fos = Teuchos::fancyOStream(rcp(&std::cout,false)); + RCP > map = + createContigMapWithNode(INVALID,ONE,comm); /* Create the following matrix: 0 [2 1 ] [2 1] @@ -134,9 +170,11 @@ namespace { MAT A(map,4); MAT B(map,4); A.setObjectLabel("The Matrix"); - A.setObjectLabel("The Other Matrix"); + B.setObjectLabel("The Other Matrix"); if (myImageID != numImages-1) { // last image assigns none - Array vals(tuple(static_cast(2)*STS::one(),STS::one(),static_cast(2)*STS::one())); + Array vals(tuple(static_cast(2)*STS::one(), + STS::one(), + static_cast(2)*STS::one())); Array cols(tuple(myImageID,myImageID + 1)); A.insertGlobalValues(myImageID ,cols(),vals(0,2)); // insert [2 1] A.insertGlobalValues(myImageID+1,cols(),vals(1,2)); // insert [1 2] @@ -146,59 +184,136 @@ namespace { A.fillComplete(); B.fillComplete(); + // Build a one-process Map. - if (comm->getSize() > 1) { - MT norm = STM::zero (); - - // we know the map is contiguous... - const size_t NumMyElements = (comm->getRank () == 0) ? - A.getDomainMap ()->getGlobalNumElements () : 0; - RCP > NewMap = - rcp (new Map (INVALID, NumMyElements, ZERO, comm)); - RCP > NewImport = - rcp (new Import (NewMap, A.getColMap ())); - - B.replaceDomainMapAndImporter (NewMap, NewImport); - - // Fill a random vector on the original map - Vector AVecX(A.getDomainMap()); - AVecX.randomize(); - - // Import this vector to the new domainmap - Vector BVecX(B.getDomainMap()); - Tpetra::Import TempImport(A.getDomainMap(),NewMap); // (source,target) - BVecX.doImport(AVecX,TempImport,Tpetra::ADD); - - // Now do some multiplies - Vector AVecY(A.getRangeMap()); - Vector BVecY(B.getRangeMap()); - A.apply(AVecX,AVecY); - B.apply(BVecX,BVecY); - - BVecY.update (-STS::one (), AVecY, STS::one ()); - norm = BVecY.norm2(); - - out << "Residual 2-norm: " << norm << endl - << "Residual 1-norm: " << BVecY.norm1 () << endl - << "Residual Inf-norm: " << BVecY.normInf () << endl; - - // Macros don't like spaces, so we put the test outside the - // macro. Use <= rather than <, so the test passes even if - // Scalar is an integer type. - // - // FIXME (mfh 10 Mar 2013) We should pick the tolerance relative - // to the Scalar type and problem size. - const bool normSmallEnough = norm <= as (1e-10); - TEST_EQUALITY ( normSmallEnough, true ); + // we know the map is contiguous... + const size_t NumMyElements = (comm->getRank () == 0) ? + A.getDomainMap ()->getGlobalNumElements () : 0; + RCP > NewMap = + rcp (new Map (INVALID, NumMyElements, ZERO, comm)); + + B.replaceDomainMap (NewMap); + + applyAndCheckResult(A, B, NewMap, out, success); + } + + ////////////////////////////////////////////////////////////////////////// + + TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL( CrsMatrix, ReplaceDomainMapAndImporter, LO, GO, Scalar, Node ) + { + // Based on the FullTriDiag tests... + + typedef CrsMatrix MAT; + typedef ScalarTraits STS; + + const size_t ONE = OrdinalTraits::one(); + const size_t ZERO = OrdinalTraits::zero(); + const global_size_t INVALID = OrdinalTraits::invalid(); + // get a comm + RCP > comm = Tpetra::getDefaultComm(); + const size_t numImages = comm->getSize(); + const size_t myImageID = comm->getRank(); + if (numImages < 3) return; + // create a Map + RCP > map = + createContigMapWithNode(INVALID,ONE,comm); + + /* Create the following matrix: + 0 [2 1 ] [2 1] + 1 [1 4 1 ] [1 2] + [2 1] + 2 [ 1 4 1 ] [1 2] + + 3 [ 1 ] = + [ 4 1] + n-1 [ 1 2] + */ + + MAT A(map,4); + MAT B(map,4); + A.setObjectLabel("The Matrix"); + B.setObjectLabel("The Other Matrix"); + if (myImageID != numImages-1) { // last image assigns none + Array vals(tuple(static_cast(2)*STS::one(), + STS::one(), + static_cast(2)*STS::one())); + Array cols(tuple(myImageID,myImageID + 1)); + A.insertGlobalValues(myImageID ,cols(),vals(0,2)); // insert [2 1] + A.insertGlobalValues(myImageID+1,cols(),vals(1,2)); // insert [1 2] + B.insertGlobalValues(myImageID ,cols(),vals(0,2)); // insert [2 1] + B.insertGlobalValues(myImageID+1,cols(),vals(1,2)); // insert [1 2] } + A.fillComplete(); + B.fillComplete(); + + // we know the map is contiguous... + const size_t NumMyElements = (comm->getRank () == 0) ? + A.getDomainMap ()->getGlobalNumElements () : 0; + RCP > NewMap = + rcp (new Map (INVALID, NumMyElements, ZERO, comm)); + RCP > NewImport = + rcp (new Import (NewMap, A.getColMap ())); + + B.replaceDomainMapAndImporter (NewMap, NewImport); + + applyAndCheckResult(A, B, NewMap, out, success); } + ////////////////////////////////////////////////////////////////////////// + + TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL( CrsMatrix, DomainMapEqualsColMap, LO, GO, Scalar, Node ) + { + typedef CrsMatrix MAT; + typedef ScalarTraits STS; + + const global_size_t INVALID = OrdinalTraits::invalid(); + // get a comm + RCP > comm = Tpetra::getDefaultComm(); + const size_t numImages = comm->getSize(); + const size_t myImageID = comm->getRank(); + if (numImages < 3) return; + // create a Map + RCP > map = + createContigMapWithNode(INVALID,2,comm); + + /* Create the following matrix: + 0 [2 1 ] [2 1] + 1 [1 2 ] [1 2] + 2 [ 2 1 ] + [2 1] + 3 [ 1 2 ] = [1 2] + + [ 2 1] + 2n-1 [ 1 2] + */ + + MAT A(map,2); + MAT B(map,2); + A.setObjectLabel("The Matrix"); + B.setObjectLabel("The Other Matrix"); + Array vals(tuple(static_cast(2)*STS::one(), + STS::one(), + static_cast(2)*STS::one())); + Array cols(tuple(2*myImageID,2*myImageID + 1)); + A.insertGlobalValues(2*myImageID ,cols(),vals(0,2)); // insert [2 1] + A.insertGlobalValues(2*myImageID+1,cols(),vals(1,2)); // insert [1 2] + B.insertGlobalValues(2*myImageID ,cols(),vals(0,2)); // insert [2 1] + B.insertGlobalValues(2*myImageID+1,cols(),vals(1,2)); // insert [1 2] + A.fillComplete(); + B.fillComplete(); + + // Use the column map to exercise the check for Domain Map == Column Map + // in replaceDomainMap; By construction, Column Map is 1-to-1 + RCP > NewMap = B.getColMap(); + + B.replaceDomainMap (NewMap); + + applyAndCheckResult(A, B, NewMap, out, success); + } // // INSTANTIATIONS // #define UNIT_TEST_GROUP( SCALAR, LO, GO, NODE ) \ - TEUCHOS_UNIT_TEST_TEMPLATE_4_INSTANT( CrsMatrix, ReplaceDomainMapAndImporter, LO, GO, SCALAR, NODE ) + TEUCHOS_UNIT_TEST_TEMPLATE_4_INSTANT( CrsMatrix, ReplaceDomainMap, LO, GO, SCALAR, NODE ) \ + TEUCHOS_UNIT_TEST_TEMPLATE_4_INSTANT( CrsMatrix, ReplaceDomainMapAndImporter, LO, GO, SCALAR, NODE ) \ + TEUCHOS_UNIT_TEST_TEMPLATE_4_INSTANT( CrsMatrix, DomainMapEqualsColMap, LO, GO, SCALAR, NODE ) TPETRA_ETI_MANGLING_TYPEDEFS() diff --git a/packages/tpetra/core/test/CrsMatrix/CrsMatrix_ReplaceRangeMapAndExporter.cpp b/packages/tpetra/core/test/CrsMatrix/CrsMatrix_ReplaceRangeMapAndExporter.cpp new file mode 100644 index 000000000000..1496d0deae2c --- /dev/null +++ b/packages/tpetra/core/test/CrsMatrix/CrsMatrix_ReplaceRangeMapAndExporter.cpp @@ -0,0 +1,321 @@ +/* +// @HEADER +// *********************************************************************** +// +// Tpetra: Templated Linear Algebra Services Package +// Copyright (2008) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Michael A. Heroux (maherou@sandia.gov) +// +// ************************************************************************ +// @HEADER +*/ + +#include "Tpetra_TestingUtilities.hpp" +#include "Tpetra_MultiVector.hpp" +#include "Tpetra_CrsMatrix.hpp" + +namespace { + + using std::endl; + using std::string; + + using Teuchos::as; + using Teuchos::RCP; + using Teuchos::ArrayRCP; + using Teuchos::rcp; + using Teuchos::arcp; + using Teuchos::outArg; + using Teuchos::arcpClone; + using Teuchos::arrayView; + using Teuchos::broadcast; + using Teuchos::OrdinalTraits; + using Teuchos::ScalarTraits; + using Teuchos::Comm; + using Teuchos::Array; + using Teuchos::ArrayView; + using Teuchos::tuple; + using Teuchos::null; + using Teuchos::ParameterList; + using Teuchos::parameterList; + + using Tpetra::Map; + using Tpetra::MultiVector; + using Tpetra::Vector; + using Tpetra::Operator; + using Tpetra::CrsMatrix; + using Tpetra::CrsGraph; + using Tpetra::RowMatrix; + using Tpetra::Export; + using Tpetra::global_size_t; + using Tpetra::createContigMapWithNode; + using Tpetra::createVector; + using Tpetra::ProfileType; + using Tpetra::StaticProfile; + using Tpetra::OptimizeOption; + using Tpetra::DoOptimizeStorage; + using Tpetra::DoNotOptimizeStorage; + using Tpetra::GloballyDistributed; + using Tpetra::INSERT; + + TEUCHOS_STATIC_SETUP() + { + Teuchos::CommandLineProcessor &clp = Teuchos::UnitTestRepository::getCLP(); + clp.addOutputSetupOptions(true); + } + + // + // UNIT TESTS + // + template + void applyAndCheckResult( + Tpetra::CrsMatrix &A, + Tpetra::CrsMatrix &B, + Teuchos::RCP > &newMap, + Teuchos::FancyOStream &out, + bool &success + ) + { + typedef Teuchos::ScalarTraits STS; + typedef typename STS::magnitudeType MT; + MT norm = ScalarTraits::zero (); + + /* Fill a random vector on the original map */ + Tpetra::Vector vecX(A.getDomainMap()); + vecX.randomize(); + + /* Now do some multiplies */ + Tpetra::Vector AVecY(A.getRangeMap()); + Tpetra::Vector BVecY(B.getRangeMap()); + A.apply(vecX,AVecY); + B.apply(vecX,BVecY); + + /* Export BVecY to the original range map for comparisons */ + Tpetra::Vector BVecYOrig(A.getRangeMap()); + Tpetra::Export TempExport(newMap, A.getRangeMap()); + BVecYOrig.doExport(BVecY,TempExport,Tpetra::INSERT); + + BVecYOrig.update (-STS::one (), AVecY, STS::one ()); + norm = BVecYOrig.norm2(); + + std::cout << "Residual 2-norm: " << norm << std::endl + << "Residual 1-norm: " << BVecYOrig.norm1 () << std::endl + << "Residual Inf-norm: " << BVecYOrig.normInf () << std::endl; + + const bool normSmallEnough = (norm <= as (1e-10)); + TEST_EQUALITY ( normSmallEnough, true ); + } + + ////////////////////////////////////////////////////////////////////////// + + TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL( CrsMatrix, ReplaceRangeMap, LO, GO, Scalar, Node ) + { + // Based on the FullTriDiag tests... + + typedef CrsMatrix MAT; + typedef ScalarTraits STS; + + const size_t ONE = OrdinalTraits::one(); + const size_t ZERO = OrdinalTraits::zero(); + const global_size_t INVALID = OrdinalTraits::invalid(); + // get a comm + RCP > comm = Tpetra::getDefaultComm(); + const size_t numImages = comm->getSize(); + const size_t myImageID = comm->getRank(); + if (numImages < 3) return; + // create a Map + RCP > map = + createContigMapWithNode(INVALID,ONE,comm); + + /* Create the following matrix: + 0 [2 1 ] [2 1] + 1 [1 4 1 ] [1 2] + [2 1] + 2 [ 1 4 1 ] [1 2] + + 3 [ 1 ] = + [ 4 1] + n-1 [ 1 2] + */ + + MAT A(map,4); + MAT B(map,4); + A.setObjectLabel("The Matrix"); + B.setObjectLabel("The Other Matrix"); + if (myImageID != numImages-1) { // last image assigns none + Array vals(tuple(static_cast(2)*STS::one(), + STS::one(), + static_cast(2)*STS::one())); + Array cols(tuple(myImageID,myImageID + 1)); + A.insertGlobalValues(myImageID ,cols(),vals(0,2)); // insert [2 1] + A.insertGlobalValues(myImageID+1,cols(),vals(1,2)); // insert [1 2] + B.insertGlobalValues(myImageID ,cols(),vals(0,2)); // insert [2 1] + B.insertGlobalValues(myImageID+1,cols(),vals(1,2)); // insert [1 2] + } + A.fillComplete(); + B.fillComplete(); + + + // Build a one-process Map. + // we know the map is contiguous... + const size_t NumMyElements = (comm->getRank () == 0) ? + A.getRangeMap ()->getGlobalNumElements () : 0; + RCP > NewMap = + rcp (new Map (INVALID, NumMyElements, ZERO, comm)); + + B.replaceRangeMap (NewMap); + + applyAndCheckResult(A, B, NewMap, out, success); + } + + ////////////////////////////////////////////////////////////////////////// + + TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL( CrsMatrix, ReplaceRangeMapAndExporter, LO, GO, Scalar, Node ) + { + // Based on the FullTriDiag tests... + + typedef CrsMatrix MAT; + typedef ScalarTraits STS; + + const size_t ONE = OrdinalTraits::one(); + const size_t ZERO = OrdinalTraits::zero(); + const global_size_t INVALID = OrdinalTraits::invalid(); + // get a comm + RCP > comm = Tpetra::getDefaultComm(); + const size_t numImages = comm->getSize(); + const size_t myImageID = comm->getRank(); + if (numImages < 3) return; + // create a Map + RCP > map = + createContigMapWithNode(INVALID,ONE,comm); + + /* Create the following matrix: + 0 [2 1 ] [2 1] + 1 [1 4 1 ] [1 2] + [2 1] + 2 [ 1 4 1 ] [1 2] + + 3 [ 1 ] = + [ 4 1] + n-1 [ 1 2] + */ + + MAT A(map,4); + MAT B(map,4); + A.setObjectLabel("The Matrix"); + B.setObjectLabel("The Other Matrix"); + if (myImageID != numImages-1) { // last image assigns none + Array vals(tuple(static_cast(2)*STS::one(), + STS::one(), + static_cast(2)*STS::one())); + Array cols(tuple(myImageID,myImageID + 1)); + A.insertGlobalValues(myImageID ,cols(),vals(0,2)); // insert [2 1] + A.insertGlobalValues(myImageID+1,cols(),vals(1,2)); // insert [1 2] + B.insertGlobalValues(myImageID ,cols(),vals(0,2)); // insert [2 1] + B.insertGlobalValues(myImageID+1,cols(),vals(1,2)); // insert [1 2] + } + A.fillComplete(); + B.fillComplete(); + + // we know the map is contiguous... + const size_t NumMyElements = (comm->getRank () == 0) ? + A.getRangeMap ()->getGlobalNumElements () : 0; + RCP > NewMap = + rcp (new Map (INVALID, NumMyElements, ZERO, comm)); + RCP > NewExport = + rcp (new Export (B.getRowMap (), NewMap)); + + B.replaceRangeMapAndExporter (NewMap, NewExport); + + applyAndCheckResult(A, B, NewMap, out, success); + } + + ////////////////////////////////////////////////////////////////////////// + + TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL( CrsMatrix, RangeMapEqualsRowMap, LO, GO, Scalar, Node ) + { + typedef CrsMatrix MAT; + typedef ScalarTraits STS; + + const global_size_t INVALID = OrdinalTraits::invalid(); + // get a comm + RCP > comm = Tpetra::getDefaultComm(); + const size_t numImages = comm->getSize(); + const size_t myImageID = comm->getRank(); + if (numImages < 3) return; + // create a Map + RCP > map = + createContigMapWithNode(INVALID,2,comm); + + /* Create the following matrix: + 0 [2 1 ] [2 1] + 1 [1 2 ] [1 2] + 2 [ 2 1 ] + [2 1] + 3 [ 1 2 ] = [1 2] + + [ 2 1] + 2n-1 [ 1 2] + */ + + MAT A(map,2); + MAT B(map,2); + A.setObjectLabel("The Matrix"); + B.setObjectLabel("The Other Matrix"); + Array vals(tuple(static_cast(2)*STS::one(), + STS::one(), + static_cast(2)*STS::one())); + Array cols(tuple(2*myImageID,2*myImageID + 1)); + A.insertGlobalValues(2*myImageID ,cols(),vals(0,2)); // insert [2 1] + A.insertGlobalValues(2*myImageID+1,cols(),vals(1,2)); // insert [1 2] + B.insertGlobalValues(2*myImageID ,cols(),vals(0,2)); // insert [2 1] + B.insertGlobalValues(2*myImageID+1,cols(),vals(1,2)); // insert [1 2] + A.fillComplete(); + B.fillComplete(); + + // Use the row map to exercise the check for Range Map == Row Map + // in replaceRangeMap; By construction, Column Map is 1-to-1 + RCP > NewMap = B.getRowMap(); + + B.replaceRangeMap (NewMap); + + applyAndCheckResult(A, B, NewMap, out, success); + } +// +// INSTANTIATIONS +// + +#define UNIT_TEST_GROUP( SCALAR, LO, GO, NODE ) \ + TEUCHOS_UNIT_TEST_TEMPLATE_4_INSTANT( CrsMatrix, ReplaceRangeMap, LO, GO, SCALAR, NODE ) \ + TEUCHOS_UNIT_TEST_TEMPLATE_4_INSTANT( CrsMatrix, ReplaceRangeMapAndExporter, LO, GO, SCALAR, NODE ) \ + TEUCHOS_UNIT_TEST_TEMPLATE_4_INSTANT( CrsMatrix, RangeMapEqualsRowMap, LO, GO, SCALAR, NODE ) + + TPETRA_ETI_MANGLING_TYPEDEFS() + + TPETRA_INSTANTIATE_SLGN( UNIT_TEST_GROUP ) + +}