feat(ForcingsEngineGriddedDataProvider): integrate tests; add gridded…

… domain; simplify test files

data/forcing/forcings-engine/config_aorc_gridded.yml.in

@@ -0,0 +1,61 @@
+time_step_seconds: 3600
+initial_time: 0
+NWM_VERSION: 4.0
+NWM_CONFIG: "AORC"
+InputForcings: [12] 
+InputForcingDirectories: ['']
+InputForcingTypes: ["NETCDF4"]
+InputMandatory: [1]
+OutputFrequency: 60
+SubOutputHour: 0
+SubOutFreq: 0
+ScratchDir: "@FORCINGS_ENGINE_SCRATCH_DIR@"
+Output: 1
+compressOutput: 0
+floatOutput: 0
+AnAFlag: 0
+LookBack : -9999
+RefcstBDateProc: "202301170000"
+RefcstEDateProc: "202301170100"
+ForecastFrequency: 60
+ForecastShift: 0
+ForecastInputHorizons: [14400]
+ForecastInputOffsets: [0]
+GeogridIn: "data/forcing/forcings-engine/Gauge_01073000_Grid.nc"
+SpatialMetaIn: ""
+GRID_TYPE: "gridded"
+LONVAR: 'Longitude'
+LATVAR: 'Latitude'
+HGTVAR: 'Elevation'
+SLOPE: 'Slope'
+SLOPE_AZIMUTH: 'Aspect'
+IgnoredBorderWidths: [0]
+RegridOpt: [1]
+ForcingTemporalInterpolation: [0]
+TemperatureBiasCorrection: [0]
+PressureBiasCorrection: [0]
+HumidityBiasCorrection: [0]
+WindBiasCorrection: [0]
+SwBiasCorrection: [0]
+LwBiasCorrection: [0]
+PrecipBiasCorrection: [0]
+TemperatureDownscaling: [0]
+ShortwaveDownscaling: [0]
+PressureDownscaling: [0]
+PrecipDownscaling: [0]
+HumidityDownscaling: [0]
+DownscalingParamDirs: ["@FORCINGS_ENGINE_SCRATCH_DIR@"]
+SuppPcp: []
+SuppPcpForcingTypes: ''
+SuppPcpDirectories: ''
+SuppPcpParamDir: ''
+RegridOptSuppPcp: []
+SuppPcpTemporalInterpolation: []
+SuppPcpInputOffsets: []
+SuppPcpMandatory: []
+RqiMethod: 0
+RqiThreshold: 0.9
+cfsEnsNumber: ''
+custom_input_fcst_freq: []
+includeLQFrac: 0
+

include/forcing/ForcingsEngineGriddedDataProvider.hpp

@@ -94,7 +94,7 @@ struct ForcingsEngineGriddedDataProvider final :
         data_access::ReSampleMethod m
     ) override;
 
-    const GridMask& mask();
+    const GridMask& mask() const noexcept;
 
   private:
     int               var_grid_id_;

src/forcing/CMakeLists.txt

@@ -32,6 +32,7 @@ if(NGEN_WITH_PYTHON)
       PRIVATE
         "${CMAKE_CURRENT_LIST_DIR}/ForcingsEngineDataProvider.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/ForcingsEngineLumpedDataProvider.cpp"
+        "${CMAKE_CURRENT_LIST_DIR}/ForcingsEngineGriddedDataProvider.cpp"
     )
     target_link_libraries(forcing PUBLIC pybind11::embed NGen::ngen_bmi)
 endif()

src/forcing/ForcingsEngineGriddedDataProvider.cpp

@@ -140,4 +140,9 @@ std::vector<Provider::data_type> Provider::get_values(
     return values;
 }
 
+const GridMask& Provider::mask() const noexcept
+{
+    return var_grid_mask_;
+}
+
 } // namespace data_access

test/CMakeLists.txt

@@ -188,6 +188,7 @@ ngen_add_test(
     OBJECTS
         forcing/ForcingsEngineDataProvider_Test.cpp
         forcing/ForcingsEngineLumpedDataProvider_Test.cpp
+        forcing/ForcingsEngineGriddedDataProvider_Test.cpp
     LIBRARIES
         NGen::forcing
     REQUIRES
@@ -197,14 +198,23 @@ ngen_add_test(
 if(TARGET test_forcings_engine)
     set(FORCINGS_ENGINE_SCRATCH_DIR "${CMAKE_CURRENT_BINARY_DIR}/forcings_engine_scratch")
     file(MAKE_DIRECTORY "${FORCINGS_ENGINE_SCRATCH_DIR}")
+
+    # Configure Lumped
+    configure_file(
+        "${CMAKE_CURRENT_LIST_DIR}/../data/forcing/forcings-engine/config_aorc_lumped.yml.in" # Input
+        "${CMAKE_CURRENT_BINARY_DIR}/config_aorc_lumped.yml"
+    )
+
+    # Configure Gridded
     configure_file(
-        "${CMAKE_CURRENT_LIST_DIR}/../data/forcing/forcings-engine/config_aorc.yml.in" # Input
-        "${CMAKE_CURRENT_BINARY_DIR}/config_aorc.yml"
+        "${CMAKE_CURRENT_LIST_DIR}/../data/forcing/forcings-engine/config_aorc_gridded.yml.in" # Input
+        "${CMAKE_CURRENT_BINARY_DIR}/config_aorc_gridded.yml"
     )
 
     # Use global add instead of target add so that both `test_forcings_engine` and `test_all`
     # receive this compile definition.
-    add_compile_definitions(NGEN_LUMPED_CONFIG_PATH="${CMAKE_CURRENT_BINARY_DIR}/config_aorc.yml")
+    add_compile_definitions(NGEN_FORCINGS_LUMPED_CONFIG_PATH="${CMAKE_CURRENT_BINARY_DIR}/config_aorc_lumped.yml")
+    add_compile_definitions(NGEN_FORCINGS_GRIDDED_CONFIG_PATH="${CMAKE_CURRENT_BINARY_DIR}/config_aorc_gridded.yml")
 endif()
 
 ########################## Series Unit Tests

test/forcing/ForcingsEngineDataProvider_Test.cpp

@@ -1,5 +1,13 @@
 #include <forcing/ForcingsEngineDataProvider.hpp>
 
 #include "DataProviderSelectors.hpp"
+#include "GriddedDataSelector.hpp"
+
+#include "ForcingsEngineTestHelpers.hpp"
 
 template struct data_access::ForcingsEngineDataProvider<double, CatchmentAggrDataSelector>;
+template struct data_access::ForcingsEngineDataProvider<double, GriddedDataSelector>;
+
+std::shared_ptr<utils::ngenPy::InterpreterUtil> ForcingsEngineDataProviderTest::gil_ = nullptr;
+std::time_t ForcingsEngineDataProviderTest::time_start = 0;
+std::time_t ForcingsEngineDataProviderTest::time_end = 0;

test/forcing/ForcingsEngineGriddedDataProvider_Test.cpp

@@ -0,0 +1,109 @@
+#include <gtest/gtest.h>
+
+#ifndef NGEN_FORCINGS_GRIDDED_CONFIG_PATH
+#error "Gridded config file path not defined! Set `-DNGEN_FORCINGS_GRIDDED_CONFIG_PATH`. " \
+       "Note: you should not be seeing this compile error. Please contact an ngen developer."
+#endif
+
+#include "ForcingsEngineTestHelpers.hpp"
+
+#include <forcing/ForcingsEngineGriddedDataProvider.hpp>
+
+struct ForcingsEngineGriddedDataProviderTest
+  : public ForcingsEngineDataProviderTest
+{
+    using provider_type = data_access::ForcingsEngineGriddedDataProvider;
+
+    ForcingsEngineGriddedDataProviderTest()
+      : ForcingsEngineDataProviderTest()
+    {}
+
+    static std::unique_ptr<provider_type> provider_;
+
+    static void SetUpTestSuite();
+};
+
+using TestFixture = ForcingsEngineGriddedDataProviderTest;
+
+std::unique_ptr<TestFixture::provider_type> TestFixture::provider_ = nullptr;
+
+constexpr const char* config_file = NGEN_FORCINGS_GRIDDED_CONFIG_PATH;
+
+constexpr auto cat_11223_mask = geojson::box_t{
+  /*min_corner=*/{ -71.0554067, 43.1423006 },
+  /*max_corner=*/{ -70.9668753, 43.185111 }
+};
+
+constexpr auto cat_11410_mask = geojson::box_t{
+  /*min_corner=*/{ -71.0577593, 43.1375584 },
+  /*max_corner=*/{ -71.0210404, 43.1761626 }
+};
+
+void TestFixture::SetUpTestSuite()
+{
+    ForcingsEngineDataProviderTest::SetUpTestSuite();
+
+    TestFixture::time_start = data_access::detail::parse_time("2023-01-17 01:00:00");
+    TestFixture::time_end   = TestFixture::time_start + 3600 + 3600;
+
+    TestFixture::provider_ = std::make_unique<data_access::ForcingsEngineGriddedDataProvider>(
+        /*init=*/config_file,
+        /*time_begin_seconds=*/TestFixture::time_start,
+        /*time_end_seconds=*/TestFixture::time_end,
+        /*mask=*/cat_11223_mask
+    );
+}
+
+/**
+ * Tests for the flyweight-like design of provider storage by getting
+ * a new instance of the forcings engine and verifying that it points
+ * to the same address as the static initialized `provider_` member,
+ * based on matching `init`, and shared over distinct `divide_id`.
+ */
+TEST_F(ForcingsEngineGriddedDataProviderTest, Storage)
+{
+    auto new_inst = std::make_unique<data_access::ForcingsEngineGriddedDataProvider>(
+        /*init=*/config_file,
+        /*time_begin_seconds=*/TestFixture::time_start,
+        /*time_end_seconds=*/TestFixture::time_end,
+        /*mask=*/cat_11410_mask
+    );
+
+    ASSERT_EQ(new_inst->model(), provider_->model());
+}
+
+TEST_F(ForcingsEngineGriddedDataProviderTest, VariableAccess)
+{
+    ASSERT_TRUE(boost::geometry::equals(provider_->mask().extent, cat_11223_mask));
+
+    constexpr std::array<const char*, 8> expected_variables = {
+        "U2D_ELEMENT",
+        "V2D_ELEMENT",
+        "LWDOWN_ELEMENT",
+        "SWDOWN_ELEMENT",
+        "T2D_ELEMENT",
+        "Q2D_ELEMENT",
+        "PSFC_ELEMENT",
+        "RAINRATE_ELEMENT"
+    };
+
+    const auto outputs = provider_->get_available_variable_names();
+
+    // Check that each expected variable is in the list of available outputs.
+    for (const auto& expected : expected_variables) {
+        EXPECT_NE(
+            std::find(outputs.begin(), outputs.end(), expected),
+            outputs.end()
+        );
+    }
+
+    auto selector = GriddedDataSelector{"PSFC", time_start, 3600, "seconds"};
+    auto result   = provider_->get_values(selector, data_access::ReSampleMethod::SUM);
+    EXPECT_EQ(result.size(), 48980);
+    // EXPECT_NEAR(result, 99580.52, 1e-2);
+
+    // selector = CatchmentAggrDataSelector{"cat-11223", "LWDOWN", time_start + 3600, 3600, "seconds"};
+    // auto result2 = provider_->get_values(selector, data_access::ReSampleMethod::SUM);
+    // ASSERT_GT(result2.size(), 0);
+    // EXPECT_NEAR(result2[0], 0, 1e-6);
+}

test/forcing/ForcingsEngineLumpedDataProvider_Test.cpp

@@ -1,82 +1,51 @@
 #include <gtest/gtest.h>
 
-#ifndef NGEN_LUMPED_CONFIG_PATH
-#error "Lumped config file path not defined! Set `-DNGEN_LUMPED_CONFIG_PATH`"
+#ifndef NGEN_FORCINGS_LUMPED_CONFIG_PATH
+#error "Lumped config file path not defined! Set `-DNGEN_FORCINGS_LUMPED_CONFIG_PATH`. " \
+       "Note: you should not be seeing this compile error. Please contact an ngen developer."
 #endif
 
-#include <NGenConfig.h>
-#if NGEN_WITH_MPI
-#include <mpi.h>
-#endif
+#include "ForcingsEngineTestHelpers.hpp"
 
 #include <forcing/ForcingsEngineLumpedDataProvider.hpp>
-#include <forcing/AorcForcing.hpp>
-#include <utilities/python/InterpreterUtil.hpp>
 
 #include <string>
 
 struct ForcingsEngineLumpedDataProviderTest
-  : public testing::Test
+  : public ForcingsEngineDataProviderTest
 {
     using provider_type = data_access::ForcingsEngineLumpedDataProvider;
 
     ForcingsEngineLumpedDataProviderTest()
-    {
-        #if NGEN_WITH_MPI
-        MPI_Comm_size(MPI_COMM_WORLD, &mpi_.size);
-        MPI_Comm_rank(MPI_COMM_WORLD, &mpi_.rank);
-        #endif
-    }
+      : ForcingsEngineDataProviderTest()
+    {}
 
-    static constexpr const char* config_file = NGEN_LUMPED_CONFIG_PATH;
-    static const std::time_t time_start;
-    static const std::time_t time_end;
-    static std::shared_ptr<utils::ngenPy::InterpreterUtil> gil_;
     static std::unique_ptr<provider_type> provider_;
 
-    struct { int initialized = 0, finalized = 0, size = 1, rank = 0; } mpi_{};
-
     static void SetUpTestSuite();
-    static void TearDownTestSuite();
 };
 
 using TestFixture = ForcingsEngineLumpedDataProviderTest;
 
-constexpr const char* TestFixture::config_file;
-const std::time_t TestFixture::time_start = data_access::detail::parse_time("2023-01-17 01:00:00");
-const std::time_t TestFixture::time_end = TestFixture::time_start + 3600 + 3600;
-
-std::shared_ptr<utils::ngenPy::InterpreterUtil> TestFixture::gil_ = nullptr;
 std::unique_ptr<TestFixture::provider_type> TestFixture::provider_ = nullptr;
 
+constexpr const char* config_file = NGEN_FORCINGS_LUMPED_CONFIG_PATH;
+
 void TestFixture::SetUpTestSuite()
 {
-    #if NGEN_WITH_MPI
-    MPI_Init(nullptr, nullptr);
-    #endif
+    ForcingsEngineDataProviderTest::SetUpTestSuite();
 
-    TestFixture::gil_ = utils::ngenPy::InterpreterUtil::getInstance();
-
-    data_access::detail::assert_forcings_engine_requirements();
+    TestFixture::time_start = data_access::detail::parse_time("2023-01-17 01:00:00");
+    TestFixture::time_end   = TestFixture::time_start + 3600 + 3600;
 
     TestFixture::provider_ = std::make_unique<data_access::ForcingsEngineLumpedDataProvider>(
-        /*init=*/TestFixture::config_file,
+        /*init=*/config_file,
         /*time_begin_seconds=*/TestFixture::time_start,
         /*time_end_seconds=*/TestFixture::time_end,
         /*divide_id=*/"cat-11223"
     );
 }
 
-void TestFixture::TearDownTestSuite()
-{
-    data_access::detail::ForcingsEngineStorage::instances.clear();
-    gil_.reset();
-
-    #if NGEN_WITH_MPI
-    PMPI_Finalize();
-    #endif
-}
-
 /**
  * Tests for the flyweight-like design of provider storage by getting
  * a new instance of the forcings engine and verifying that it points
@@ -86,7 +55,7 @@ void TestFixture::TearDownTestSuite()
 TEST_F(ForcingsEngineLumpedDataProviderTest, Storage)
 {
     auto new_inst = std::make_unique<data_access::ForcingsEngineLumpedDataProvider>(
-        /*init=*/TestFixture::config_file,
+        /*init=*/config_file,
         /*time_begin_seconds=*/TestFixture::time_start,
         /*time_end_seconds=*/TestFixture::time_end,
         /*divide_id=*/"cat-11371"
@@ -128,5 +97,5 @@ TEST_F(ForcingsEngineLumpedDataProviderTest, VariableAccess)
     selector = CatchmentAggrDataSelector{"cat-11223", "LWDOWN", time_start + 3600, 3600, "seconds"};
     auto result2 = provider_->get_values(selector, data_access::ReSampleMethod::SUM);
     ASSERT_GT(result2.size(), 0);
-    EXPECT_NEAR(result2[0], 0, 1e-6);
+    EXPECT_NEAR(result2[0], 309.800018, 1e-6);
 }