Merge branch '349_add_performance_summary_to_netcdf_results' into 'de…

…velop'

349 add performance summary to netcdf results

See merge request iek-3/shared-code/fine!355

fine/IOManagement/xarrayIO.py

@@ -77,6 +77,26 @@ def convertOptimizationInputToDatasets(esM, useProcessedValues=False):
     return xr_dss
 
 
+def convertPerformanceSummaryToDatasets(esM):
+    import pandas as pd
+
+    df = esM.performanceSummary.squeeze()
+    df = df.droplevel("Category")
+    df = df.apply(lambda x: pd.to_numeric(x, errors="ignore"))
+    # convert datetime to string
+    for idx, value in df.items():
+        if isinstance(value, pd.Timestamp):
+            print(value)
+            df.loc[idx] = value.strftime("%Y-%m-%d %H:%M:%S")
+    summary_dict = df.to_dict()
+    summary_xr = xr.Dataset()
+    summary_xr.attrs = summary_dict
+
+    xr_dss = {"PerformanceSummary": summary_xr}
+
+    return xr_dss
+
+
 def convertOptimizationOutputToDatasets(esM, optSumOutputLevel=0):
     """
     Takes esM instance output and converts it into an xarray dataset.
@@ -388,7 +408,7 @@ def writeDatasetsToNetCDF(
             pass
 
     for group in datasets.keys():
-        if group == "Parameters":
+        if group == "Parameters" or group == "PerformanceSummary":
             xarray_dataset = datasets[group]
             _xarray_dataset = (
                 xarray_dataset.copy()
@@ -452,7 +472,6 @@ def writeDatasetsToNetCDF(
                 # Use mode='a' to append datasets to existing file. Variables will be overwritten.
                 mode=mode,
             )
-            continue
 
         elif group == "Results":
             for ip in datasets[group].keys():
@@ -474,7 +493,7 @@ def writeDatasetsToNetCDF(
                                 # Use zlib variable compression to reduce filesize with little performance loss
                                 # for our use-case. Complevel 9 for best compression.
                                 encoding={
-                                    var: {"zlib": True, "complevel": 9}
+                                    var: {"zlib": True, "complevel": 5}
                                     for var in list(
                                         datasets[group][ip][model][component].data_vars
                                     )
@@ -497,7 +516,7 @@ def writeDatasetsToNetCDF(
                             # Use zlib variable compression to reduce filesize with little performance loss
                             # for our use-case. Complevel 9 for best compression.
                             encoding={
-                                var: {"zlib": True, "complevel": 9}
+                                var: {"zlib": True, "complevel": 5}
                                 for var in list(
                                     datasets[group][model][component].data_vars
                                 )
@@ -995,6 +1014,12 @@ def writeEnergySystemModelToNetCDF(
     writeDatasetsToNetCDF(xr_dss_input, outputFilePath, groupPrefix=groupPrefix)
     if esM.objectiveValue is not None:  # model was optimized
         xr_dss_output = convertOptimizationOutputToDatasets(esM, optSumOutputLevel)
+        if hasattr(esM, "performanceSummary"):
+            xr_dss_performance = convertPerformanceSummaryToDatasets(esM)
+            xr_dss_output["PerformanceSummary"] = xr_dss_performance[
+                "PerformanceSummary"
+            ]
+            print(xr_dss_output.keys())
         writeDatasetsToNetCDF(xr_dss_output, outputFilePath, groupPrefix=groupPrefix)
 
     utils.output("Done. (%.4f" % (time.time() - _t) + " sec)", esM.verbose, 0)
@@ -1013,11 +1038,21 @@ def writeEnergySystemModelToDatasets(esM):
     if esM.objectiveValue is not None:  # model was optimized
         xr_dss_output = convertOptimizationOutputToDatasets(esM)
         xr_dss_input = convertOptimizationInputToDatasets(esM)
-        xr_dss_results = {
-            "Results": xr_dss_output["Results"],
-            "Input": xr_dss_input["Input"],
-            "Parameters": xr_dss_input["Parameters"],
-        }
+        if hasattr(esM, "performanceSummary"):
+            xr_dss_performance = convertPerformanceSummaryToDatasets(esM)
+
+            xr_dss_results = {
+                "Results": xr_dss_output["Results"],
+                "Input": xr_dss_input["Input"],
+                "Parameters": xr_dss_input["Parameters"],
+                "PerformanceSummary": xr_dss_performance["PerformanceSummary"],
+            }
+        else:
+            xr_dss_results = {
+                "Results": xr_dss_output["Results"],
+                "Input": xr_dss_input["Input"],
+                "Parameters": xr_dss_input["Parameters"],
+            }
     else:
         xr_dss_input = convertOptimizationInputToDatasets(esM)
         xr_dss_results = {
@@ -1089,7 +1124,12 @@ def readNetCDFToDatasets(filePath="my_esm.nc", groupPrefix=None, lazy_load=False
                 for ip_key in group_keys["Results"].groups
             }
         # read parameters from netcdf
-        xr_dss["Parameters"] = loader(filePath, group="Parameters")
+        xr_dss["Parameters"] = xr.load_dataset(filePath, group="Parameters")
+        # read performance summary from netcdf (if exists)
+        if "PerformanceSummary" in group_keys:
+            xr_dss["PerformanceSummary"] = xr.load_dataset(
+                filePath, group="PerformanceSummary"
+            )
     else:
         xr_dss = {}
         # read input from netcdf
@@ -1119,7 +1159,14 @@ def readNetCDFToDatasets(filePath="my_esm.nc", groupPrefix=None, lazy_load=False
                 for ip_key in group_keys["Results"].groups
             }
         # read parameters from netcdf
-        xr_dss["Parameters"] = loader(filePath, group=f"{groupPrefix}/Parameters")
+        xr_dss["Parameters"] = xr.load_dataset(
+            filePath, group=f"{groupPrefix}/Parameters"
+        )
+        # read performance summary from netcdf (if exists)
+        if "PerformanceSummary" in group_keys:
+            xr_dss["PerformanceSummary"] = xr.load_dataset(
+                filePath, group=f"{groupPrefix}/PerformanceSummary"
+            )
 
     return xr_dss