diff --git a/powersimdata/data_access/data_access.py b/powersimdata/data_access/data_access.py index e5a468743..097a649da 100644 --- a/powersimdata/data_access/data_access.py +++ b/powersimdata/data_access/data_access.py @@ -9,10 +9,7 @@ from fs.path import basename, dirname from fs.tempfs import TempFS -from powersimdata.data_access.profile_helper import ( - get_profile_version_cloud, - get_profile_version_local, -) +from powersimdata.data_access.profile_helper import get_profile_version from powersimdata.data_access.ssh_fs import WrapSSHFS from powersimdata.utility import server_setup @@ -186,8 +183,9 @@ def get_profile_version(self, grid_model, kind): :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*. :return: (*list*) -- available profile version. """ - blob_version = get_profile_version_cloud(grid_model, kind) - local_version = get_profile_version_local(grid_model, kind) + bfs = fs.open_fs("azblob://besciences@profiles") + blob_version = get_profile_version(bfs, grid_model, kind) + local_version = get_profile_version(self.local_fs, grid_model, kind) return list(set(blob_version + local_version)) def checksum(self, relative_path): diff --git a/powersimdata/data_access/profile_helper.py b/powersimdata/data_access/profile_helper.py index dc1ec8c79..1708c15e9 100644 --- a/powersimdata/data_access/profile_helper.py +++ b/powersimdata/data_access/profile_helper.py @@ -1,51 +1,20 @@ -import fs +def get_profile_version(_fs, grid_model, kind): + """Returns available raw profile from the given filesystem -from powersimdata.utility import server_setup - - -def _get_profile_version(_fs, kind): - """Returns available raw profiles from the given filesystem :param fs.base.FS _fs: filesystem instance + :param str grid_model: grid model. :param str kind: *'demand'*, *'hydro'*, *'solar'*, *'wind'*, *'demand_flexibility_up'*, *'demand_flexibility_dn'*, *'demand_flexibility_cost_up'*, or *'demand_flexibility_cost_dn'*. :return: (*list*) -- available profile version. """ + _fs = _fs.makedirs(f"raw/{grid_model}", recreate=True) matching = [f for f in _fs.listdir(".") if kind in f] # Don't include demand flexibility profiles as possible demand profiles if kind == "demand": - matching_ = [p for p in matching if "demand_flexibility" not in p] - return [f.lstrip(f"{kind}_").rstrip(".csv") for f in matching_] - else: - return [f.lstrip(f"{kind}_").rstrip(".csv") for f in matching] - - -def get_profile_version_cloud(grid_model, kind): - """Returns available raw profile from blob storage. - - :param str grid_model: grid model. - :param str kind: *'demand'*, *'hydro'*, *'solar'*, *'wind'*, - *'demand_flexibility_up'*, *'demand_flexibility_dn'*, - *'demand_flexibility_cost_up'*, or *'demand_flexibility_cost_dn'*. - :return: (*list*) -- available profile version. - """ - bfs = fs.open_fs("azblob://besciences@profiles").opendir(f"raw/{grid_model}") - return _get_profile_version(bfs, kind) - - -def get_profile_version_local(grid_model, kind): - """Returns available raw profile from local file. - - :param str grid_model: grid model. - :param str kind: *'demand'*, *'hydro'*, *'solar'*, *'wind'*, - *'demand_flexibility_up'*, *'demand_flexibility_dn'*, - *'demand_flexibility_cost_up'*, or *'demand_flexibility_cost_dn'*. - :return: (*list*) -- available profile version. - """ - profile_dir = fs.path.join(server_setup.LOCAL_DIR, "raw", grid_model) - lfs = fs.open_fs(profile_dir, create=True) - return _get_profile_version(lfs, kind) + matching = [p for p in matching if "demand_flexibility" not in p] + return [f.lstrip(f"{kind}_").rstrip(".csv") for f in matching] class ProfileHelper: diff --git a/powersimdata/data_access/tests/test_profile_helper.py b/powersimdata/data_access/tests/test_profile_helper.py index 8ed6825e5..eaf313500 100644 --- a/powersimdata/data_access/tests/test_profile_helper.py +++ b/powersimdata/data_access/tests/test_profile_helper.py @@ -1,16 +1,17 @@ from fs.tempfs import TempFS -from powersimdata.data_access.profile_helper import ProfileHelper, _get_profile_version +from powersimdata.data_access.profile_helper import ProfileHelper, get_profile_version def test_get_profile_version(): with TempFS() as tmp_fs: - tfs = tmp_fs.makedirs("raw/usa_tamu", recreate=True) - tfs.touch("solar_vOct2022.csv") - tfs.touch("foo_v1.0.1.csv") - v_solar = _get_profile_version(tfs, "solar") - v_foo = _get_profile_version(tfs, "foo") - v_missing = _get_profile_version(tfs, "missing") + grid_model = "usa_tamu" + sub_fs = tmp_fs.makedirs(f"raw/{grid_model}", recreate=True) + sub_fs.touch("solar_vOct2022.csv") + sub_fs.touch("foo_v1.0.1.csv") + v_solar = get_profile_version(tmp_fs, grid_model, "solar") + v_foo = get_profile_version(tmp_fs, grid_model, "foo") + v_missing = get_profile_version(tmp_fs, grid_model, "missing") assert "vOct2022" == v_solar[0] assert "v1.0.1" == v_foo[0] assert [] == v_missing diff --git a/powersimdata/input/tests/test_change_table.py b/powersimdata/input/tests/test_change_table.py index 23b320ada..04df52571 100644 --- a/powersimdata/input/tests/test_change_table.py +++ b/powersimdata/input/tests/test_change_table.py @@ -1,12 +1,10 @@ -import os -import shutil -from pathlib import Path - import pandas as pd import pytest +from powersimdata.data_access.context import Context from powersimdata.input.change_table import ChangeTable from powersimdata.input.grid import Grid +from powersimdata.tests.mock_context import MockContext grid = Grid(["USA"]) @@ -512,7 +510,7 @@ def test_remove_bus(ct): ct.remove_bus({845}) -def test_add_demand_flexibility(ct): +def test_add_demand_flexibility(ct, monkeypatch): with pytest.raises(ValueError): # Fails because "demand_flexibility_dn", a required key, is not included ct.add_demand_flexibility( @@ -540,15 +538,18 @@ def test_add_demand_flexibility(ct): } ) + monkeypatch.setattr(Context, "get_data_access", MockContext().get_data_access) + data_access = Context.get_data_access() + # Create fake files in the expected directory path - exp_path = os.path.join(Path.home(), "ScenarioData", "raw", str(grid.grid_model)) - dir_exists_prev = True - if not os.path.isdir(exp_path): - os.makedirs(exp_path) - dir_exists_prev = False - fake_df = pd.DataFrame() - fake_df.to_csv(os.path.join(exp_path, "demand_flexibility_up_Test.csv")) - fake_df.to_csv(os.path.join(exp_path, "demand_flexibility_dn_Test.csv")) + exp_path = f"raw/{grid.grid_model}" + + for csv_file in ( + "demand_flexibility_up_Test.csv", + "demand_flexibility_dn_Test.csv", + ): + with data_access.write(exp_path + "/" + csv_file) as f: + pd.DataFrame().to_csv(f) # Add a test instance of demand flexibility to the change table ct.add_demand_flexibility( @@ -566,10 +567,3 @@ def test_add_demand_flexibility(ct): }, } assert ct.ct == exp_dict - - # Delete the created directory and fake data - if dir_exists_prev: - os.remove(os.path.join(exp_path, "demand_flexibility_up_Test.csv")) - os.remove(os.path.join(exp_path, "demand_flexibility_dn_Test.csv")) - else: - shutil.rmtree(os.path.join(Path.home(), "ScenarioData")) diff --git a/powersimdata/input/tests/test_transform_profile.py b/powersimdata/input/tests/test_transform_profile.py index 7dc4edd11..c960c49c1 100644 --- a/powersimdata/input/tests/test_transform_profile.py +++ b/powersimdata/input/tests/test_transform_profile.py @@ -1,6 +1,3 @@ -import os -import shutil -from pathlib import Path from unittest.mock import patch import numpy as np @@ -8,10 +5,12 @@ import pytest from numpy.testing import assert_almost_equal +from powersimdata.data_access.context import Context from powersimdata.input.change_table import ChangeTable from powersimdata.input.grid import Grid from powersimdata.input.transform_grid import TransformGrid from powersimdata.input.transform_profile import TransformProfile +from powersimdata.tests.mock_context import MockContext from powersimdata.tests.mock_input_data import MockInputData interconnect = ["Western"] @@ -366,24 +365,25 @@ def test_new_hydro_profile(base_grid, raw_hydro): def test_flexible_demand_profiles_are_trimmed( - base_grid, raw_demand_flexibility_up, raw_demand_flexibility_dn + base_grid, raw_demand_flexibility_up, raw_demand_flexibility_dn, monkeypatch ): + monkeypatch.setattr(Context, "get_data_access", MockContext().get_data_access) + data_access = Context.get_data_access() + # Specify the fake demand flexibility profiles from MockInputData zone_keys = [f"zone.{z}" for z in base_grid.id2zone.keys()] base_demand_flexibility_up = raw_demand_flexibility_up[zone_keys] base_demand_flexibility_dn = raw_demand_flexibility_dn[zone_keys] # Create fake files in the expected directory path - exp_path = os.path.join( - Path.home(), "ScenarioData", "raw", str(base_grid.grid_model) - ) - dir_exists_prev = True - if not os.path.isdir(exp_path): - os.makedirs(exp_path) - dir_exists_prev = False - fake_df = pd.DataFrame() - fake_df.to_csv(os.path.join(exp_path, "demand_flexibility_up_Test.csv")) - fake_df.to_csv(os.path.join(exp_path, "demand_flexibility_dn_Test.csv")) + exp_path = f"raw/{base_grid.grid_model}" + + for csv_file in ( + "demand_flexibility_up_Test.csv", + "demand_flexibility_dn_Test.csv", + ): + with data_access.write(exp_path + "/" + csv_file) as f: + pd.DataFrame().to_csv(f) # Specify the change table ct = ChangeTable(base_grid) @@ -406,10 +406,3 @@ def test_flexible_demand_profiles_are_trimmed( transformed_demand_flexibility_dn = tp.get_profile("demand_flexibility_dn") assert base_demand_flexibility_up.equals(transformed_demand_flexibility_up) assert base_demand_flexibility_dn.equals(transformed_demand_flexibility_dn) - - # Delete the created directory and fake data - if dir_exists_prev: - os.remove(os.path.join(exp_path, "demand_flexibility_up_Test.csv")) - os.remove(os.path.join(exp_path, "demand_flexibility_dn_Test.csv")) - else: - shutil.rmtree(os.path.join(Path.home(), "ScenarioData"))