Use temp filesystem for profile test

powersimdata/data_access/data_access.py

@@ -9,10 +9,7 @@
 from fs.path import basename, dirname
 from fs.tempfs import TempFS
 
-from powersimdata.data_access.profile_helper import (
-    get_profile_version_cloud,
-    get_profile_version_local,
-)
+from powersimdata.data_access.profile_helper import get_profile_version
 from powersimdata.data_access.ssh_fs import WrapSSHFS
 from powersimdata.utility import server_setup
 
@@ -186,8 +183,9 @@ def get_profile_version(self, grid_model, kind):
         :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*.
         :return: (*list*) -- available profile version.
         """
-        blob_version = get_profile_version_cloud(grid_model, kind)
-        local_version = get_profile_version_local(grid_model, kind)
+        bfs = fs.open_fs("azblob://besciences@profiles")
+        blob_version = get_profile_version(bfs, grid_model, kind)
+        local_version = get_profile_version(self.local_fs, grid_model, kind)
         return list(set(blob_version + local_version))
 
     def checksum(self, relative_path):

powersimdata/data_access/profile_helper.py

@@ -1,51 +1,20 @@
-import fs
+def get_profile_version(_fs, grid_model, kind):
+    """Returns available raw profile from the given filesystem
 
-from powersimdata.utility import server_setup
-
-
-def _get_profile_version(_fs, kind):
-    """Returns available raw profiles from the given filesystem
     :param fs.base.FS _fs: filesystem instance
+    :param str grid_model: grid model.
     :param str kind: *'demand'*, *'hydro'*, *'solar'*, *'wind'*,
         *'demand_flexibility_up'*, *'demand_flexibility_dn'*,
         *'demand_flexibility_cost_up'*, or *'demand_flexibility_cost_dn'*.
     :return: (*list*) -- available profile version.
     """
+    _fs = _fs.makedirs(f"raw/{grid_model}", recreate=True)
     matching = [f for f in _fs.listdir(".") if kind in f]
 
     # Don't include demand flexibility profiles as possible demand profiles
     if kind == "demand":
-        matching_ = [p for p in matching if "demand_flexibility" not in p]
-        return [f.lstrip(f"{kind}_").rstrip(".csv") for f in matching_]
-    else:
-        return [f.lstrip(f"{kind}_").rstrip(".csv") for f in matching]
-
-
-def get_profile_version_cloud(grid_model, kind):
-    """Returns available raw profile from blob storage.
-
-    :param str grid_model: grid model.
-    :param str kind: *'demand'*, *'hydro'*, *'solar'*, *'wind'*,
-        *'demand_flexibility_up'*, *'demand_flexibility_dn'*,
-        *'demand_flexibility_cost_up'*, or *'demand_flexibility_cost_dn'*.
-    :return: (*list*) -- available profile version.
-    """
-    bfs = fs.open_fs("azblob://besciences@profiles").opendir(f"raw/{grid_model}")
-    return _get_profile_version(bfs, kind)
-
-
-def get_profile_version_local(grid_model, kind):
-    """Returns available raw profile from local file.
-
-    :param str grid_model: grid model.
-    :param str kind: *'demand'*, *'hydro'*, *'solar'*, *'wind'*,
-        *'demand_flexibility_up'*, *'demand_flexibility_dn'*,
-        *'demand_flexibility_cost_up'*, or *'demand_flexibility_cost_dn'*.
-    :return: (*list*) -- available profile version.
-    """
-    profile_dir = fs.path.join(server_setup.LOCAL_DIR, "raw", grid_model)
-    lfs = fs.open_fs(profile_dir, create=True)
-    return _get_profile_version(lfs, kind)
+        matching = [p for p in matching if "demand_flexibility" not in p]
+    return [f.lstrip(f"{kind}_").rstrip(".csv") for f in matching]
 
 
 class ProfileHelper:

powersimdata/data_access/tests/test_profile_helper.py

@@ -1,16 +1,17 @@
 from fs.tempfs import TempFS
 
-from powersimdata.data_access.profile_helper import ProfileHelper, _get_profile_version
+from powersimdata.data_access.profile_helper import ProfileHelper, get_profile_version
 
 
 def test_get_profile_version():
     with TempFS() as tmp_fs:
-        tfs = tmp_fs.makedirs("raw/usa_tamu", recreate=True)
-        tfs.touch("solar_vOct2022.csv")
-        tfs.touch("foo_v1.0.1.csv")
-        v_solar = _get_profile_version(tfs, "solar")
-        v_foo = _get_profile_version(tfs, "foo")
-        v_missing = _get_profile_version(tfs, "missing")
+        grid_model = "usa_tamu"
+        sub_fs = tmp_fs.makedirs(f"raw/{grid_model}", recreate=True)
+        sub_fs.touch("solar_vOct2022.csv")
+        sub_fs.touch("foo_v1.0.1.csv")
+        v_solar = get_profile_version(tmp_fs, grid_model, "solar")
+        v_foo = get_profile_version(tmp_fs, grid_model, "foo")
+        v_missing = get_profile_version(tmp_fs, grid_model, "missing")
         assert "vOct2022" == v_solar[0]
         assert "v1.0.1" == v_foo[0]
         assert [] == v_missing

powersimdata/input/tests/test_change_table.py

@@ -1,12 +1,10 @@
-import os
-import shutil
-from pathlib import Path
-
 import pandas as pd
 import pytest
 
+from powersimdata.data_access.context import Context
 from powersimdata.input.change_table import ChangeTable
 from powersimdata.input.grid import Grid
+from powersimdata.tests.mock_context import MockContext
 
 grid = Grid(["USA"])
 
@@ -512,7 +510,7 @@ def test_remove_bus(ct):
     ct.remove_bus({845})
 
 
-def test_add_demand_flexibility(ct):
+def test_add_demand_flexibility(ct, monkeypatch):
     with pytest.raises(ValueError):
         # Fails because "demand_flexibility_dn", a required key, is not included
         ct.add_demand_flexibility(
@@ -540,15 +538,18 @@ def test_add_demand_flexibility(ct):
             }
         )
 
+    monkeypatch.setattr(Context, "get_data_access", MockContext().get_data_access)
+    data_access = Context.get_data_access()
+
     # Create fake files in the expected directory path
-    exp_path = os.path.join(Path.home(), "ScenarioData", "raw", str(grid.grid_model))
-    dir_exists_prev = True
-    if not os.path.isdir(exp_path):
-        os.makedirs(exp_path)
-        dir_exists_prev = False
-    fake_df = pd.DataFrame()
-    fake_df.to_csv(os.path.join(exp_path, "demand_flexibility_up_Test.csv"))
-    fake_df.to_csv(os.path.join(exp_path, "demand_flexibility_dn_Test.csv"))
+    exp_path = f"raw/{grid.grid_model}"
+
+    for csv_file in (
+        "demand_flexibility_up_Test.csv",
+        "demand_flexibility_dn_Test.csv",
+    ):
+        with data_access.write(exp_path + "/" + csv_file) as f:
+            pd.DataFrame().to_csv(f)
 
     # Add a test instance of demand flexibility to the change table
     ct.add_demand_flexibility(
@@ -566,10 +567,3 @@ def test_add_demand_flexibility(ct):
         },
     }
     assert ct.ct == exp_dict
-
-    # Delete the created directory and fake data
-    if dir_exists_prev:
-        os.remove(os.path.join(exp_path, "demand_flexibility_up_Test.csv"))
-        os.remove(os.path.join(exp_path, "demand_flexibility_dn_Test.csv"))
-    else:
-        shutil.rmtree(os.path.join(Path.home(), "ScenarioData"))

powersimdata/input/tests/test_transform_profile.py

@@ -1,17 +1,16 @@
-import os
-import shutil
-from pathlib import Path
 from unittest.mock import patch
 
 import numpy as np
 import pandas as pd
 import pytest
 from numpy.testing import assert_almost_equal
 
+from powersimdata.data_access.context import Context
 from powersimdata.input.change_table import ChangeTable
 from powersimdata.input.grid import Grid
 from powersimdata.input.transform_grid import TransformGrid
 from powersimdata.input.transform_profile import TransformProfile
+from powersimdata.tests.mock_context import MockContext
 from powersimdata.tests.mock_input_data import MockInputData
 
 interconnect = ["Western"]
@@ -366,24 +365,25 @@ def test_new_hydro_profile(base_grid, raw_hydro):
 
 
 def test_flexible_demand_profiles_are_trimmed(
-    base_grid, raw_demand_flexibility_up, raw_demand_flexibility_dn
+    base_grid, raw_demand_flexibility_up, raw_demand_flexibility_dn, monkeypatch
 ):
+    monkeypatch.setattr(Context, "get_data_access", MockContext().get_data_access)
+    data_access = Context.get_data_access()
+
     # Specify the fake demand flexibility profiles from MockInputData
     zone_keys = [f"zone.{z}" for z in base_grid.id2zone.keys()]
     base_demand_flexibility_up = raw_demand_flexibility_up[zone_keys]
     base_demand_flexibility_dn = raw_demand_flexibility_dn[zone_keys]
 
     # Create fake files in the expected directory path
-    exp_path = os.path.join(
-        Path.home(), "ScenarioData", "raw", str(base_grid.grid_model)
-    )
-    dir_exists_prev = True
-    if not os.path.isdir(exp_path):
-        os.makedirs(exp_path)
-        dir_exists_prev = False
-    fake_df = pd.DataFrame()
-    fake_df.to_csv(os.path.join(exp_path, "demand_flexibility_up_Test.csv"))
-    fake_df.to_csv(os.path.join(exp_path, "demand_flexibility_dn_Test.csv"))
+    exp_path = f"raw/{base_grid.grid_model}"
+
+    for csv_file in (
+        "demand_flexibility_up_Test.csv",
+        "demand_flexibility_dn_Test.csv",
+    ):
+        with data_access.write(exp_path + "/" + csv_file) as f:
+            pd.DataFrame().to_csv(f)
 
     # Specify the change table
     ct = ChangeTable(base_grid)
@@ -406,10 +406,3 @@ def test_flexible_demand_profiles_are_trimmed(
     transformed_demand_flexibility_dn = tp.get_profile("demand_flexibility_dn")
     assert base_demand_flexibility_up.equals(transformed_demand_flexibility_up)
     assert base_demand_flexibility_dn.equals(transformed_demand_flexibility_dn)
-
-    # Delete the created directory and fake data
-    if dir_exists_prev:
-        os.remove(os.path.join(exp_path, "demand_flexibility_up_Test.csv"))
-        os.remove(os.path.join(exp_path, "demand_flexibility_dn_Test.csv"))
-    else:
-        shutil.rmtree(os.path.join(Path.home(), "ScenarioData"))