Merge pull request #572 from Breakthrough-Energy/develop

chore: merge develop into master for v0.4.5 release

.github/ISSUE_TEMPLATE/bug_report.md

@@ -3,7 +3,7 @@ name: Bug report
 about: Create a report to help us improve
 title: Bug report
 labels: bug
-assignees: ahurli, BainanXia, danielolsen, jon-hagg, rouille
+assignees: BainanXia, danielolsen, jon-hagg, rouille
 
 ---
 

.github/ISSUE_TEMPLATE/feature_request.md

@@ -3,7 +3,7 @@ name: Feature request
 about: Suggest an idea for this project
 title: Feature request
 labels: feature request
-assignees: ahurli, BainanXia, danielolsen, jon-hagg, rouille
+assignees: BainanXia, danielolsen, jon-hagg, rouille
 
 ---
 

.github/workflows/coverage.yml

@@ -18,14 +18,8 @@ jobs:
         with:
           python-version: 3.9
 
-      - name: Start postgres container
-        run: |
-          docker-compose -f stack.yml up -d
-          while ! nc -z localhost 5432; do sleep 1; done;
-        working-directory: powersimdata/data_access
-
       - run: python -m pip install --upgrade pip tox
-      - run: tox -e pytest-ci -- --cov-report=xml
+      - run: tox -e pytest-local -- --cov-report=xml
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1

.github/workflows/external.yml

@@ -0,0 +1,25 @@
+name: Trigger external workflows
+
+on:
+  push:
+    branches:
+      - develop
+
+jobs:
+  run-workflows:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Build and publish docs website
+        uses: Breakthrough-Energy/actions/workflow-trigger@main
+        with:
+          repo: docs
+          branch: master
+          workflow_id: 2386877
+          token: ${{ secrets.CI_TOKEN_CLONE_REPO }}
+
+      - name: Build and publish docker image
+        uses: Breakthrough-Energy/actions/workflow-trigger@main
+        with:
+          repo: plug
+          workflow_id: 12413223
+          token: ${{ secrets.CI_TOKEN_CLONE_REPO }}

.github/workflows/test.yml

@@ -20,11 +20,5 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Start postgres container
-        run: |
-          docker-compose -f stack.yml up -d
-          while ! nc -z localhost 5432; do sleep 1; done;
-        working-directory: powersimdata/data_access
-
       - run: python -m pip install --upgrade pip tox
-      - run: tox -e pytest-ci
+      - run: tox -e pytest-local

Pipfile

@@ -14,7 +14,6 @@ networkx = "~=2.5"
 numpy = "~=1.20"
 pandas = "~=1.2"
 paramiko = "==2.7.2"
-psycopg2 = "~=2.8.5"
 scipy = "~=1.5"
 tqdm = "==4.29.1"
 requests = "~=2.25"

README.md

@@ -1,6 +1,6 @@
-
 ![logo](https://raw.githubusercontent.com/Breakthrough-Energy/docs/master/source/_static/img/BE_Sciences_RGB_Horizontal_Color.svg)
 
+[![PyPI](https://img.shields.io/pypi/v/powersimdata?color=purple)](https://pypi.org/project/powersimdata/)
 [![codecov](https://codecov.io/gh/Breakthrough-Energy/PowerSimData/branch/develop/graph/badge.svg?token=5A20TCV5XL)](https://codecov.io/gh/Breakthrough-Energy/PowerSimData)
 [![made-with-python](https://img.shields.io/badge/Made%20with-Python-1f425f.svg)](https://www.python.org/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
@@ -32,7 +32,8 @@ A detailed tutorial can be found on our [docs].
 
 
 ## Where to get it
-For now, only the source code is available. Clone or Fork the code here on GitHub.
+* Clone or Fork the source code on [GitHub](https://github.com/Breakthrough-Energy/PowerSimData)
+* Get latest release from PyPi: `pip install powersimdata`
 
 
 ## Dependencies
@@ -66,7 +67,8 @@ pip install -r requirements.txt
 
 
 ## Documentation
-The official documentation can be found [here][docs].
+[Code documentation][docstrings] in form of Python docstrings along with an overview of
+the [package][docs] are available on our [website][website].
 
 
 ## Communication Channels
@@ -81,4 +83,6 @@ Guide](https://breakthrough-energy.github.io/docs/dev/contribution_guide.html).
 
 
 
-[docs]: https://breakthrough-energy.github.io/docs/index.html
+[docs]: https://breakthrough-energy.github.io/docs/powersimdata/index.html
+[docstrings]: https://breakthrough-energy.github.io/docs/powersimdata.html
+[website]: https://breakthrough-energy.github.io/docs/

powersimdata/data_access/data_access.py

@@ -1,4 +1,3 @@
-import operator
 import posixpath
 import time
 from subprocess import Popen
@@ -84,11 +83,11 @@ def _check_file_exists(self, path, should_exist=True):
         :param bool should_exist: whether the file is expected to exist
         :raises OSError: if the expected condition is not met
         """
-        result = self.fs.exists(path)
-        compare = operator.ne if should_exist else operator.eq
-        if compare(result, True):
-            msg = "not found" if should_exist else "already exists"
-            raise OSError(f"{path} {msg} on {self.description}")
+        exists = self.fs.exists(path)
+        if should_exist and not exists:
+            raise OSError(f"{path} not found on {self.description}")
+        if not should_exist and exists:
+            raise OSError(f"{path} already exists on {self.description}")
 
     def makedir(self, path):
         """Create path in current environment
@@ -128,7 +127,9 @@ def get_profile_version(self, grid_model, kind):
         :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*.
         :return: (*list*) -- available profile version.
         """
-        return ProfileHelper.get_profile_version_cloud(grid_model, kind)
+        blob_version = ProfileHelper.get_profile_version_cloud(grid_model, kind)
+        local_version = ProfileHelper.get_profile_version_local(grid_model, kind)
+        return list(set(blob_version + local_version))
 
 
 class LocalDataAccess(DataAccess):
@@ -170,17 +171,6 @@ def move_to(self, file_name, to_dir, change_name_to=None):
         self.makedir(to_dir)
         self.fs.move(file_name, dest)
 
-    def get_profile_version(self, grid_model, kind):
-        """Returns available raw profile from blob storage or local disk
-
-        :param str grid_model: grid model.
-        :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*.
-        :return: (*list*) -- available profile version.
-        """
-        blob_version = super().get_profile_version(grid_model, kind)
-        local_version = ProfileHelper.get_profile_version_local(grid_model, kind)
-        return list(set(blob_version + local_version))
-
 
 class SSHDataAccess(DataAccess):
     """Interface to a remote data store, accessed via SSH."""

powersimdata/data_access/launcher.py

@@ -46,7 +46,10 @@ class Launcher:
 
     def __init__(self, scenario):
         self.scenario = scenario
-        self.scenario_id = scenario.scenario_id
+
+    @property
+    def scenario_id(self):
+        return self.scenario.scenario_id
 
     def _launch(self, threads=None, solver=None, extract_data=True):
         """Launches simulation on target environment
@@ -93,7 +96,7 @@ def _run_script(self, script, extra_args=None):
         if extra_args is None:
             extra_args = []
 
-        engine = self.scenario._scenario_info["engine"]
+        engine = self.scenario.state._scenario_info["engine"]
         path_to_package = posixpath.join(server_setup.MODEL_DIR, engine)
         folder = "pyreise" if engine == "REISE" else "pyreisejl"
 
@@ -108,7 +111,7 @@ def _run_script(self, script, extra_args=None):
         ]
         cmd_io_redirect = ["</dev/null >/dev/null 2>&1 &"]
         cmd = cmd_pythonpath + cmd_pythoncall + extra_args + cmd_io_redirect
-        process = self.scenario._data_access.execute_command_async(cmd)
+        process = self.scenario.data_access.execute_command_async(cmd)
         print("PID: %s" % process.pid)
         return process
 

powersimdata/data_access/profile_helper.py

@@ -1,6 +1,6 @@
-import json
 import os
 
+import fs
 import requests
 from tqdm.auto import tqdm
 
@@ -87,10 +87,7 @@ def get_profile_version_local(grid_model, kind):
         :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*.
         :return: (*list*) -- available profile version.
         """
-
-        version_file = os.path.join(server_setup.LOCAL_DIR, "version.json")
-        if not os.path.exists(version_file):
-            return []
-        with open(version_file) as f:
-            version = json.load(f)
-            return ProfileHelper.parse_version(grid_model, kind, version)
+        profile_dir = fs.path.join(server_setup.LOCAL_DIR, "raw", grid_model)
+        lfs = fs.open_fs(profile_dir)
+        matching = [f for f in lfs.listdir(".") if kind in f]
+        return [f.lstrip(f"{kind}_").rstrip(".csv") for f in matching]

powersimdata/data_access/tests/test_data_access.py

@@ -42,7 +42,7 @@ def test_tmp_folder(ssh_data_access):
 @pytest.mark.integration
 @pytest.mark.ssh
 def test_setup_server_connection(ssh_data_access):
-    _, stdout, _ = ssh_data_access.ssh.exec_command("whoami")
+    _, stdout, _ = ssh_data_access.fs.exec_command("whoami")
     assert stdout.read().decode("utf-8").strip() == server_setup.get_server_user()
 
 

powersimdata/design/investment/create_mapping_files.py

@@ -4,6 +4,7 @@
 
 from powersimdata.design.investment import const
 from powersimdata.input.grid import Grid
+from powersimdata.utility.distance import haversine
 from powersimdata.utility.helpers import _check_import
 
 
@@ -67,6 +68,29 @@ def _find_nearest(series, polygons, search_dist):
     points_in_regions = gpd.sjoin(left_df=left_df, right_df=right_df, op="intersects")
     points_in_regions["dist"] = 0
 
+    # Since polygons may overlap, there can be duplicated buses that we want to filter
+    duplicated = points_in_regions.loc[points_in_regions.index.duplicated(keep=False)]
+    to_drop = set()
+    for bus in set(duplicated["bus_id"]):
+        entries = duplicated.query("bus_id == @bus")
+        coords = entries["geometry"].iloc[0].coords[0]  # First duped entry, only point
+        regions = set(entries["name_abbr"])  # noqa: F841
+        candidates = points_in_regions.query(
+            "index not in @duplicated.index and name_abbr in @regions"
+        )
+        neighbor = candidates.apply(
+            lambda x: haversine((x.geometry.x, x.geometry.y), coords), axis=1
+        ).idxmin()
+        closest_region = candidates.loc[neighbor, "name_abbr"]  # noqa: F841
+        # There may be more than two overlapping geometries, capture all but the closest
+        drop_regions = set(entries.query("name_abbr != @closest_region")["name_abbr"])
+        # Since indices are duplicated, we need to drop via two-column tuples
+        to_drop |= {(bus, d) for d in drop_regions}
+
+    points_in_regions = points_in_regions.loc[
+        ~points_in_regions.set_index(["bus_id", "name_abbr"]).index.isin(to_drop)
+    ]
+
     # Find closest Polygons, for points that don't fall within any
     missing_indices = set(left_df.index) - set(points_in_regions.index)
     points_not_in_regions = left_df.loc[missing_indices]
@@ -184,7 +208,7 @@ def write_bus_neem_map(base_grid):
         raise TypeError("base_grid must be a Grid instance")
     df_pts_bus = bus_to_neem_reg(base_grid.bus)
     df_pts_bus.sort_index(inplace=True)
-    os.makedirs(const.bus_neem_regions_path, exist_ok=True)
+    os.makedirs(os.path.dirname(const.bus_neem_regions_path), exist_ok=True)
     df_pts_bus.to_csv(const.bus_neem_regions_path)
 
 
@@ -198,7 +222,7 @@ def write_bus_reeds_map(base_grid):
         raise TypeError("base_grid must be a Grid instance")
     df_pts_bus = bus_to_reeds_reg(base_grid.bus)
     df_pts_bus.sort_index(inplace=True)
-    os.makedirs(const.bus_reeds_regions_path, exist_ok=True)
+    os.makedirs(os.path.dirname(const.bus_reeds_regions_path), exist_ok=True)
     df_pts_bus.to_csv(const.bus_reeds_regions_path)
 
 

powersimdata/design/investment/data/buses_NEEMregion.csv

@@ -12358,7 +12358,6 @@ bus_id,name_abbr,dist,lat,lon
 12357,PJM E,0.0,40.0887,-75.6346
 12358,PJM E,0.0,40.0887,-75.6346
 12359,PJM ROM,0.0,39.9515,-75.826
-12359,PJM E,0.0,39.9515,-75.826
 12360,PJM E,0.0,39.8624,-75.8187
 12361,PJM E,0.0,39.9648,-75.7912
 12362,PJM E,0.0,39.9648,-75.7912
@@ -12458,10 +12457,8 @@ bus_id,name_abbr,dist,lat,lon
 12456,PJM E,0.0,39.8364,-75.8027
 12457,PJM E,0.0,39.8364,-75.8027
 12458,PJM E,0.0,39.8364,-75.8027
-12459,PJM E,0.0,39.9024,-75.8392
 12459,PJM ROM,0.0,39.9024,-75.8392
 12460,PJM ROM,0.0,39.9024,-75.8392
-12460,PJM E,0.0,39.9024,-75.8392
 12461,PJM ROM,0.0,41.1524,-77.3123
 12462,PJM ROM,0.0,41.1524,-77.3123
 12463,PJM ROM,0.0,41.2429,-77.2389
@@ -36057,7 +36054,6 @@ bus_id,name_abbr,dist,lat,lon
 36053,PJM ROR,0.0,39.3891,-83.3393
 36054,PJM ROR,0.0,39.2689,-83.432
 36055,NonRTO Midwest,0.0,39.6566,-83.5378
-36055,PJM ROR,0.0,39.6566,-83.5378
 36056,PJM ROR,0.0,39.7052,-83.3107
 36057,PJM ROR,0.0,39.6489,-83.3377
 36058,NonRTO Midwest,0.003978078694527869,39.479,-83.5037
@@ -42496,7 +42492,6 @@ bus_id,name_abbr,dist,lat,lon
 42491,MISO IN,0.0,38.0288,-87.5769
 42492,MISO IN,0.0,38.0288,-87.5769
 42493,MISO IN,0.0,38.0811,-87.553
-42494,NonRTO Midwest,0.0,37.9335,-87.5617
 42494,MISO IN,0.0,37.9335,-87.5617
 42495,MISO IN,0.0,38.0137,-87.5274
 42496,NonRTO Midwest,0.0,37.9065,-87.5288
@@ -43089,7 +43084,6 @@ bus_id,name_abbr,dist,lat,lon
 43083,PJM ROR,0.0,42.2138,-88.7904
 43084,PJM ROR,0.0,42.3539,-88.6689
 43085,MISO WUMS,0.0,42.4947,-88.6446
-43085,PJM ROR,0.0,42.4947,-88.6446
 43086,PJM ROR,0.0,42.4093,-88.6218
 43087,PJM ROR,0.0,42.4093,-88.6218
 43088,PJM ROR,0.0,42.3461,-88.9741
@@ -47993,9 +47987,7 @@ bus_id,name_abbr,dist,lat,lon
 47986,MISO WUMS,0.0,42.598,-89.1008
 47987,MISO WUMS,0.0,42.598,-89.1008
 47988,MISO WUMS,0.0,42.5008,-89.1299
-47988,PJM ROR,0.0,42.5008,-89.1299
 47989,MISO WUMS,0.0,42.5008,-89.1299
-47989,PJM ROR,0.0,42.5008,-89.1299
 47990,MISO WUMS,0.0,42.5598,-88.7798
 47991,PJM ROR,0.0,42.4831,-88.8725
 47992,PJM ROR,0.0,42.4831,-88.8725
@@ -48141,7 +48133,6 @@ bus_id,name_abbr,dist,lat,lon
 48132,MISO WUMS,0.0,42.6161,-88.3851
 48133,MISO WUMS,0.0,42.6161,-88.3851
 48134,MISO WUMS,0.0,42.496,-88.5134
-48134,PJM ROR,0.0,42.496,-88.5134
 48135,MISO W,0.0,45.9493,-91.938
 48136,MISO WUMS,0.0,43.2861,-88.2452
 48137,MISO WUMS,0.0,43.2861,-88.2452