refactor with pathlib

.github/workflows/python-package.yml

@@ -31,14 +31,14 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        py-version: ['3.7', '3.8', '3.9', '3.10']
+        py-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     # general Python setup
     - name: Set up Python ${{ matrix.py-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.py-version }}
 

lciafmt/__init__.py

@@ -225,7 +225,7 @@ def generate_endpoints(file: Union[str, pd.DataFrame],
     if isinstance(file, pd.DataFrame):
         endpoints = file
     else:
-        endpoints = pd.read_csv(util.datapath+"/"+file+".csv")
+        endpoints = pd.read_csv(util.datapath / f'{file}.csv')
     if matching_fields is None:
         matching_fields = ['Indicator']
     method = ep.apply_endpoints(endpoints, matching_fields,

lciafmt/ipcc.py

@@ -17,7 +17,7 @@ def get() -> pd.DataFrame:
     log.info("get method IPCC")
 
     filename = 'IPCC_GWP_values.csv'
-    df = pd.read_csv(datapath + filename)
+    df = pd.read_csv(datapath / filename)
     df['Indicator'] = df['AR'] + '-' + df['Parameter'].str.replace('GWP', '')
     df['Method'] = 'IPCC'
     df['Indicator unit'] = 'kg CO2 eq'

lciafmt/recipe.py

@@ -34,7 +34,7 @@
         'sea water': 'water/sea water',
         'Sea water': 'water/sea water',
         'marine water': 'water/sea water'}
-flowables_split = pd.read_csv(datapath + 'ReCiPe2016_split.csv')
+flowables_split = pd.read_csv(datapath / 'ReCiPe2016_split.csv')
 
 
 def get(add_factors_for_missing_contexts=True, endpoint=True,
@@ -181,7 +181,7 @@ def _read_endpoints(file: str) -> pd.DataFrame:
     endpoint.loc[endpoint['EndpointUnit'].str.contains('species', case=False), 'EndpointUnit'] = 'species-year'
     endpoint.loc[endpoint['EndpointUnit'].str.contains('USD', case=False), 'EndpointUnit'] = 'USD2013'
 
-    endpoint_map = pd.read_csv(datapath + 'ReCiPe2016_endpoint_to_midpoint.csv')
+    endpoint_map = pd.read_csv(datapath / 'ReCiPe2016_endpoint_to_midpoint.csv')
     endpoint = endpoint.merge(endpoint_map, how="left", on='EndpointIndicator')
 
     # split into two dataframes

lciafmt/traci.py

@@ -18,8 +18,8 @@
     datapath
 
 
-flowables_replace = pd.read_csv(datapath+'TRACI_2.1_replacement.csv')
-flowables_split = pd.read_csv(datapath+'TRACI_2.1_split.csv')
+flowables_replace = pd.read_csv(datapath / 'TRACI_2.1_replacement.csv')
+flowables_split = pd.read_csv(datapath / 'TRACI_2.1_split.csv')
 
 
 def get(add_factors_for_missing_contexts=True, file=None,

lciafmt/util.py

@@ -5,8 +5,6 @@
 This module contains common functions for processing LCIA methods
 """
 
-import os
-from os.path import join
 import sys
 
 import lciafmt
@@ -15,16 +13,18 @@
 import numpy as np
 import yaml
 import pkg_resources
+from pathlib import Path
 from esupy.processed_data_mgmt import Paths, FileMeta, load_preprocessed_output,\
-    write_df_to_file, write_metadata_to_file, download_from_remote
+    write_df_to_file, write_metadata_to_file, download_from_remote, \
+    mkdir_if_missing
 from esupy.util import get_git_hash
 from fedelemflowlist.globals import flow_list_specs
 
 
 # set version number of package, needs to be updated with setup.py
 pkg_version_number = '1.0.3'
-modulepath = os.path.dirname(os.path.realpath(__file__)).replace('\\', '/')
-datapath = modulepath + '/data/'
+MODULEPATH = Path(__file__).resolve().parent
+datapath = MODULEPATH / 'data'
 
 log.basicConfig(level=log.INFO, format='%(asctime)s %(levelname)-8s %(message)s',
                 datefmt='%Y-%m-%d %H:%M:%S', stream=sys.stdout)
@@ -33,11 +33,11 @@
 write_format = "parquet"
 
 paths = Paths()
-paths.local_path = os.path.realpath(paths.local_path + "/lciafmt")
-outputpath = paths.local_path
+paths.local_path = paths.local_path / 'lciafmt'
+OUTPUTPATH = paths.local_path
 
 pkg = pkg_resources.get_distribution('lciafmt')
-git_hash = get_git_hash()
+GIT_HASH = get_git_hash()
 
 method_metadata = {
     'Name': '',
@@ -61,7 +61,7 @@ def set_lcia_method_meta(method_id):
     lcia_method_meta.tool = pkg.project_name
     lcia_method_meta.tool_version = pkg_version_number
     lcia_method_meta.ext = write_format
-    lcia_method_meta.git_hash = git_hash
+    lcia_method_meta.git_hash = GIT_HASH
     return lcia_method_meta
 
 
@@ -105,7 +105,7 @@ def aggregate_factors_for_primary_contexts(df) -> pd.DataFrame:
     indices = df['Context'].str.find('/')
     ignored_list = df['Indicator'].isin(ignored_categories)
     i = 0
-    for k in ignored_list.iteritems():
+    for k in ignored_list.items():
         if k[1]:
             indices.update(pd.Series([-1], index=[i]))
         i = i + 1
@@ -141,7 +141,7 @@ def aggregate_factors_for_primary_contexts(df) -> pd.DataFrame:
 
 def get_modification(source, name) -> pd.DataFrame:
     """Return a dataframe of modified CFs based on csv."""
-    modified_factors = pd.read_csv(datapath+"/"+source+"_"+name+".csv")
+    modified_factors = pd.read_csv(datapath / f'{source}_{name}.csv')
     return modified_factors
 
 
@@ -171,7 +171,7 @@ def check_as_class(method_id):
 
 
 def generate_method_description(name: str) -> str:
-    with open(join(datapath, "description.yaml")) as f:
+    with open(datapath / "description.yaml") as f:
         generic = yaml.safe_load(f)
     method_description = generic['description']
     method = check_as_class(name)
@@ -189,7 +189,7 @@ def generate_method_description(name: str) -> str:
             detailed_meta = '\n\n' + method_meta['methods'][name]
             method_description += detailed_meta
         except KeyError:
-            log.debug('%s not found in methods.json', name)
+            log.debug(f'{name} not found in methods.json')
     # Replace tagged fields
     if 'version' in method_meta:
         version = ' (v' + method_meta['version'] + ')'
@@ -226,14 +226,14 @@ def compile_metadata(method_id):
 def store_method(df, method_id, name=''):
     """Save the method as a dataframe to parquet file."""
     meta = set_lcia_method_meta(method_id)
-    method_path = outputpath + '/' + meta.category
+    method_path = OUTPUTPATH / meta.category
     if name != '':
         meta.name_data = name
-    elif meta.name_data == "":
+    elif meta.name_data == '':
         meta.name_data = df['Method'][0]
     meta.tool_meta = compile_metadata(method_id)
     try:
-        log.info('saving ' + meta.name_data + ' to ' + method_path)
+        log.info(f'saving {meta.name_data} to {method_path}')
         write_df_to_file(df, paths, meta)
         write_metadata_to_file(paths, meta)
     except:
@@ -244,11 +244,11 @@ def read_method(method_id):
     """Return the method stored in output."""
     meta = set_lcia_method_meta(method_id)
     method = load_preprocessed_output(meta, paths)
-    method_path = outputpath + '/' + meta.category
+    method_path = OUTPUTPATH / meta.category
     if method is None:
-        log.info(meta.name_data + ' not found in ' + method_path)
+        log.info(f'{meta.name_data} not found in {method_path}')
     else:
-        log.info('loaded ' + meta.name_data + ' from ' + method_path)
+        log.info(f'loaded {meta.name_data} from {method_path}')
     return method
 
 
@@ -274,11 +274,10 @@ def save_json(method_id, mapped_data, method=None, name=''):
     if method is not None:
         filename = method.replace('/', '_')
         mapped_data = mapped_data[mapped_data['Method'] == method]
-    path = outputpath+'/'+meta.category
-    os.makedirs(outputpath, exist_ok=True)
-    json_pack = path + '/' + filename + "_json_v" + meta.tool_version + ".zip"
-    if os.path.exists(json_pack):
-        os.remove(json_pack)
+    path = OUTPUTPATH / meta.category
+    mkdir_if_missing(OUTPUTPATH)
+    json_pack = path / f'{filename}_json_v{meta.tool_version}.zip'
+    json_pack.unlink(missing_ok=True)
     lciafmt.to_jsonld(mapped_data, json_pack)
 
 
@@ -300,8 +299,8 @@ def compare_to_remote(local_df, method_id):
     df_diff = df.query('`Characterization Factor` '
                        '!= `Characterization Factor_remote`')
     if len(df_diff) > 0:
-        path = os.path.join(paths.local_path, 'diff')
-        os.makedirs(path, exist_ok=True)
+        path = paths.local_path / 'diff'
+        mkdir_if_missing(path)
         log.info(f'Saving differences found in {method_id.name} '
                  f'versus remote to {path}')
         df_diff.to_csv(f'{path}/{method_id.name}_diff.csv', index=False)

setup.py

@@ -7,6 +7,7 @@
     package_dir={'lciafmt': 'lciafmt'},
     package_data={'lciafmt': ["data/*.*"]},
     include_package_data=True,
+    python_requires=">=3.7",
     install_requires=["fedelemflowlist @ git+https://github.com/USEPA/Federal-LCA-Commons-Elementary-Flow-List.git@develop#egg=fedelemflowlist",
                       "esupy @ git+https://github.com/USEPA/esupy.git@develop#egg=esupy",
                       "olca-ipc>=0.0.12",