diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8f5167e..95e6313 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -31,14 +31,14 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - py-version: ['3.7', '3.8', '3.9', '3.10'] + py-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 # general Python setup - name: Set up Python ${{ matrix.py-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.py-version }} diff --git a/lciafmt/__init__.py b/lciafmt/__init__.py index 05a8cc8..2ae6a68 100644 --- a/lciafmt/__init__.py +++ b/lciafmt/__init__.py @@ -225,7 +225,7 @@ def generate_endpoints(file: Union[str, pd.DataFrame], if isinstance(file, pd.DataFrame): endpoints = file else: - endpoints = pd.read_csv(util.datapath+"/"+file+".csv") + endpoints = pd.read_csv(util.datapath / f'{file}.csv') if matching_fields is None: matching_fields = ['Indicator'] method = ep.apply_endpoints(endpoints, matching_fields, diff --git a/lciafmt/ipcc.py b/lciafmt/ipcc.py index a8cf365..dfc1271 100644 --- a/lciafmt/ipcc.py +++ b/lciafmt/ipcc.py @@ -17,7 +17,7 @@ def get() -> pd.DataFrame: log.info("get method IPCC") filename = 'IPCC_GWP_values.csv' - df = pd.read_csv(datapath + filename) + df = pd.read_csv(datapath / filename) df['Indicator'] = df['AR'] + '-' + df['Parameter'].str.replace('GWP', '') df['Method'] = 'IPCC' df['Indicator unit'] = 'kg CO2 eq' diff --git a/lciafmt/recipe.py b/lciafmt/recipe.py index babffbd..87b6c3a 100644 --- a/lciafmt/recipe.py +++ b/lciafmt/recipe.py @@ -34,7 +34,7 @@ 'sea water': 'water/sea water', 'Sea water': 'water/sea water', 'marine water': 'water/sea water'} -flowables_split = pd.read_csv(datapath + 'ReCiPe2016_split.csv') +flowables_split = pd.read_csv(datapath / 'ReCiPe2016_split.csv') def get(add_factors_for_missing_contexts=True, endpoint=True, @@ -181,7 +181,7 @@ def _read_endpoints(file: str) -> pd.DataFrame: endpoint.loc[endpoint['EndpointUnit'].str.contains('species', case=False), 'EndpointUnit'] = 'species-year' endpoint.loc[endpoint['EndpointUnit'].str.contains('USD', case=False), 'EndpointUnit'] = 'USD2013' - endpoint_map = pd.read_csv(datapath + 'ReCiPe2016_endpoint_to_midpoint.csv') + endpoint_map = pd.read_csv(datapath / 'ReCiPe2016_endpoint_to_midpoint.csv') endpoint = endpoint.merge(endpoint_map, how="left", on='EndpointIndicator') # split into two dataframes diff --git a/lciafmt/traci.py b/lciafmt/traci.py index 36b4af4..a003349 100644 --- a/lciafmt/traci.py +++ b/lciafmt/traci.py @@ -18,8 +18,8 @@ datapath -flowables_replace = pd.read_csv(datapath+'TRACI_2.1_replacement.csv') -flowables_split = pd.read_csv(datapath+'TRACI_2.1_split.csv') +flowables_replace = pd.read_csv(datapath / 'TRACI_2.1_replacement.csv') +flowables_split = pd.read_csv(datapath / 'TRACI_2.1_split.csv') def get(add_factors_for_missing_contexts=True, file=None, diff --git a/lciafmt/util.py b/lciafmt/util.py index 690ec9c..d25d935 100644 --- a/lciafmt/util.py +++ b/lciafmt/util.py @@ -5,8 +5,6 @@ This module contains common functions for processing LCIA methods """ -import os -from os.path import join import sys import lciafmt @@ -15,16 +13,18 @@ import numpy as np import yaml import pkg_resources +from pathlib import Path from esupy.processed_data_mgmt import Paths, FileMeta, load_preprocessed_output,\ - write_df_to_file, write_metadata_to_file, download_from_remote + write_df_to_file, write_metadata_to_file, download_from_remote, \ + mkdir_if_missing from esupy.util import get_git_hash from fedelemflowlist.globals import flow_list_specs # set version number of package, needs to be updated with setup.py pkg_version_number = '1.0.3' -modulepath = os.path.dirname(os.path.realpath(__file__)).replace('\\', '/') -datapath = modulepath + '/data/' +MODULEPATH = Path(__file__).resolve().parent +datapath = MODULEPATH / 'data' log.basicConfig(level=log.INFO, format='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', stream=sys.stdout) @@ -33,11 +33,11 @@ write_format = "parquet" paths = Paths() -paths.local_path = os.path.realpath(paths.local_path + "/lciafmt") -outputpath = paths.local_path +paths.local_path = paths.local_path / 'lciafmt' +OUTPUTPATH = paths.local_path pkg = pkg_resources.get_distribution('lciafmt') -git_hash = get_git_hash() +GIT_HASH = get_git_hash() method_metadata = { 'Name': '', @@ -61,7 +61,7 @@ def set_lcia_method_meta(method_id): lcia_method_meta.tool = pkg.project_name lcia_method_meta.tool_version = pkg_version_number lcia_method_meta.ext = write_format - lcia_method_meta.git_hash = git_hash + lcia_method_meta.git_hash = GIT_HASH return lcia_method_meta @@ -105,7 +105,7 @@ def aggregate_factors_for_primary_contexts(df) -> pd.DataFrame: indices = df['Context'].str.find('/') ignored_list = df['Indicator'].isin(ignored_categories) i = 0 - for k in ignored_list.iteritems(): + for k in ignored_list.items(): if k[1]: indices.update(pd.Series([-1], index=[i])) i = i + 1 @@ -141,7 +141,7 @@ def aggregate_factors_for_primary_contexts(df) -> pd.DataFrame: def get_modification(source, name) -> pd.DataFrame: """Return a dataframe of modified CFs based on csv.""" - modified_factors = pd.read_csv(datapath+"/"+source+"_"+name+".csv") + modified_factors = pd.read_csv(datapath / f'{source}_{name}.csv') return modified_factors @@ -171,7 +171,7 @@ def check_as_class(method_id): def generate_method_description(name: str) -> str: - with open(join(datapath, "description.yaml")) as f: + with open(datapath / "description.yaml") as f: generic = yaml.safe_load(f) method_description = generic['description'] method = check_as_class(name) @@ -189,7 +189,7 @@ def generate_method_description(name: str) -> str: detailed_meta = '\n\n' + method_meta['methods'][name] method_description += detailed_meta except KeyError: - log.debug('%s not found in methods.json', name) + log.debug(f'{name} not found in methods.json') # Replace tagged fields if 'version' in method_meta: version = ' (v' + method_meta['version'] + ')' @@ -226,14 +226,14 @@ def compile_metadata(method_id): def store_method(df, method_id, name=''): """Save the method as a dataframe to parquet file.""" meta = set_lcia_method_meta(method_id) - method_path = outputpath + '/' + meta.category + method_path = OUTPUTPATH / meta.category if name != '': meta.name_data = name - elif meta.name_data == "": + elif meta.name_data == '': meta.name_data = df['Method'][0] meta.tool_meta = compile_metadata(method_id) try: - log.info('saving ' + meta.name_data + ' to ' + method_path) + log.info(f'saving {meta.name_data} to {method_path}') write_df_to_file(df, paths, meta) write_metadata_to_file(paths, meta) except: @@ -244,11 +244,11 @@ def read_method(method_id): """Return the method stored in output.""" meta = set_lcia_method_meta(method_id) method = load_preprocessed_output(meta, paths) - method_path = outputpath + '/' + meta.category + method_path = OUTPUTPATH / meta.category if method is None: - log.info(meta.name_data + ' not found in ' + method_path) + log.info(f'{meta.name_data} not found in {method_path}') else: - log.info('loaded ' + meta.name_data + ' from ' + method_path) + log.info(f'loaded {meta.name_data} from {method_path}') return method @@ -274,11 +274,10 @@ def save_json(method_id, mapped_data, method=None, name=''): if method is not None: filename = method.replace('/', '_') mapped_data = mapped_data[mapped_data['Method'] == method] - path = outputpath+'/'+meta.category - os.makedirs(outputpath, exist_ok=True) - json_pack = path + '/' + filename + "_json_v" + meta.tool_version + ".zip" - if os.path.exists(json_pack): - os.remove(json_pack) + path = OUTPUTPATH / meta.category + mkdir_if_missing(OUTPUTPATH) + json_pack = path / f'{filename}_json_v{meta.tool_version}.zip' + json_pack.unlink(missing_ok=True) lciafmt.to_jsonld(mapped_data, json_pack) @@ -300,8 +299,8 @@ def compare_to_remote(local_df, method_id): df_diff = df.query('`Characterization Factor` ' '!= `Characterization Factor_remote`') if len(df_diff) > 0: - path = os.path.join(paths.local_path, 'diff') - os.makedirs(path, exist_ok=True) + path = paths.local_path / 'diff' + mkdir_if_missing(path) log.info(f'Saving differences found in {method_id.name} ' f'versus remote to {path}') df_diff.to_csv(f'{path}/{method_id.name}_diff.csv', index=False) diff --git a/setup.py b/setup.py index f44a94e..3f18b41 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ package_dir={'lciafmt': 'lciafmt'}, package_data={'lciafmt': ["data/*.*"]}, include_package_data=True, + python_requires=">=3.7", install_requires=["fedelemflowlist @ git+https://github.com/USEPA/Federal-LCA-Commons-Elementary-Flow-List.git@develop#egg=fedelemflowlist", "esupy @ git+https://github.com/USEPA/esupy.git@develop#egg=esupy", "olca-ipc>=0.0.12",