pass config as kwarg through to load_yaml_dict to enable generation o…

…f FBS within another FBS method; to use deepcopy, convert method_config_keys from `dict_keys` to `set`
USEPA · Oct 16, 2023 · b4f4804 · b4f4804
1 parent b5f0891
commit b4f4804
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 8 deletions.
diff --git a/flowsa/common.py b/flowsa/common.py
@@ -10,6 +10,7 @@
 import yaml
 import pandas as pd
 import numpy as np
+from copy import deepcopy
 from dotenv import load_dotenv
 import flowsa.flowsa_yaml as flowsa_yaml
 import flowsa.exceptions
@@ -113,7 +114,7 @@ def return_bea_codes_used_as_naics():
     return code_list
 
 
-def load_yaml_dict(filename, flowbytype=None, filepath=None):
+def load_yaml_dict(filename, flowbytype=None, filepath=None, **kwargs):
     """
     Load the information in a yaml file, from source_catalog, or FBA,
     or FBS files
@@ -156,8 +157,11 @@ def load_yaml_dict(filename, flowbytype=None, filepath=None):
         with open(yaml_path, 'r', encoding='utf-8') as f:
             config = flowsa_yaml.load(f, filepath)
     except FileNotFoundError:
-        raise flowsa.exceptions.FlowsaMethodNotFoundError(
-            method_type=flowbytype, method=filename)
+        if 'config' in kwargs:
+            return deepcopy(kwargs['config'])
+        else:
+            raise flowsa.exceptions.FlowsaMethodNotFoundError(
+                method_type=flowbytype, method=filename)
     return config
 
 

diff --git a/flowsa/flowbysector.py b/flowsa/flowbysector.py
@@ -99,7 +99,7 @@ def getFlowBySector(
 
         flowby_generator = (
             lambda x=method, y=external_config_path, z=download_sources_ok:
-                cls.generateFlowBySector(x, y, z)
+                cls.generateFlowBySector(x, y, z, config=config)
             )
         return super()._getFlowBy(
             file_metadata=file_metadata,
@@ -117,6 +117,7 @@ def generateFlowBySector(
         method: str,
         external_config_path: str = None,
         download_sources_ok: bool = settings.DEFAULT_DOWNLOAD_IF_MISSING,
+        **kwargs
     ) -> 'FlowBySector':
         '''
         Generates a FlowBySector dataset.
@@ -126,10 +127,13 @@ def generateFlowBySector(
         :param download_fba_ok: bool, optional. Whether to attempt to download
             source data FlowByActivity files from EPA server rather than
             generating them.
+        :kwargs: keyword arguments to pass to load_yaml_dict(). Possible kwargs
+            include config.
         '''
         log.info('Beginning FlowBySector generation for %s', method)
         method_config = common.load_yaml_dict(method, 'FBS',
-                                              external_config_path)
+                                              external_config_path,
+                                              **kwargs)
 
         # Cache one or more sources by attaching to method_config
         to_cache = method_config.pop('sources_to_cache', {})
@@ -144,7 +148,7 @@ def generateFlowBySector(
                     name=source_name,
                     config={
                         **method_config,
-                        'method_config_keys': method_config.keys(),
+                        'method_config_keys': set(method_config.keys()),
                         **get_catalog_info(source_name),
                         **config
                     },
@@ -165,7 +169,7 @@ def generateFlowBySector(
                 name=source_name,
                 config={
                     **method_config,
-                    'method_config_keys': method_config.keys(),
+                    'method_config_keys': set(method_config.keys()),
                     **get_catalog_info(source_name),
                     **config
                 },
@@ -206,7 +210,7 @@ def generateFlowBySector(
         reset_log_file(method, meta)
         metadata.write_metadata(source_name=method,
                                 config=common.load_yaml_dict(
-                                    method, 'FBS', external_config_path),
+                                    method, 'FBS', external_config_path, **kwargs),
                                 fb_meta=meta,
                                 category='FlowBySector')