Skip to content

Commit

Permalink
fix: --skip_until_file not implemented
Browse files Browse the repository at this point in the history
  • Loading branch information
Sieboldianus committed Jul 12, 2022
1 parent 9e50f27 commit 18e7665
Showing 1 changed file with 23 additions and 8 deletions.
31 changes: 23 additions & 8 deletions lbsntransform/input/load_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,6 @@ def __init__(
self.start_number = startwith_db_rownumber
else:
self.continue_number = 0
if skip_until_file is not None:
self.continue_number = skip_until_file
self.skip_until_record = skip_until_record
self.source_web = source_web
if zip_records is None:
Expand Down Expand Up @@ -476,7 +474,7 @@ def initialize_connection(

@staticmethod
def _read_local_files(input_path=None, recursive_load=None,
local_file_type=None, skip_until_file=None):
local_file_type=None, skip_until_file=None) -> List[str]:
"""Read Local Files according to config parameters and
returns list of file-paths
"""
Expand All @@ -485,23 +483,40 @@ def _read_local_files(input_path=None, recursive_load=None,
"02_UserData", "03_ClippedData", "04_MapVis"]
excludestartswithfile = ["log", "settings", "GridCoordinates"]
loc_filelist = \
LoadData.scan_rec(input_path,
file_format=local_file_type,
excludefolderlist=excludefolderlist,
excludestartswithfile=excludestartswithfile)
LoadData.scan_rec(
input_path,
file_format=local_file_type,
excludefolderlist=excludefolderlist,
excludestartswithfile=excludestartswithfile)
else:
loc_filelist_gen = input_path.glob(f'*.{local_file_type}')
loc_filelist = []
for file_path in loc_filelist_gen:
loc_filelist.append(file_path)
if skip_until_file:
print("Implement skip until file")
file_index = LoadData._item_index_list(
loc_filelist, skip_until_file)
logging.getLogger('__main__').info(
f"\nSkipped {len(loc_filelist) - len(loc_filelist[file_index:])}"
f" input files. \n")
loc_filelist = loc_filelist[file_index:]
input_count = (len(loc_filelist))
if input_count == 0:
sys.exit("No location files found.")
else:
return loc_filelist

@staticmethod
def _item_index_list(item_list, value, split_filename = True):
index = 0
for element in item_list:
if split_filename:
element = os.path.split(element)[1]
if element == value:
return index
index += 1
return -1

@staticmethod
def load_geocodes(geo_config):
"""Loads coordinates-string tuples for geocoding
Expand Down

0 comments on commit 18e7665

Please sign in to comment.