fix: recognize transfer_limit

lbsntransform/__main__.py

@@ -43,6 +43,7 @@ def main():
         origin_id=config.origin,
         logging_level=config.logging_level,
         is_local_input=config.is_local_input,
+        transfer_count=config.transfer_count,
         csv_output=config.csv_output,
         csv_suppress_linebreaks=config.csv_suppress_linebreaks,
         dbuser_output=config.dbuser_output,
@@ -110,7 +111,9 @@ def main():
                     f'Count per type: '
                     f'{lbsntransform.lbsn_records.get_type_counts()}'
                     f'records.', end='\r')
-
+            if (config.transferlimit and
+                    lbsntransform.processed_total >= config.transferlimit):
+                break
     # finalize output (close db connection, submit remaining)
     print(f'Transferring remaining '
           f'{lbsntransform.lbsn_records.count_glob} to db.. '

lbsntransform/lbsntransform_.py

@@ -49,7 +49,6 @@ class LBSNTransform():
     def __init__(
             self, origin_id=3, logging_level=None,
             is_local_input: bool = False, transfer_count: int = 50000,
-            transferlimit: int = None,
             csv_output: bool = True, csv_suppress_linebreaks: bool = True,
             dbuser_output=None, dbserveraddress_output=None, dbname_output=None,
             dbpassword_output=None, dbserverport_output=None,
@@ -69,9 +68,6 @@ def __init__(
         # init global settings
 
         self.transfer_count = transfer_count
-        self.transferlimit = None
-        if transferlimit:
-            self.transferlimit = transferlimit
         self.importer = HF.load_importer_mapping_module(
             origin_id)
         # get origin name and id from importer
@@ -110,7 +106,6 @@ def __init__(
         self.processed_total = 0
         self.skipped_low_geoaccuracy = 0
         self.initial_loop = True
-        self.max_records = None
         self.how_long = None
         # field mapping structure
         # this is where all the converted data will be stored
@@ -126,6 +121,7 @@ def add_processed_records(self, lbsn_record):
         self.lbsn_records.add_records_to_dict(
             lbsn_record)
         self.processed_records += 1
+        self.processed_total += 1
         # On the first loop
         # or after 50.000 (default) processed records,
         # store results
@@ -143,12 +139,8 @@ def store_lbsn_records(self):
         self.output.commit_changes()
         self.lbsn_records.clear()
         # update statistics
-        self.processed_total += self.processed_records
         self.processed_records = 0
 
-        if self.transferlimit:
-            self.max_records = self.transferlimit - self.processed_total
-
     def finalize_output(self):
         """finalize all transactions (csv merge etc.)
         """