From 5d1d6402ab52b18f94e0dfbc834d61d4c85f251c Mon Sep 17 00:00:00 2001 From: Paolo Losi Date: Wed, 4 Mar 2020 16:01:16 +0100 Subject: [PATCH] counter.py: reduce output without fix when redirecting the training log the output could exceed 1G --- DeepSpeech/counter.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/DeepSpeech/counter.py b/DeepSpeech/counter.py index 1a01602a..5bd72111 100644 --- a/DeepSpeech/counter.py +++ b/DeepSpeech/counter.py @@ -12,11 +12,10 @@ def main(input_file, output_file, top_count=100000): fcontent = input.readlines() all_file = len(fcontent) print('Ingesting {}: {}'.format(input_file, all_file)) - current = 0 - for line in fcontent: - print('Feeding {}: {}/{} ({:.2f}%)'.format(input_file, current, all_file, (current / all_file) * 100), end='\r') + for current, line in enumerate(fcontent): + if (current % 1000) == 0: + print('Feeding {}: {}/{} ({:.2f}%)'.format(input_file, current, all_file, (current / all_file) * 100), end='\r') counter.update(line.split()) - current += 1 print('Counting {}'.format(input_file)) vocab_str = '\n'.join(word for word, count in counter.most_common(top_count)) print('Writing {}'.format(output_file))