From d1f4ac88f163485b6a8c7cd8c22b72b86e437117 Mon Sep 17 00:00:00 2001 From: Guilherme Salustiano Date: Fri, 28 Apr 2023 16:25:40 -0300 Subject: [PATCH] feat: add count lines script --- .../informacao_consolidada/code/count_lines.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 bases/br_ans_beneficiario/informacao_consolidada/code/count_lines.py diff --git a/bases/br_ans_beneficiario/informacao_consolidada/code/count_lines.py b/bases/br_ans_beneficiario/informacao_consolidada/code/count_lines.py new file mode 100644 index 000000000..f8e6a9331 --- /dev/null +++ b/bases/br_ans_beneficiario/informacao_consolidada/code/count_lines.py @@ -0,0 +1,15 @@ +from glob import glob +import pandas as pd +from loguru import logger + +csv_filepaths = glob('../input/**/*.csv', recursive=True) + +count = 0 +for filepath in csv_filepaths: + logger.info(f"reading {filepath}") + df = pd.read_csv(filepath, encoding="utf-8", index_col=0) + size = len(df.index) + logger.debug(f"{filepath} has {size}") + count += size + +logger.info(f"Total: {count}")