From 0df1c71b78e947901e42905a57609bd5c2b00779 Mon Sep 17 00:00:00 2001 From: Praateek Date: Wed, 9 Oct 2024 14:07:05 -0700 Subject: [PATCH] pop meta Signed-off-by: Praateek --- nemo_curator/utils/distributed_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nemo_curator/utils/distributed_utils.py b/nemo_curator/utils/distributed_utils.py index 3f37eb901..aea24d301 100644 --- a/nemo_curator/utils/distributed_utils.py +++ b/nemo_curator/utils/distributed_utils.py @@ -347,6 +347,9 @@ def read_data( ] else: input_files = [[file] for file in input_files] + + meta_as_kwarg = {"meta": input_meta} if input_meta is not None else dict() + return dd.from_map( read_single_partition, input_files, @@ -355,6 +358,7 @@ def read_data( add_filename=add_filename, input_meta=input_meta, enforce_metadata=False, + **meta_as_kwarg, ) else: raise RuntimeError("Could not read data, please check file type")