Skip to content

Commit a77d37d

Browse files
committed
fix json file size computation
1 parent 8a7e870 commit a77d37d

File tree

2 files changed

+16
-1
lines changed

2 files changed

+16
-1
lines changed

src/lib/src/core/df/tabular.rs

+14
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,12 @@ pub fn scan_df_jsonl(
123123
)
124124
}
125125

126+
pub fn scan_df_json(path: impl AsRef<Path>) -> Result<LazyFrame, OxenError> {
127+
// cannot lazy read json array
128+
let df = read_df_json(path)?;
129+
Ok(df.lazy())
130+
}
131+
126132
pub fn read_df_parquet(path: impl AsRef<Path>) -> Result<DataFrame, OxenError> {
127133
let path = path.as_ref();
128134
let error_str = format!("Could not read tabular data from path {path:?}");
@@ -876,6 +882,7 @@ pub fn scan_df(
876882
Some(extension) => match extension {
877883
"ndjson" => scan_df_jsonl(path, opts, total_rows),
878884
"jsonl" => scan_df_jsonl(path, opts, total_rows),
885+
"json" => scan_df_json(path),
879886
"csv" | "data" => {
880887
let delimiter = sniff_db_csv_delimiter(&path, opts)?;
881888
scan_df_csv(path, delimiter, opts, total_rows)
@@ -924,6 +931,13 @@ pub fn get_size(path: impl AsRef<Path>) -> Result<DataFrameSize, OxenError> {
924931
let height = reader._num_rows()?;
925932
Ok(DataFrameSize { width, height })
926933
}
934+
"json" => {
935+
let df = lazy_df
936+
.collect()
937+
.map_err(|_| OxenError::basic_str("Could not collect json df"))?;
938+
let height = df.height();
939+
Ok(DataFrameSize { width, height })
940+
}
927941
_ => Err(OxenError::basic_str(err)),
928942
},
929943
None => Err(OxenError::basic_str(err)),

src/lib/src/core/index/commit_entry_writer.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,8 @@ impl CommitEntryWriter {
216216
let full_path = origin_path.join(file_path);
217217

218218
// // Get last modified time
219-
let metadata = fs::metadata(&full_path).map_err(|e| OxenError::file_error(&full_path, e))?;
219+
let metadata =
220+
fs::metadata(&full_path).map_err(|e| OxenError::file_error(&full_path, e))?;
220221
let mtime = FileTime::from_last_modification_time(&metadata);
221222

222223
let metadata = fs::metadata(&full_path)?;

0 commit comments

Comments
 (0)