-
Notifications
You must be signed in to change notification settings - Fork 761
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: csv/tsv/ndjson support querying file meta data.
- Loading branch information
1 parent
ffcf689
commit 38f44f6
Showing
2 changed files
with
41 additions
and
0 deletions.
There are no files selected for viewing
21 changes: 21 additions & 0 deletions
21
tests/sqllogictests/suites/stage/formats/ndjson/ndjson_metadata.test
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
query ok | ||
select metadata$filename, $1, metadata$file_row_number from @data_s3/ndjson/ts.ndjson (file_format=>'ndjson') | ||
---- | ||
testbucket/data/ndjson/ts.ndjson {"t":1736305864} 0 | ||
testbucket/data/ndjson/ts.ndjson {"t":1736305865000} 1 | ||
testbucket/data/ndjson/ts.ndjson {"t":1736305866000000} 2 | ||
|
||
statement ok | ||
create or replace table t(file_name string, value variant, row int) | ||
|
||
query ok | ||
copy into t from (select metadata$filename, $1, metadata$file_row_number + 1 from @data_s3/ndjson/ts.ndjson) file_format=(type=ndjson) | ||
---- | ||
ndjson/ts.ndjson 3 0 NULL NULL | ||
|
||
query ok | ||
select * from t | ||
---- | ||
testbucket/data/ndjson/ts.ndjson {"t":1736305864} 1 | ||
testbucket/data/ndjson/ts.ndjson {"t":1736305865000} 2 | ||
testbucket/data/ndjson/ts.ndjson {"t":1736305866000000} 3 |
20 changes: 20 additions & 0 deletions
20
tests/sqllogictests/suites/stage/formats/tsv/tsv_metadata.test
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
query ok | ||
select metadata$filename, $2, metadata$file_row_number, $1 from @data_s3/tsv/no_newline.tsv (file_format=>'tsv') | ||
---- | ||
testbucket/data/tsv/no_newline.tsv 2 0 1 | ||
testbucket/data/tsv/no_newline.tsv 4 1 3 | ||
|
||
|
||
statement ok | ||
create or replace table t(file_name string, c2 string, row int, c1 int) | ||
|
||
query ok | ||
copy into t from (select metadata$filename, $2, metadata$file_row_number + 1, $1 from @data_s3/tsv/no_newline.tsv) file_format=(type=tsv) | ||
---- | ||
tsv/no_newline.tsv 2 0 NULL NULL | ||
|
||
query ok | ||
select * from t | ||
---- | ||
testbucket/data/tsv/no_newline.tsv 2 1 1 | ||
testbucket/data/tsv/no_newline.tsv 4 2 3 |