Skip to content

Commit

Permalink
feat: csv/tsv/ndjson support querying file meta data.
Browse files Browse the repository at this point in the history
  • Loading branch information
youngsofun committed Feb 24, 2025
1 parent ffcf689 commit 38f44f6
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
query ok
select metadata$filename, $1, metadata$file_row_number from @data_s3/ndjson/ts.ndjson (file_format=>'ndjson')
----
testbucket/data/ndjson/ts.ndjson {"t":1736305864} 0
testbucket/data/ndjson/ts.ndjson {"t":1736305865000} 1
testbucket/data/ndjson/ts.ndjson {"t":1736305866000000} 2

statement ok
create or replace table t(file_name string, value variant, row int)

query ok
copy into t from (select metadata$filename, $1, metadata$file_row_number + 1 from @data_s3/ndjson/ts.ndjson) file_format=(type=ndjson)
----
ndjson/ts.ndjson 3 0 NULL NULL

query ok
select * from t
----
testbucket/data/ndjson/ts.ndjson {"t":1736305864} 1
testbucket/data/ndjson/ts.ndjson {"t":1736305865000} 2
testbucket/data/ndjson/ts.ndjson {"t":1736305866000000} 3
20 changes: 20 additions & 0 deletions tests/sqllogictests/suites/stage/formats/tsv/tsv_metadata.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
query ok
select metadata$filename, $2, metadata$file_row_number, $1 from @data_s3/tsv/no_newline.tsv (file_format=>'tsv')
----
testbucket/data/tsv/no_newline.tsv 2 0 1
testbucket/data/tsv/no_newline.tsv 4 1 3


statement ok
create or replace table t(file_name string, c2 string, row int, c1 int)

query ok
copy into t from (select metadata$filename, $2, metadata$file_row_number + 1, $1 from @data_s3/tsv/no_newline.tsv) file_format=(type=tsv)
----
tsv/no_newline.tsv 2 0 NULL NULL

query ok
select * from t
----
testbucket/data/tsv/no_newline.tsv 2 1 1
testbucket/data/tsv/no_newline.tsv 4 2 3

0 comments on commit 38f44f6

Please sign in to comment.