Skip to content

Commit

Permalink
c
Browse files Browse the repository at this point in the history
  • Loading branch information
nameexhaustion committed Nov 21, 2024
1 parent 5f63e49 commit c4f20a5
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 15 deletions.
13 changes: 7 additions & 6 deletions crates/polars-plan/src/plans/conversion/dsl_to_ir.rs
Original file line number Diff line number Diff line change
Expand Up @@ -234,20 +234,21 @@ pub fn to_alp_impl(lp: DslPlan, ctxt: &mut DslConversionContext) -> PolarsResult
},
};

if file_options.hive_options.enabled.is_none() {
// TODO: Auto-enable this if `hive_options.schema` was given.
file_options.hive_options.enabled = Some(false);
}

if file_options.hive_options.schema.is_some()
&& !file_options.hive_options.enabled.unwrap()
&& file_options.hive_options.enabled == Some(false)
{
// hive_partitioning was explicitly disabled
polars_bail!(
ComputeError:
"a hive schema was given but hive_partitioning was disabled"
)
}

if file_options.hive_options.enabled.is_none() {
// TODO: Auto-enable this if `hive_options.schema` was given.
file_options.hive_options.enabled = Some(false);
}

let hive_parts = if file_options.hive_options.enabled.unwrap()
&& file_info.reader_schema.is_some()
{
Expand Down
19 changes: 10 additions & 9 deletions py-polars/tests/unit/io/test_hive.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ def test_hive_partition_directory_scan(
).collect()
assert_frame_equal(out, df)

out = scan(tmp_path, hive_partitioning=False).collect()
out = scan(tmp_path, hive_partitioning=False, hive_schema=None).collect()
assert_frame_equal(out, df.drop("a", "b"))

out = scan(
Expand All @@ -352,18 +352,15 @@ def test_hive_partition_directory_scan(
).collect()
assert_frame_equal(out, df.filter(a=1).drop("a"))

out = scan(
tmp_path / "a=1",
hive_partitioning=False,
).collect()
out = scan(tmp_path / "a=1", hive_partitioning=False, hive_schema=None).collect()
assert_frame_equal(out, df.filter(a=1).drop("a", "b"))

path = tmp_path / "a=1/b=1/data.bin"

out = scan(path, hive_partitioning=True).collect()
assert_frame_equal(out, dfs[0])

out = scan(path, hive_partitioning=False).collect()
out = scan(path, hive_partitioning=False, hive_schema=None).collect()
assert_frame_equal(out, dfs[0].drop("a", "b"))

# Test default behavior with `hive_partitioning=None`, which should only
Expand Down Expand Up @@ -429,14 +426,18 @@ def test_hive_partition_directory_scan(
)

# Test `hive_partitioning=False`
out = scan(tmp_path, hive_partitioning=False).collect()
out = scan(tmp_path, hive_partitioning=False, hive_schema=None).collect()
assert_frame_equal(out, df.drop("a", "b"))

if glob:
out = scan(tmp_path / "**/*.bin", hive_partitioning=False).collect()
out = scan(
tmp_path / "**/*.bin", hive_partitioning=False, hive_schema=None
).collect()
assert_frame_equal(out, df.drop("a", "b"))

out = scan(tmp_path / "a=1/b=1/data.bin", hive_partitioning=False).collect()
out = scan(
tmp_path / "a=1/b=1/data.bin", hive_partitioning=False, hive_schema=None
).collect()
assert_frame_equal(out, df.filter(a=1, b=1).drop("a", "b"))


Expand Down

0 comments on commit c4f20a5

Please sign in to comment.