Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

remove SubqueryFilterToJoin #4731

Merged
merged 2 commits into from
Dec 28, 2022
Merged

remove SubqueryFilterToJoin #4731

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
edd7990
remove `SubqueryFilterToJoin`
jackwener Dec 26, 2022
9af19bd
correct UT
jackwener Dec 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 187 additions & 1 deletion datafusion/optimizer/src/decorrelate_where_in.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,10 +220,196 @@ mod tests {
use crate::test::*;
use datafusion_common::Result;
use datafusion_expr::{
col, in_subquery, lit, logical_plan::LogicalPlanBuilder, not_in_subquery,
and, binary_expr, col, in_subquery, lit, logical_plan::LogicalPlanBuilder,
not_in_subquery, or, Operator,
};
use std::ops::Add;

fn assert_optimized_plan_equal(plan: &LogicalPlan, expected: &str) -> Result<()> {
assert_optimized_plan_eq_display_indent(
Arc::new(DecorrelateWhereIn::new()),
plan,
expected,
);
Ok(())
}

fn test_subquery_with_name(name: &str) -> Result<Arc<LogicalPlan>> {
let table_scan = test_table_scan_with_name(name)?;
Ok(Arc::new(
LogicalPlanBuilder::from(table_scan)
.project(vec![col("c")])?
.build()?,
))
}

/// Test for several IN subquery expressions
#[test]
fn in_subquery_multiple() -> Result<()> {
let table_scan = test_table_scan()?;
let plan = LogicalPlanBuilder::from(table_scan)
.filter(and(
in_subquery(col("c"), test_subquery_with_name("sq_1")?),
in_subquery(col("b"), test_subquery_with_name("sq_2")?),
))?
.project(vec![col("test.b")])?
.build()?;

let expected = "Projection: test.b [b:UInt32]\
\n LeftSemi Join: test.b = __sq_2.c [a:UInt32, b:UInt32, c:UInt32]\
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice 👍

\n LeftSemi Join: test.c = __sq_1.c [a:UInt32, b:UInt32, c:UInt32]\
\n TableScan: test [a:UInt32, b:UInt32, c:UInt32]\
\n SubqueryAlias: __sq_1 [c:UInt32]\
\n Projection: sq_1.c AS c [c:UInt32]\
\n TableScan: sq_1 [a:UInt32, b:UInt32, c:UInt32]\
\n SubqueryAlias: __sq_2 [c:UInt32]\
\n Projection: sq_2.c AS c [c:UInt32]\
\n TableScan: sq_2 [a:UInt32, b:UInt32, c:UInt32]";

assert_optimized_plan_equal(&plan, expected)
}

/// Test for IN subquery with additional AND filter
#[test]
fn in_subquery_with_and_filters() -> Result<()> {
let table_scan = test_table_scan()?;
let plan = LogicalPlanBuilder::from(table_scan)
.filter(and(
in_subquery(col("c"), test_subquery_with_name("sq")?),
and(
binary_expr(col("a"), Operator::Eq, lit(1_u32)),
binary_expr(col("b"), Operator::Lt, lit(30_u32)),
),
))?
.project(vec![col("test.b")])?
.build()?;

let expected = "Projection: test.b [b:UInt32]\
\n Filter: test.a = UInt32(1) AND test.b < UInt32(30) [a:UInt32, b:UInt32, c:UInt32]\
\n LeftSemi Join: test.c = __sq_1.c [a:UInt32, b:UInt32, c:UInt32]\
\n TableScan: test [a:UInt32, b:UInt32, c:UInt32]\
\n SubqueryAlias: __sq_1 [c:UInt32]\
\n Projection: sq.c AS c [c:UInt32]\
\n TableScan: sq [a:UInt32, b:UInt32, c:UInt32]";

assert_optimized_plan_equal(&plan, expected)
}

/// Test for IN subquery with additional OR filter
/// filter expression not modified
#[test]
fn in_subquery_with_or_filters() -> Result<()> {
let table_scan = test_table_scan()?;
let plan = LogicalPlanBuilder::from(table_scan)
.filter(or(
and(
binary_expr(col("a"), Operator::Eq, lit(1_u32)),
binary_expr(col("b"), Operator::Lt, lit(30_u32)),
),
in_subquery(col("c"), test_subquery_with_name("sq")?),
))?
.project(vec![col("test.b")])?
.build()?;

let expected = "Projection: test.b [b:UInt32]\
\n Filter: test.a = UInt32(1) AND test.b < UInt32(30) OR test.c IN (<subquery>) [a:UInt32, b:UInt32, c:UInt32]\
\n Subquery: [c:UInt32]\
\n Projection: sq.c [c:UInt32]\
\n TableScan: sq [a:UInt32, b:UInt32, c:UInt32]\
\n TableScan: test [a:UInt32, b:UInt32, c:UInt32]";

assert_optimized_plan_equal(&plan, expected)
}

#[test]
fn in_subquery_with_and_or_filters() -> Result<()> {
let table_scan = test_table_scan()?;
let plan = LogicalPlanBuilder::from(table_scan)
.filter(and(
or(
binary_expr(col("a"), Operator::Eq, lit(1_u32)),
in_subquery(col("b"), test_subquery_with_name("sq1")?),
),
in_subquery(col("c"), test_subquery_with_name("sq2")?),
))?
.project(vec![col("test.b")])?
.build()?;

let expected = "Projection: test.b [b:UInt32]\
\n Filter: test.a = UInt32(1) OR test.b IN (<subquery>) [a:UInt32, b:UInt32, c:UInt32]\
\n Subquery: [c:UInt32]\
\n Projection: sq1.c [c:UInt32]\
\n TableScan: sq1 [a:UInt32, b:UInt32, c:UInt32]\
\n LeftSemi Join: test.c = __sq_1.c [a:UInt32, b:UInt32, c:UInt32]\
\n TableScan: test [a:UInt32, b:UInt32, c:UInt32]\
\n SubqueryAlias: __sq_1 [c:UInt32]\
\n Projection: sq2.c AS c [c:UInt32]\
\n TableScan: sq2 [a:UInt32, b:UInt32, c:UInt32]";

assert_optimized_plan_equal(&plan, expected)
}

/// Test for nested IN subqueries
#[test]
fn in_subquery_nested() -> Result<()> {
let table_scan = test_table_scan()?;

let subquery = LogicalPlanBuilder::from(test_table_scan_with_name("sq")?)
.filter(in_subquery(col("a"), test_subquery_with_name("sq_nested")?))?
.project(vec![col("a")])?
.build()?;

let plan = LogicalPlanBuilder::from(table_scan)
.filter(in_subquery(col("b"), Arc::new(subquery)))?
.project(vec![col("test.b")])?
.build()?;

let expected = "Projection: test.b [b:UInt32]\
\n LeftSemi Join: test.b = __sq_1.a [a:UInt32, b:UInt32, c:UInt32]\
\n TableScan: test [a:UInt32, b:UInt32, c:UInt32]\
\n SubqueryAlias: __sq_1 [a:UInt32]\
\n Projection: sq.a AS a [a:UInt32]\
\n LeftSemi Join: sq.a = __sq_2.c [a:UInt32, b:UInt32, c:UInt32]\
\n TableScan: sq [a:UInt32, b:UInt32, c:UInt32]\
\n SubqueryAlias: __sq_2 [c:UInt32]\
\n Projection: sq_nested.c AS c [c:UInt32]\
\n TableScan: sq_nested [a:UInt32, b:UInt32, c:UInt32]";

assert_optimized_plan_equal(&plan, expected)
}

/// Test for filter input modification in case filter not supported
/// Outer filter expression not modified while inner converted to join
#[test]
fn in_subquery_input_modified() -> Result<()> {
let table_scan = test_table_scan()?;
let plan = LogicalPlanBuilder::from(table_scan)
.filter(in_subquery(col("c"), test_subquery_with_name("sq_inner")?))?
.project(vec![col("b"), col("c")])?
.alias("wrapped")?
.filter(or(
binary_expr(col("b"), Operator::Lt, lit(30_u32)),
in_subquery(col("c"), test_subquery_with_name("sq_outer")?),
))?
.project(vec![col("b")])?
.build()?;

let expected = "Projection: wrapped.b [b:UInt32]\
\n Filter: wrapped.b < UInt32(30) OR wrapped.c IN (<subquery>) [b:UInt32, c:UInt32]\
\n Subquery: [c:UInt32]\
\n Projection: sq_outer.c [c:UInt32]\
\n TableScan: sq_outer [a:UInt32, b:UInt32, c:UInt32]\
\n SubqueryAlias: wrapped [b:UInt32, c:UInt32]\
\n Projection: test.b, test.c [b:UInt32, c:UInt32]\
\n LeftSemi Join: test.c = __sq_1.c [a:UInt32, b:UInt32, c:UInt32]\
\n TableScan: test [a:UInt32, b:UInt32, c:UInt32]\
\n SubqueryAlias: __sq_1 [c:UInt32]\
\n Projection: sq_inner.c AS c [c:UInt32]\
\n TableScan: sq_inner [a:UInt32, b:UInt32, c:UInt32]";

assert_optimized_plan_equal(&plan, expected)
}

#[cfg(test)]
#[ctor::ctor]
fn init() {
Expand Down
1 change: 0 additions & 1 deletion datafusion/optimizer/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ pub mod push_down_projection;
pub mod scalar_subquery_to_join;
pub mod simplify_expressions;
pub mod single_distinct_to_groupby;
pub mod subquery_filter_to_join;
pub mod type_coercion;
pub mod utils;

Expand Down
2 changes: 0 additions & 2 deletions datafusion/optimizer/src/optimizer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ use crate::rewrite_disjunctive_predicate::RewriteDisjunctivePredicate;
use crate::scalar_subquery_to_join::ScalarSubqueryToJoin;
use crate::simplify_expressions::SimplifyExpressions;
use crate::single_distinct_to_groupby::SingleDistinctToGroupBy;
use crate::subquery_filter_to_join::SubqueryFilterToJoin;
use crate::type_coercion::TypeCoercion;
use crate::unwrap_cast_in_comparison::UnwrapCastInComparison;
use chrono::{DateTime, Utc};
Expand Down Expand Up @@ -242,7 +241,6 @@ impl Optimizer {
Arc::new(DecorrelateWhereExists::new()),
Arc::new(DecorrelateWhereIn::new()),
Arc::new(ScalarSubqueryToJoin::new()),
Arc::new(SubqueryFilterToJoin::new()),
// simplify expressions does not simplify expressions in subqueries, so we
// run it again after running the optimizations that potentially converted
// subqueries to joins
Expand Down
Loading