Skip to content

Commit

Permalink
add session option 'datafusion.explain.logical_plan'. when set to tru…
Browse files Browse the repository at this point in the history 
…e, the explain statement will only print logical plans. (#2895)

* add session option 'datafusion.explain.logical_plan'. when set to true, the explain statement will only print logical plans.

* 1. add test cases 2. add session options 'datafusion.explain.logical_plan_only' and 'datafusion.explain.physical_plan_only' to
seperately print logical plans and physical plans.

* update docs/source/user-guide/configs.md
  • Loading branch information
@HuSen8891
HuSen8891 authored Jul 14, 2022
1 parent 6dc9dea commit 1b8117e
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 16 deletions.
16 changes: 16 additions & 0 deletions datafusion/core/src/config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,12 @@ use std::env;
/// Configuration option "datafusion.optimizer.filter_null_join_keys"
pub const OPT_FILTER_NULL_JOIN_KEYS: &str = "datafusion.optimizer.filter_null_join_keys";

/// Configuration option "datafusion.explain.logical_plan_only"
pub const OPT_EXPLAIN_LOGICAL_PLAN_ONLY: &str = "datafusion.explain.logical_plan_only";

/// Configuration option "datafusion.explain.physical_plan_only"
pub const OPT_EXPLAIN_PHYSICAL_PLAN_ONLY: &str = "datafusion.explain.physical_plan_only";

/// Configuration option "datafusion.execution.batch_size"
pub const OPT_BATCH_SIZE: &str = "datafusion.execution.batch_size";

Expand Down Expand Up @@ -119,6 +125,16 @@ impl BuiltInConfigs {
predicate push down.",
false,
),
ConfigDefinition::new_bool(
OPT_EXPLAIN_LOGICAL_PLAN_ONLY,
"When set to true, the explain statement will only print logical plans.",
false,
),
ConfigDefinition::new_bool(
OPT_EXPLAIN_PHYSICAL_PLAN_ONLY,
"When set to true, the explain statement will only print physical plans.",
false,
),
ConfigDefinition::new_u64(
OPT_BATCH_SIZE,
"Default batch size while creating new batches, it's especially useful for \
Expand Down
47 changes: 32 additions & 15 deletions datafusion/core/src/physical_plan/planner.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ use super::{
aggregates, empty::EmptyExec, hash_join::PartitionMode, udaf, union::UnionExec,
values::ValuesExec, windows,
};
use crate::config::{OPT_EXPLAIN_LOGICAL_PLAN_ONLY, OPT_EXPLAIN_PHYSICAL_PLAN_ONLY};
use crate::datasource::source_as_provider;
use crate::execution::context::{ExecutionProps, SessionState};
use crate::logical_expr::utils::generate_sort_key;
Expand Down Expand Up @@ -1487,26 +1488,42 @@ impl DefaultPhysicalPlanner {
) -> Result<Option<Arc<dyn ExecutionPlan>>> {
if let LogicalPlan::Explain(e) = logical_plan {
use PlanType::*;
let mut stringified_plans = e.stringified_plans.clone();
let mut stringified_plans = vec![];

stringified_plans.push(e.plan.to_stringified(FinalLogicalPlan));
if !session_state
.config
.config_options
.get_bool(OPT_EXPLAIN_PHYSICAL_PLAN_ONLY)
{
stringified_plans = e.stringified_plans.clone();

let input = self
.create_initial_plan(e.plan.as_ref(), session_state)
.await?;
stringified_plans.push(e.plan.to_stringified(FinalLogicalPlan));
}

if !session_state
.config
.config_options
.get_bool(OPT_EXPLAIN_LOGICAL_PLAN_ONLY)
{
let input = self
.create_initial_plan(e.plan.as_ref(), session_state)
.await?;

stringified_plans
.push(displayable(input.as_ref()).to_stringified(InitialPhysicalPlan));
stringified_plans.push(
displayable(input.as_ref()).to_stringified(InitialPhysicalPlan),
);

let input =
self.optimize_internal(input, session_state, |plan, optimizer| {
let optimizer_name = optimizer.name().to_string();
let plan_type = OptimizedPhysicalPlan { optimizer_name };
stringified_plans.push(displayable(plan).to_stringified(plan_type));
})?;
let input =
self.optimize_internal(input, session_state, |plan, optimizer| {
let optimizer_name = optimizer.name().to_string();
let plan_type = OptimizedPhysicalPlan { optimizer_name };
stringified_plans
.push(displayable(plan).to_stringified(plan_type));
})?;

stringified_plans
.push(displayable(input.as_ref()).to_stringified(FinalPhysicalPlan));
stringified_plans
.push(displayable(input.as_ref()).to_stringified(FinalPhysicalPlan));
}

Ok(Some(Arc::new(ExplainExec::new(
SchemaRef::new(e.schema.as_ref().to_owned().into()),
Expand Down
41 changes: 40 additions & 1 deletion datafusion/core/tests/sql/explain_analyze.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@
// under the License.

use super::*;
use datafusion::physical_plan::display::DisplayableExecutionPlan;
use datafusion::{
config::{OPT_EXPLAIN_LOGICAL_PLAN_ONLY, OPT_EXPLAIN_PHYSICAL_PLAN_ONLY},
physical_plan::display::DisplayableExecutionPlan,
};

#[tokio::test]
async fn explain_analyze_baseline_metrics() {
Expand Down Expand Up @@ -810,3 +813,39 @@ async fn csv_explain_analyze_verbose() {
let verbose_needle = "Output Rows";
assert_contains!(formatted, verbose_needle);
}

#[tokio::test]
async fn explain_logical_plan_only() {
let config = SessionConfig::new().set_bool(OPT_EXPLAIN_LOGICAL_PLAN_ONLY, true);
let ctx = SessionContext::with_config(config);
let sql = "EXPLAIN select count(*) from (values ('a', 1, 100), ('a', 2, 150)) as t (c1,c2,c3)";
let actual = execute(&ctx, sql).await;
let actual = normalize_vec_for_explain(actual);

let expected = vec![
vec![
"logical_plan",
"Projection: #COUNT(UInt8(1))\
\n Aggregate: groupBy=[[]], aggr=[[COUNT(UInt8(1))]]\
\n Values: (Utf8(\"a\"), Int64(1), Int64(100)), (Utf8(\"a\"), Int64(2), Int64(150))",
]];
assert_eq!(expected, actual);
}

#[tokio::test]
async fn explain_physical_plan_only() {
let config = SessionConfig::new().set_bool(OPT_EXPLAIN_PHYSICAL_PLAN_ONLY, true);
let ctx = SessionContext::with_config(config);
let sql = "EXPLAIN select count(*) from (values ('a', 1, 100), ('a', 2, 150)) as t (c1,c2,c3)";
let actual = execute(&ctx, sql).await;
let actual = normalize_vec_for_explain(actual);

let expected = vec![vec![
"physical_plan",
"ProjectionExec: expr=[COUNT(UInt8(1))@0 as COUNT(UInt8(1))]\
\n ProjectionExec: expr=[2 as COUNT(UInt8(1))]\
\n EmptyExec: produce_one_row=true\
\n",
]];
assert_eq!(expected, actual);
}
2 changes: 2 additions & 0 deletions docs/source/user-guide/configs.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,6 @@ Environment variables are read during `SessionConfig` initialisation so they mus
| datafusion.execution.batch_size | UInt64 | 8192 | Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would results in too much metadata memory consumption. |
| datafusion.execution.coalesce_batches | Boolean | true | When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting 'datafusion.execution.coalesce_target_batch_size'. |
| datafusion.execution.coalesce_target_batch_size | UInt64 | 4096 | Target batch size when coalescing batches. Uses in conjunction with the configuration setting 'datafusion.execution.coalesce_batches'. |
| datafusion.explain.logical_plan_only | Boolean | false | When set to true, the explain statement will only print logical plans. |
| datafusion.explain.physical_plan_only | Boolean | false | When set to true, the explain statement will only print physical plans. |
| datafusion.optimizer.filter_null_join_keys | Boolean | false | When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down. |

0 comments on commit 1b8117e

Please sign in to comment.